mirror of
https://github.com/valitydev/file-storage.git
synced 2024-11-06 00:35:22 +00:00
BJ-323: update application logic
now application can to use cyrillic file names fix tests logic
This commit is contained in:
a.karlov
parent
113769201c
commit
f3489cca37
@ -1,2 +1,3 @@
|
||||
# file-storage
|
||||
file-storage service
|
||||
Прокси, связывающий rbkmoney сервисы и ceph. Имплементирует Amazon S3 клиент, который используется, как клиент для подключения к ceph.
|
||||
Ceph используется для сохранения файлов.
|
||||
|
||||
2
pom.xml
2
pom.xml
@ -26,7 +26,7 @@
|
||||
<dockerfile.base.service.tag>22c57470c4fc47161894f036b7cf9d70f42b75f5</dockerfile.base.service.tag>
|
||||
<shared.resources.version>0.2.1</shared.resources.version>
|
||||
<woody.thrift.version>1.1.15</woody.thrift.version>
|
||||
<file.storage.proto.version>1.10-f44896e</file.storage.proto.version>
|
||||
<file.storage.proto.version>1.14-b375587</file.storage.proto.version>
|
||||
<geck.version>0.6.8</geck.version>
|
||||
</properties>
|
||||
|
||||
|
||||
@ -18,7 +18,6 @@ import java.net.URL;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.rbkmoney.file.storage.util.CheckerUtil.checkFileName;
|
||||
import static com.rbkmoney.file.storage.util.CheckerUtil.checkString;
|
||||
|
||||
@RequiredArgsConstructor
|
||||
@ -28,14 +27,12 @@ public class FileStorageHandler implements FileStorageSrv.Iface {
|
||||
private final StorageService storageService;
|
||||
|
||||
@Override
|
||||
public NewFileResult createNewFile(String fileName, Map<String, Value> metadata, String expiresAt) throws TException {
|
||||
public NewFileResult createNewFile(Map<String, Value> metadata, String expiresAt) throws TException {
|
||||
try {
|
||||
log.info("Request createNewFile fileName: {}, metadata: {}, expiresAt: {}", fileName, metadata, expiresAt);
|
||||
checkString(fileName, "Bad request parameter, fileName required and not empty arg");
|
||||
checkFileName(fileName, "Bad request parameter, enter the correct fileName");
|
||||
log.info("Request createNewFile metadata: {}, expiresAt: {}", metadata, expiresAt);
|
||||
// stringToInstant уже содержит проверки аргемента
|
||||
Instant instant = TypeUtil.stringToInstant(expiresAt);
|
||||
NewFileResult newFile = storageService.createNewFile(fileName, metadata, instant);
|
||||
NewFileResult newFile = storageService.createNewFile(metadata, instant);
|
||||
log.info("Response: newFileResult: {}", newFile);
|
||||
return newFile;
|
||||
} catch (StorageFileNotFoundException e) {
|
||||
@ -51,14 +48,14 @@ public class FileStorageHandler implements FileStorageSrv.Iface {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String generateDownloadUrl(String fileDataId, String expiresAt) throws TException {
|
||||
public String generateDownloadUrl(String id, String expiresAt) throws TException {
|
||||
try {
|
||||
log.info("Request generateDownloadUrl fileDataId: {}, expiresAt: {}", fileDataId, expiresAt);
|
||||
checkString(fileDataId, "Bad request parameter, fileDataId required and not empty arg");
|
||||
log.info("Request generateDownloadUrl id: {}, expiresAt: {}", id, expiresAt);
|
||||
checkString(id, "Bad request parameter, id required and not empty arg");
|
||||
checkString(expiresAt, "Bad request parameter, expiresAt required and not empty arg");
|
||||
// stringToInstant уже содержит проверки аргемента
|
||||
Instant instant = TypeUtil.stringToInstant(expiresAt);
|
||||
URL url = storageService.generateDownloadUrl(fileDataId, instant);
|
||||
URL url = storageService.generateDownloadUrl(id, instant);
|
||||
log.info("Response: url: {}", url);
|
||||
return url.toString();
|
||||
} catch (StorageFileNotFoundException e) {
|
||||
@ -74,11 +71,11 @@ public class FileStorageHandler implements FileStorageSrv.Iface {
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileData getFileData(String fileDataId) throws TException {
|
||||
public FileData getFileData(String id) throws TException {
|
||||
try {
|
||||
log.info("Request getFileData fileDataId: {}", fileDataId);
|
||||
checkString(fileDataId, "Bad request parameter, fileDataId required and not empty arg");
|
||||
FileData fileData = storageService.getFileData(fileDataId);
|
||||
log.info("Request getFileData id: {}", id);
|
||||
checkString(id, "Bad request parameter, id required and not empty arg");
|
||||
FileData fileData = storageService.getFileData(id);
|
||||
log.info("Response: fileData: {}", fileData);
|
||||
return fileData;
|
||||
} catch (StorageFileNotFoundException e) {
|
||||
|
||||
@ -6,12 +6,14 @@ import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.model.*;
|
||||
import com.amazonaws.services.s3.transfer.TransferManager;
|
||||
import com.amazonaws.services.s3.transfer.Upload;
|
||||
import com.rbkmoney.damsel.msgpack.Value;
|
||||
import com.rbkmoney.file.storage.FileData;
|
||||
import com.rbkmoney.file.storage.NewFileResult;
|
||||
import com.rbkmoney.file.storage.configuration.properties.StorageProperties;
|
||||
import com.rbkmoney.file.storage.service.exception.StorageException;
|
||||
import com.rbkmoney.file.storage.service.exception.StorageFileNotFoundException;
|
||||
import com.rbkmoney.file.storage.util.DamselUtil;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
@ -30,11 +32,11 @@ import java.util.stream.Collectors;
|
||||
@RequiredArgsConstructor
|
||||
public class AmazonS3StorageService implements StorageService {
|
||||
|
||||
private static final String FILEDATA_FILE_ID = "x-rbkmoney-filedata-file-id";
|
||||
private static final String FILEDATA_FILEDATA_ID = "x-rbkmoney-filedata-filedata-id";
|
||||
private static final String FILEDATA_FILE_NAME = "x-rbkmoney-filedata-file-name";
|
||||
private static final String FILEDATA_CREATED_AT = "x-rbkmoney-filedata-created-at";
|
||||
private static final String FILEDATA_METADATA = "x-rbkmoney-filedata-metadata-";
|
||||
private static final String ID = "x-rbkmoney-id";
|
||||
private static final String FILE_ID = "x-rbkmoney-file-id";
|
||||
private static final String CREATED_AT = "x-rbkmoney-created-at";
|
||||
private static final String METADATA = "x-rbkmoney-metadata-";
|
||||
private static final String FILENAME_PARAM = "filename=";
|
||||
|
||||
private final TransferManager transferManager;
|
||||
private final AmazonS3 s3Client;
|
||||
@ -51,43 +53,35 @@ public class AmazonS3StorageService implements StorageService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public NewFileResult createNewFile(String fileName, Map<String, com.rbkmoney.damsel.msgpack.Value> metadata, Instant expirationTime) throws StorageException {
|
||||
log.info("Trying to create new file to storage, filename='{}', bucketId='{}'", fileName, bucketName);
|
||||
public NewFileResult createNewFile(Map<String, Value> metadata, Instant expirationTime) throws StorageException {
|
||||
log.info("Trying to create new file to storage, bucketId='{}'", bucketName);
|
||||
|
||||
InputStream emptyContent = getEmptyContent();
|
||||
String id = createId();
|
||||
String fileId = createId();
|
||||
FileDto fileDto = new FileDto(
|
||||
id,
|
||||
fileId,
|
||||
Instant.now().toString(),
|
||||
metadata
|
||||
);
|
||||
|
||||
try {
|
||||
InputStream emptyContent = getEmptyContent();
|
||||
// в хранилище записывается неизменяемый фейковый файл с метаданными,
|
||||
// в котором находится ссылка на реальный файл
|
||||
uploadRequest(id, fileDto, emptyContent);
|
||||
|
||||
String fileDataId = getId();
|
||||
String fileId = getId();
|
||||
String createdAt = Instant.now().toString();
|
||||
|
||||
FileData fileData = new FileData(
|
||||
fileDataId,
|
||||
fileId,
|
||||
fileName,
|
||||
createdAt,
|
||||
metadata
|
||||
);
|
||||
|
||||
// записываем в хранилище пустой файл с метаданными по ключу fileDataId
|
||||
uploadRequest(fileDataId, fileData, emptyContent);
|
||||
|
||||
// генерируем ссылку на запись файла в хранилище напрямую в цеф по ключу fileId
|
||||
// генерируем ссылку на выгрузку файла в хранилище напрямую в цеф по ключу fileId
|
||||
URL uploadUrl = generateUploadUrl(fileId, expirationTime);
|
||||
|
||||
log.info(
|
||||
"File have been successfully created, fileId='{}', bucketId='{}', filename='{}'",
|
||||
fileId,
|
||||
bucketName,
|
||||
fileName
|
||||
);
|
||||
log.info("File have been successfully created, id='{}', bucketId='{}'", id, bucketName);
|
||||
|
||||
return new NewFileResult(uploadUrl.toString(), fileData);
|
||||
return new NewFileResult(id, uploadUrl.toString());
|
||||
} catch (AmazonClientException ex) {
|
||||
throw new StorageException(
|
||||
String.format(
|
||||
"Failed to create new file to storage, filename='%s', bucketId='%s'",
|
||||
fileName,
|
||||
"Failed to create new file to storage, id='%s', bucketId='%s'",
|
||||
id,
|
||||
bucketName
|
||||
),
|
||||
ex
|
||||
@ -96,14 +90,36 @@ public class AmazonS3StorageService implements StorageService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public URL generateDownloadUrl(String fileDataId, Instant expirationTime) throws StorageException {
|
||||
String fileId = getValidFileData(fileDataId).getFileId();
|
||||
public URL generateDownloadUrl(String id, Instant expirationTime) throws StorageException {
|
||||
String fileId = getFileDto(id).getFileId();
|
||||
// генерируем ссылку на загрузку файла из хранилища напрямую в цеф по ключу fileId
|
||||
return generatePresignedUrl(fileId, expirationTime, HttpMethod.GET);
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileData getFileData(String fileDataId) throws StorageException {
|
||||
return getValidFileData(fileDataId);
|
||||
public FileData getFileData(String id) throws StorageException {
|
||||
FileDto fileDto = getFileDto(id);
|
||||
|
||||
S3Object object = getS3Object(fileDto.getFileId());
|
||||
|
||||
String fileName;
|
||||
try {
|
||||
String contentDisposition = object.getObjectMetadata().getContentDisposition();
|
||||
int fileNameIndex = contentDisposition.lastIndexOf(FILENAME_PARAM) + FILENAME_PARAM.length();
|
||||
|
||||
fileName = contentDisposition.substring(fileNameIndex);
|
||||
} catch (NullPointerException ex) {
|
||||
throw new StorageException(
|
||||
String.format(
|
||||
"Failed to extract fileName, id='%s', bucketId='%s'",
|
||||
id,
|
||||
bucketName
|
||||
),
|
||||
ex
|
||||
);
|
||||
}
|
||||
|
||||
return new FileData(fileDto.getId(), fileName, fileDto.getCreatedAt(), fileDto.getMetadata());
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
@ -111,19 +127,19 @@ public class AmazonS3StorageService implements StorageService {
|
||||
transferManager.shutdownNow(true);
|
||||
}
|
||||
|
||||
private S3Object getS3Object(String fileDataId) throws StorageException {
|
||||
private S3Object getS3Object(String id) throws StorageException {
|
||||
try {
|
||||
log.info(
|
||||
"Trying to get file from storage, fileDataId='{}', bucketId='{}'",
|
||||
fileDataId,
|
||||
"Trying to get file from storage, id='{}', bucketId='{}'",
|
||||
id,
|
||||
bucketName
|
||||
);
|
||||
GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, fileDataId);
|
||||
GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, id);
|
||||
S3Object object = s3Client.getObject(getObjectRequest);
|
||||
checkNotNull(object, fileDataId, "File");
|
||||
checkNotNull(object, id, "File");
|
||||
log.info(
|
||||
"File have been successfully got from storage, fileDataId='{}', bucketId='{}'",
|
||||
fileDataId,
|
||||
"File have been successfully got from storage, id='{}', bucketId='{}'",
|
||||
id,
|
||||
bucketName
|
||||
);
|
||||
return object;
|
||||
@ -131,7 +147,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
throw new StorageException(
|
||||
String.format(
|
||||
"Failed to get file from storage, fileDataId='%s', bucketId='%s'",
|
||||
fileDataId,
|
||||
id,
|
||||
bucketName
|
||||
),
|
||||
ex
|
||||
@ -139,47 +155,45 @@ public class AmazonS3StorageService implements StorageService {
|
||||
}
|
||||
}
|
||||
|
||||
private void checkFileStatus(S3Object s3Object) throws StorageFileNotFoundException {
|
||||
log.info("Check file expiration and uploaded status: ETag='{}'", s3Object.getObjectMetadata().getETag());
|
||||
private void checkRealFileStatus(S3Object s3Object) throws StorageFileNotFoundException {
|
||||
log.info("Check real file expiration and uploaded status: ETag='{}'", s3Object.getObjectMetadata().getETag());
|
||||
ObjectMetadata objectMetadata = s3Object.getObjectMetadata();
|
||||
|
||||
String fileId = getFileIdFromObjectMetadata(objectMetadata);
|
||||
if (s3Client.doesObjectExist(bucketName, fileId)) {
|
||||
log.info("File was uploaded: ETag='{}'", s3Object.getObjectMetadata().getETag());
|
||||
log.info("Real file was uploaded: ETag='{}'", s3Object.getObjectMetadata().getETag());
|
||||
return;
|
||||
}
|
||||
|
||||
// если файл не соотвествует условиям, блокируем доступ к нему
|
||||
throw new StorageFileNotFoundException(String.format("File not found: fileId='%s', bucketId='%s', create a new file", s3Object.getKey(), bucketName));
|
||||
throw new StorageFileNotFoundException(String.format("Real file not found: id='%s', bucketId='%s', create a new file", s3Object.getKey(), bucketName));
|
||||
}
|
||||
|
||||
private FileData extractFileData(ObjectMetadata objectMetadata) {
|
||||
log.info("Trying to extract metadata from storage: ETag='{}'", objectMetadata.getETag());
|
||||
String fileId = getUserMetadataParameter(objectMetadata, FILEDATA_FILE_ID);
|
||||
String fileDataId = getUserMetadataParameter(objectMetadata, FILEDATA_FILEDATA_ID);
|
||||
String fileName = getUserMetadataParameter(objectMetadata, FILEDATA_FILE_NAME);
|
||||
String createdAt = getUserMetadataParameter(objectMetadata, FILEDATA_CREATED_AT);
|
||||
|
||||
private FileDto getFileDtoByFakeFile(ObjectMetadata objectMetadata) {
|
||||
log.info("Trying to extract real file metadata by fake file from storage: ETag='{}'", objectMetadata.getETag());
|
||||
String id = getUserMetadataParameter(objectMetadata, ID);
|
||||
String fileId = getFileIdFromObjectMetadata(objectMetadata);
|
||||
String createdAt = getUserMetadataParameter(objectMetadata, CREATED_AT);
|
||||
Map<String, com.rbkmoney.damsel.msgpack.Value> metadata = objectMetadata.getUserMetadata().entrySet().stream()
|
||||
.filter(entry -> entry.getKey().startsWith(FILEDATA_METADATA) && entry.getValue() != null)
|
||||
.filter(entry -> entry.getKey().startsWith(METADATA) && entry.getValue() != null)
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
o -> o.getKey().substring(FILEDATA_METADATA.length()),
|
||||
o -> o.getKey().substring(METADATA.length()),
|
||||
o -> DamselUtil.fromJson(o.getValue(), com.rbkmoney.damsel.msgpack.Value.class)
|
||||
)
|
||||
);
|
||||
log.info(
|
||||
"Metadata have been successfully extracted from storage, fileId='{}', bucketId='{}'",
|
||||
fileId,
|
||||
"Real file metadata have been successfully extracted by fake file from storage, id='{}', bucketId='{}'",
|
||||
id,
|
||||
bucketName
|
||||
);
|
||||
return new FileData(fileDataId, fileId, fileName, createdAt, metadata);
|
||||
return new FileDto(id, fileId, createdAt, metadata);
|
||||
}
|
||||
|
||||
private URL generatePresignedUrl(String fileId, Instant expirationTime, HttpMethod httpMethod) throws StorageException {
|
||||
try {
|
||||
log.info(
|
||||
"Trying to generate presigned url, fileId='{}', bucketId='{}', expirationTime='{}', httpMethod='{}'",
|
||||
"Trying to generate presigned url for real file, fileId='{}', bucketId='{}', expirationTime='{}', httpMethod='{}'",
|
||||
fileId,
|
||||
bucketName,
|
||||
expirationTime,
|
||||
@ -218,8 +232,8 @@ public class AmazonS3StorageService implements StorageService {
|
||||
return new ByteArrayInputStream(new byte[0]);
|
||||
}
|
||||
|
||||
private void uploadRequest(String id, FileData fileData, InputStream inputStream) throws AmazonClientException {
|
||||
PutObjectRequest putObjectRequest = createS3Request(id, fileData, inputStream);
|
||||
private void uploadRequest(String id, FileDto fileDto, InputStream inputStream) throws AmazonClientException {
|
||||
PutObjectRequest putObjectRequest = createS3Request(id, fileDto, inputStream);
|
||||
|
||||
Upload upload = transferManager.upload(putObjectRequest);
|
||||
try {
|
||||
@ -229,33 +243,32 @@ public class AmazonS3StorageService implements StorageService {
|
||||
}
|
||||
}
|
||||
|
||||
private PutObjectRequest createS3Request(String id, FileData fileData, InputStream inputStream) {
|
||||
private PutObjectRequest createS3Request(String id, FileDto fileDto, InputStream inputStream) {
|
||||
return new PutObjectRequest(
|
||||
bucketName,
|
||||
id,
|
||||
inputStream,
|
||||
createObjectMetadata(fileData)
|
||||
createObjectMetadata(fileDto)
|
||||
);
|
||||
}
|
||||
|
||||
private ObjectMetadata createObjectMetadata(FileData fileData) {
|
||||
private ObjectMetadata createObjectMetadata(FileDto fileDto) {
|
||||
ObjectMetadata objectMetadata = new ObjectMetadata();
|
||||
objectMetadata.setContentDisposition("attachment;filename=" + fileData.getFileName());
|
||||
// filedata parameters
|
||||
objectMetadata.addUserMetadata(FILEDATA_FILEDATA_ID, fileData.getFiledataId());
|
||||
objectMetadata.addUserMetadata(FILEDATA_FILE_ID, fileData.getFileId());
|
||||
objectMetadata.addUserMetadata(FILEDATA_FILE_NAME, fileData.getFileName());
|
||||
objectMetadata.addUserMetadata(FILEDATA_CREATED_AT, fileData.getCreatedAt());
|
||||
fileData.getMetadata().forEach(
|
||||
(key, value) -> objectMetadata.addUserMetadata(FILEDATA_METADATA + key, DamselUtil.toJsonString(value))
|
||||
objectMetadata.addUserMetadata(ID, fileDto.getId());
|
||||
objectMetadata.addUserMetadata(FILE_ID, fileDto.getFileId());
|
||||
objectMetadata.addUserMetadata(CREATED_AT, fileDto.getCreatedAt());
|
||||
fileDto.getMetadata().forEach(
|
||||
(key, value) -> objectMetadata.addUserMetadata(METADATA + key, DamselUtil.toJsonString(value))
|
||||
);
|
||||
return objectMetadata;
|
||||
}
|
||||
|
||||
private FileData getValidFileData(String fileDataId) throws StorageException {
|
||||
S3Object s3Object = getS3Object(fileDataId);
|
||||
checkFileStatus(s3Object);
|
||||
return extractFileData(s3Object.getObjectMetadata());
|
||||
private FileDto getFileDto(String id) throws StorageException {
|
||||
// извлечение фейкового файла с метаданными о реальном файле
|
||||
S3Object s3Object = getS3Object(id);
|
||||
// функция возвращает метаданные только в случае, если реальный файл был уже выгружен в хранилище
|
||||
checkRealFileStatus(s3Object);
|
||||
return getFileDtoByFakeFile(s3Object.getObjectMetadata());
|
||||
}
|
||||
|
||||
private URL generateUploadUrl(String fileId, Instant expirationTime) throws StorageException {
|
||||
@ -263,7 +276,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
}
|
||||
|
||||
private String getFileIdFromObjectMetadata(ObjectMetadata objectMetadata) {
|
||||
return getUserMetadataParameter(objectMetadata, FILEDATA_FILE_ID);
|
||||
return getUserMetadataParameter(objectMetadata, FILE_ID);
|
||||
}
|
||||
|
||||
private String getUserMetadataParameter(ObjectMetadata objectMetadata, String key) throws StorageException {
|
||||
@ -271,7 +284,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
.orElseThrow(() -> new StorageException("Failed to extract user metadata parameter, " + key + " is null"));
|
||||
}
|
||||
|
||||
private String getId() {
|
||||
private String createId() {
|
||||
return UUID.randomUUID().toString();
|
||||
}
|
||||
|
||||
@ -280,4 +293,15 @@ public class AmazonS3StorageService implements StorageService {
|
||||
throw new StorageFileNotFoundException(String.format(objectType + " is null, fileId='%s', bucketId='%s'", fileId, bucketName));
|
||||
}
|
||||
}
|
||||
|
||||
@RequiredArgsConstructor
|
||||
@Getter
|
||||
private class FileDto {
|
||||
|
||||
private final String id;
|
||||
private final String fileId;
|
||||
private final String createdAt;
|
||||
private final Map<String, Value> metadata;
|
||||
|
||||
}
|
||||
}
|
||||
@ -11,10 +11,10 @@ import java.util.Map;
|
||||
|
||||
public interface StorageService {
|
||||
|
||||
NewFileResult createNewFile(String fileName, Map<String, Value> metadata, Instant expirationTime) throws StorageException;
|
||||
NewFileResult createNewFile(Map<String, Value> metadata, Instant expirationTime) throws StorageException;
|
||||
|
||||
URL generateDownloadUrl(String fileDataId, Instant expirationTime) throws StorageException;
|
||||
URL generateDownloadUrl(String id, Instant expirationTime) throws StorageException;
|
||||
|
||||
FileData getFileData(String fileDataId) throws StorageException;
|
||||
FileData getFileData(String id) throws StorageException;
|
||||
|
||||
}
|
||||
|
||||
@ -2,11 +2,22 @@ server.port=@server.port@
|
||||
spring.application.name=@project.name@
|
||||
info.version=@project.version@
|
||||
info.stage=dev
|
||||
# --
|
||||
spring.servlet.multipart.max-file-size=10MB
|
||||
spring.servlet.multipart.max-request-size=10MB
|
||||
spring.servlet.multipart.enabled=true
|
||||
# --
|
||||
storage.endpoint=localhost
|
||||
storage.signingRegion=RU
|
||||
storage.clientProtocol=HTTP
|
||||
storage.clientMaxErrorRetry=10
|
||||
storage.bucketName="files"
|
||||
# ----
|
||||
# for up local test version
|
||||
#storage.endpoint=localhost:32827
|
||||
#storage.signingRegion=RU
|
||||
#storage.accessKey=test
|
||||
#storage.secretKey=test
|
||||
#storage.clientProtocol=HTTP
|
||||
#storage.clientMaxErrorRetry=10
|
||||
#storage.bucketName=TEST
|
||||
|
||||
@ -16,10 +16,8 @@ import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.HttpWaitStrategy;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.net.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
@ -46,6 +44,7 @@ public abstract class AbstractIntegrationTest {
|
||||
|
||||
protected FileStorageSrv.Iface client;
|
||||
|
||||
// for up local test version comment this ClassRule
|
||||
@ClassRule
|
||||
public static GenericContainer cephContainer = new GenericContainer("dr.rbkmoney.com/ceph-demo:latest")
|
||||
.withEnv("RGW_NAME", "localhost")
|
||||
@ -66,8 +65,9 @@ public abstract class AbstractIntegrationTest {
|
||||
@Override
|
||||
public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
|
||||
TestPropertyValues.of(
|
||||
// for up local test version comment this row
|
||||
"storage.endpoint=" + cephContainer.getContainerIpAddress() + ":" + cephContainer.getMappedPort(80),
|
||||
// в случае, если поднят локальный сторедж в контейнере
|
||||
// for up local test version uncomment this row
|
||||
// "storage.endpoint=localhost:32827",
|
||||
"storage.signingRegion=" + SIGNING_REGION,
|
||||
"storage.accessKey=" + AWS_ACCESS_KEY,
|
||||
@ -100,11 +100,17 @@ public abstract class AbstractIntegrationTest {
|
||||
return ZoneOffset.systemDefault().getRules().getOffset(LocalDateTime.now());
|
||||
}
|
||||
|
||||
protected HttpURLConnection getHttpURLConnection(URL url, boolean doOutput, String method) throws IOException {
|
||||
protected HttpURLConnection getHttpURLConnection(URL url, String method, boolean doOutput) throws IOException {
|
||||
return getHttpURLConnection(url, method, null, doOutput);
|
||||
}
|
||||
|
||||
protected HttpURLConnection getHttpURLConnection(URL url, String method, String fileName, boolean doOutput) throws IOException {
|
||||
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
|
||||
connection.setDoOutput(doOutput);
|
||||
connection.setRequestMethod(method);
|
||||
if (fileName != null) {
|
||||
connection.setRequestProperty("Content-Disposition", "attachment;filename=" + URLEncoder.encode(fileName, StandardCharsets.UTF_8.name()));
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -10,6 +10,8 @@ import java.io.InputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.net.URLDecoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
@ -24,7 +26,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
// все тесты в 1 классе , чтобы сэкономить время на поднятии тест контейнера
|
||||
public class FileStorageTest extends AbstractIntegrationTest {
|
||||
|
||||
private static final String TEST_DATA = "test";
|
||||
private static final String FILE_DATA = "test";
|
||||
private static final String FILE_NAME = "rainbow-champion";
|
||||
|
||||
@Test
|
||||
public void uploadAndDownloadFileFromStorageTest() throws IOException, TException {
|
||||
@ -35,13 +38,13 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
try {
|
||||
// создание нового файла
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), expirationTime);
|
||||
uploadTestData(fileResult);
|
||||
NewFileResult fileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
uploadTestData(fileResult, FILE_NAME, FILE_DATA);
|
||||
|
||||
// генерация url с доступом только для загрузки
|
||||
URL downloadUrl = new URL(client.generateDownloadUrl(fileResult.getFileData().getFiledataId(), expirationTime));
|
||||
URL downloadUrl = new URL(client.generateDownloadUrl(fileResult.getId(), expirationTime));
|
||||
|
||||
HttpURLConnection downloadUrlConnection = getHttpURLConnection(downloadUrl, false, "GET");
|
||||
HttpURLConnection downloadUrlConnection = getHttpURLConnection(downloadUrl, "GET", false);
|
||||
InputStream inputStream = downloadUrlConnection.getInputStream();
|
||||
|
||||
// чтение записанного файла из хранилища
|
||||
@ -51,7 +54,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
assertEquals(Files.readAllLines(testFile), Collections.emptyList());
|
||||
|
||||
// запись тестовых данных в пустой testFile
|
||||
Files.write(testFile, TEST_DATA.getBytes());
|
||||
Files.write(testFile, FILE_DATA.getBytes());
|
||||
assertEquals(Files.readAllLines(testFile), Files.readAllLines(testActualFile));
|
||||
} finally {
|
||||
Files.deleteIfExists(testFile);
|
||||
@ -63,9 +66,9 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
public void accessErrorToMethodsTest() throws IOException, TException {
|
||||
// создание файла с доступом к файлу на день
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), expirationTime);
|
||||
NewFileResult fileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
String fileDataId = fileResult.getFileData().getFiledataId();
|
||||
String fileDataId = fileResult.getId();
|
||||
|
||||
// ошибка доступа - файла не существует, тк не было upload
|
||||
assertThrows(FileNotFound.class, () -> client.generateDownloadUrl(fileDataId, expirationTime));
|
||||
@ -76,53 +79,53 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
public void uploadUrlConnectionAccessTest() throws IOException, TException {
|
||||
// создание файла с доступом к файлу на день
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), expirationTime);
|
||||
NewFileResult fileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
URL uploadUrl = new URL(fileResult.getUploadUrl());
|
||||
|
||||
// ошибка при запросе по url методом get
|
||||
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(uploadUrl, false, "GET").getResponseCode());
|
||||
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(uploadUrl, "GET", false).getResponseCode());
|
||||
|
||||
// Length Required при запросе по url методом put
|
||||
assertEquals(HttpStatus.LENGTH_REQUIRED.value(), getHttpURLConnection(uploadUrl, true, "PUT").getResponseCode());
|
||||
assertEquals(HttpStatus.LENGTH_REQUIRED.value(), getHttpURLConnection(uploadUrl, "PUT", FILE_NAME, true).getResponseCode());
|
||||
|
||||
uploadTestData(fileResult);
|
||||
uploadTestData(fileResult, FILE_NAME, FILE_DATA);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void downloadUrlConnectionAccessTest() throws IOException, TException {
|
||||
// создание файла с доступом к файлу на день
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult fileResult = client.createNewFile("test_file", Collections.emptyMap(), expirationTime);
|
||||
NewFileResult fileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
String fileDataId = fileResult.getFileData().getFiledataId();
|
||||
String fileDataId = fileResult.getId();
|
||||
|
||||
// upload тестовых данных в хранилище
|
||||
uploadTestData(fileResult);
|
||||
uploadTestData(fileResult, FILE_NAME, FILE_DATA);
|
||||
|
||||
// генерация url с доступом только для загрузки
|
||||
URL url = new URL(client.generateDownloadUrl(fileDataId, expirationTime));
|
||||
|
||||
// с данной ссылкой нельзя записывать
|
||||
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, true, "PUT").getResponseCode());
|
||||
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, "PUT", FILE_NAME, true).getResponseCode());
|
||||
|
||||
// можно читать
|
||||
assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, false, "GET").getResponseCode());
|
||||
assertEquals(HttpStatus.OK.value(), getHttpURLConnection(url, "GET", false).getResponseCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void expirationTimeTest() throws TException, InterruptedException, IOException {
|
||||
// создание файла с доступом к файлу на день
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult validFileResult = client.createNewFile("test_file", Collections.emptyMap(), expirationTime);
|
||||
NewFileResult validFileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
String validFileDataId = validFileResult.getFileData().getFiledataId();
|
||||
String validFileDataId = validFileResult.getId();
|
||||
|
||||
// задержка перед upload для теста expiration
|
||||
Thread.sleep(1000);
|
||||
|
||||
// сохранение тестовых данных в хранилище
|
||||
uploadTestData(validFileResult);
|
||||
uploadTestData(validFileResult, FILE_NAME, FILE_DATA);
|
||||
|
||||
// доступ есть
|
||||
client.getFileData(validFileDataId);
|
||||
@ -130,9 +133,9 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
|
||||
// - - - - - сделаем задержку больше expiration
|
||||
// создание файла с доступом к файлу на секунду
|
||||
NewFileResult throwingFileResult = client.createNewFile("test_file", Collections.emptyMap(), generateCurrentTimePlusSecond().toString());
|
||||
NewFileResult throwingFileResult = client.createNewFile(Collections.emptyMap(), generateCurrentTimePlusSecond().toString());
|
||||
|
||||
String throwingFileDataId = throwingFileResult.getFileData().getFiledataId();
|
||||
String throwingFileDataId = throwingFileResult.getId();
|
||||
|
||||
// ошибка доступа - файла не существует, тк не было upload
|
||||
assertThrows(FileNotFound.class, () -> client.generateDownloadUrl(throwingFileDataId, expirationTime));
|
||||
@ -142,7 +145,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
Thread.sleep(2000);
|
||||
|
||||
// сохранение тестовых данных в хранилище вызывает ошибку доступа
|
||||
assertThrows(AssertionError.class, () -> uploadTestData(throwingFileResult));
|
||||
assertThrows(AssertionError.class, () -> uploadTestData(throwingFileResult, FILE_NAME, FILE_DATA));
|
||||
|
||||
// ошибка доступа
|
||||
assertThrows(FileNotFound.class, () -> client.getFileData(throwingFileDataId));
|
||||
@ -152,33 +155,48 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
@Test
|
||||
public void extractMetadataTest() throws TException, IOException {
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
String fileName = "test_file";
|
||||
Map<String, Value> metadata = new HashMap<String, Value>() {{
|
||||
put("key1", Value.b(true));
|
||||
put("key2", Value.i(1));
|
||||
put("key3", Value.flt(1));
|
||||
put("key4", Value.arr(new ArrayList<>()));
|
||||
put("key5", Value.str(TEST_DATA));
|
||||
put("key5", Value.str(FILE_DATA));
|
||||
put("key6", Value.bin(new byte[]{}));
|
||||
}};
|
||||
NewFileResult fileResult = client.createNewFile(fileName, metadata, expirationTime);
|
||||
uploadTestData(fileResult);
|
||||
|
||||
assertEquals(fileResult.getFileData().getFileName(), fileName);
|
||||
NewFileResult fileResult = client.createNewFile(metadata, expirationTime);
|
||||
uploadTestData(fileResult, FILE_NAME, FILE_DATA);
|
||||
|
||||
FileData fileData = client.getFileData(fileResult.getFileData().getFiledataId());
|
||||
FileData fileData = client.getFileData(fileResult.getId());
|
||||
|
||||
assertEquals(fileData, fileResult.getFileData());
|
||||
assertEquals(fileData.getMetadata(), metadata);
|
||||
}
|
||||
|
||||
private void uploadTestData(final NewFileResult fileResult) throws IOException {
|
||||
@Test
|
||||
public void fileNameCyrillicTest() throws TException, IOException {
|
||||
// создание файла с доступом к файлу на день
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult fileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
// upload тестовых данных в хранилище
|
||||
String fileName = "csgo-лучше-чем-1.6";
|
||||
uploadTestData(fileResult, fileName, FILE_DATA);
|
||||
|
||||
FileData fileData = client.getFileData(fileResult.getId());
|
||||
|
||||
// тут используется энкодер\декодер, потому что apache http клиент менять кодировку.
|
||||
// при аплоаде напрямую по uploadUrl в ceph такой проблемы нет
|
||||
assertEquals(fileName, URLDecoder.decode(fileData.getFileName(), StandardCharsets.UTF_8.name()));
|
||||
}
|
||||
|
||||
private void uploadTestData(NewFileResult fileResult, String fileName, String testData) throws IOException {
|
||||
// запись данных методом put
|
||||
URL uploadUrl = new URL(fileResult.getUploadUrl());
|
||||
|
||||
HttpURLConnection uploadUrlConnection = getHttpURLConnection(uploadUrl, true, "PUT");
|
||||
HttpURLConnection uploadUrlConnection = getHttpURLConnection(uploadUrl, "PUT", fileName, true);
|
||||
|
||||
OutputStreamWriter out = new OutputStreamWriter(uploadUrlConnection.getOutputStream());
|
||||
out.write(TEST_DATA);
|
||||
out.write(testData);
|
||||
out.close();
|
||||
|
||||
// чтобы завершить загрузку вызываем getResponseCode
|
||||
|
||||
Loading…
Reference in New Issue
Block a user