mirror of
https://github.com/valitydev/file-storage.git
synced 2024-11-06 00:35:22 +00:00
update protocol version
replace id on fileDataId for sync with protocol
This commit is contained in:
a.karlov
parent
94df77fce7
commit
bff37339fd
2
pom.xml
2
pom.xml
@ -26,7 +26,7 @@
|
||||
<dockerfile.base.service.tag>22c57470c4fc47161894f036b7cf9d70f42b75f5</dockerfile.base.service.tag>
|
||||
<shared.resources.version>0.2.1</shared.resources.version>
|
||||
<woody.thrift.version>1.1.15</woody.thrift.version>
|
||||
<file.storage.proto.version>1.19-13d8e22</file.storage.proto.version>
|
||||
<file.storage.proto.version>1.24-a77c0c4</file.storage.proto.version>
|
||||
<geck.version>0.6.8</geck.version>
|
||||
</properties>
|
||||
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
package com.rbkmoney.file.storage.handler;
|
||||
|
||||
import com.rbkmoney.damsel.msgpack.Value;
|
||||
import com.rbkmoney.file.storage.FileData;
|
||||
import com.rbkmoney.file.storage.FileNotFound;
|
||||
import com.rbkmoney.file.storage.FileStorageSrv;
|
||||
import com.rbkmoney.file.storage.NewFileResult;
|
||||
import com.rbkmoney.file.storage.msgpack.Value;
|
||||
import com.rbkmoney.file.storage.service.StorageService;
|
||||
import com.rbkmoney.file.storage.service.exception.StorageException;
|
||||
import com.rbkmoney.file.storage.service.exception.StorageFileNotFoundException;
|
||||
@ -48,14 +48,14 @@ public class FileStorageHandler implements FileStorageSrv.Iface {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String generateDownloadUrl(String id, String expiresAt) throws TException {
|
||||
public String generateDownloadUrl(String fileDataId, String expiresAt) throws TException {
|
||||
try {
|
||||
log.info("Request generateDownloadUrl id: {}, expiresAt: {}", id, expiresAt);
|
||||
checkString(id, "Bad request parameter, id required and not empty arg");
|
||||
log.info("Request generateDownloadUrl fileDataId: {}, expiresAt: {}", fileDataId, expiresAt);
|
||||
checkString(fileDataId, "Bad request parameter, fileDataId required and not empty arg");
|
||||
checkString(expiresAt, "Bad request parameter, expiresAt required and not empty arg");
|
||||
// stringToInstant уже содержит проверки аргемента
|
||||
Instant instant = TypeUtil.stringToInstant(expiresAt);
|
||||
URL url = storageService.generateDownloadUrl(id, instant);
|
||||
URL url = storageService.generateDownloadUrl(fileDataId, instant);
|
||||
log.info("Response: url: {}", url);
|
||||
return url.toString();
|
||||
} catch (StorageFileNotFoundException e) {
|
||||
@ -71,11 +71,11 @@ public class FileStorageHandler implements FileStorageSrv.Iface {
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileData getFileData(String id) throws TException {
|
||||
public FileData getFileData(String fileDataId) throws TException {
|
||||
try {
|
||||
log.info("Request getFileData id: {}", id);
|
||||
checkString(id, "Bad request parameter, id required and not empty arg");
|
||||
FileData fileData = storageService.getFileData(id);
|
||||
log.info("Request getFileData fileDataId: {}", fileDataId);
|
||||
checkString(fileDataId, "Bad request parameter, fileDataId required and not empty arg");
|
||||
FileData fileData = storageService.getFileData(fileDataId);
|
||||
log.info("Response: fileData: {}", fileData);
|
||||
return fileData;
|
||||
} catch (StorageFileNotFoundException e) {
|
||||
|
||||
@ -6,10 +6,10 @@ import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.model.*;
|
||||
import com.amazonaws.services.s3.transfer.TransferManager;
|
||||
import com.amazonaws.services.s3.transfer.Upload;
|
||||
import com.rbkmoney.damsel.msgpack.Value;
|
||||
import com.rbkmoney.file.storage.FileData;
|
||||
import com.rbkmoney.file.storage.NewFileResult;
|
||||
import com.rbkmoney.file.storage.configuration.properties.StorageProperties;
|
||||
import com.rbkmoney.file.storage.msgpack.Value;
|
||||
import com.rbkmoney.file.storage.service.exception.StorageException;
|
||||
import com.rbkmoney.file.storage.service.exception.StorageFileNotFoundException;
|
||||
import com.rbkmoney.file.storage.util.DamselUtil;
|
||||
@ -32,7 +32,7 @@ import java.util.stream.Collectors;
|
||||
@RequiredArgsConstructor
|
||||
public class AmazonS3StorageService implements StorageService {
|
||||
|
||||
private static final String ID = "x-rbkmoney-id";
|
||||
private static final String FILE_DATA_ID = "x-rbkmoney-file-data-id";
|
||||
private static final String FILE_ID = "x-rbkmoney-file-id";
|
||||
private static final String CREATED_AT = "x-rbkmoney-created-at";
|
||||
private static final String METADATA = "x-rbkmoney-metadata-";
|
||||
@ -57,10 +57,10 @@ public class AmazonS3StorageService implements StorageService {
|
||||
log.info("Trying to create new file to storage, bucketId='{}'", bucketName);
|
||||
|
||||
InputStream emptyContent = getEmptyContent();
|
||||
String id = createId();
|
||||
String fileDataId = createId();
|
||||
String fileId = createId();
|
||||
FileDto fileDto = new FileDto(
|
||||
id,
|
||||
fileDataId,
|
||||
fileId,
|
||||
Instant.now().toString(),
|
||||
metadata
|
||||
@ -69,19 +69,19 @@ public class AmazonS3StorageService implements StorageService {
|
||||
try {
|
||||
// в хранилище записывается неизменяемый фейковый файл с метаданными,
|
||||
// в котором находится ссылка на реальный файл
|
||||
uploadRequest(id, fileDto, emptyContent);
|
||||
uploadRequest(fileDataId, fileDto, emptyContent);
|
||||
|
||||
// генерируем ссылку на выгрузку файла в хранилище напрямую в цеф по ключу fileId
|
||||
URL uploadUrl = generateUploadUrl(fileId, expirationTime);
|
||||
|
||||
log.info("File have been successfully created, id='{}', bucketId='{}'", id, bucketName);
|
||||
log.info("File have been successfully created, fileDataId='{}', bucketId='{}'", fileDataId, bucketName);
|
||||
|
||||
return new NewFileResult(id, uploadUrl.toString());
|
||||
return new NewFileResult(fileDataId, uploadUrl.toString());
|
||||
} catch (AmazonClientException ex) {
|
||||
throw new StorageException(
|
||||
String.format(
|
||||
"Failed to create new file to storage, id='%s', bucketId='%s'",
|
||||
id,
|
||||
"Failed to create new file to storage, fileDataId='%s', bucketId='%s'",
|
||||
fileDataId,
|
||||
bucketName
|
||||
),
|
||||
ex
|
||||
@ -90,15 +90,15 @@ public class AmazonS3StorageService implements StorageService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public URL generateDownloadUrl(String id, Instant expirationTime) throws StorageException {
|
||||
String fileId = getFileDto(id).getFileId();
|
||||
public URL generateDownloadUrl(String fileDataId, Instant expirationTime) throws StorageException {
|
||||
String fileId = getFileDto(fileDataId).getFileId();
|
||||
// генерируем ссылку на загрузку файла из хранилища напрямую в цеф по ключу fileId
|
||||
return generatePresignedUrl(fileId, expirationTime, HttpMethod.GET);
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileData getFileData(String id) throws StorageException {
|
||||
FileDto fileDto = getFileDto(id);
|
||||
public FileData getFileData(String fileDataId) throws StorageException {
|
||||
FileDto fileDto = getFileDto(fileDataId);
|
||||
|
||||
S3Object object = getS3Object(fileDto.getFileId());
|
||||
|
||||
@ -112,14 +112,14 @@ public class AmazonS3StorageService implements StorageService {
|
||||
throw new StorageException(
|
||||
String.format(
|
||||
"Failed to extract fileName, id='%s', bucketId='%s'",
|
||||
id,
|
||||
fileDataId,
|
||||
bucketName
|
||||
),
|
||||
ex
|
||||
);
|
||||
}
|
||||
|
||||
return new FileData(fileDto.getId(), fileName, fileDto.getCreatedAt(), fileDto.getMetadata());
|
||||
return new FileData(fileDto.getFileDataId(), fileName, fileDto.getCreatedAt(), fileDto.getMetadata());
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
@ -171,15 +171,15 @@ public class AmazonS3StorageService implements StorageService {
|
||||
|
||||
private FileDto getFileDtoByFakeFile(ObjectMetadata objectMetadata) {
|
||||
log.info("Trying to extract real file metadata by fake file from storage: ETag='{}'", objectMetadata.getETag());
|
||||
String id = getUserMetadataParameter(objectMetadata, ID);
|
||||
String id = getUserMetadataParameter(objectMetadata, FILE_DATA_ID);
|
||||
String fileId = getFileIdFromObjectMetadata(objectMetadata);
|
||||
String createdAt = getUserMetadataParameter(objectMetadata, CREATED_AT);
|
||||
Map<String, com.rbkmoney.damsel.msgpack.Value> metadata = objectMetadata.getUserMetadata().entrySet().stream()
|
||||
Map<String, Value> metadata = objectMetadata.getUserMetadata().entrySet().stream()
|
||||
.filter(entry -> entry.getKey().startsWith(METADATA) && entry.getValue() != null)
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
o -> o.getKey().substring(METADATA.length()),
|
||||
o -> DamselUtil.fromJson(o.getValue(), com.rbkmoney.damsel.msgpack.Value.class)
|
||||
o -> DamselUtil.fromJson(o.getValue(), Value.class)
|
||||
)
|
||||
);
|
||||
log.info(
|
||||
@ -232,8 +232,8 @@ public class AmazonS3StorageService implements StorageService {
|
||||
return new ByteArrayInputStream(new byte[0]);
|
||||
}
|
||||
|
||||
private void uploadRequest(String id, FileDto fileDto, InputStream inputStream) throws AmazonClientException {
|
||||
PutObjectRequest putObjectRequest = createS3Request(id, fileDto, inputStream);
|
||||
private void uploadRequest(String fileDataId, FileDto fileDto, InputStream inputStream) throws AmazonClientException {
|
||||
PutObjectRequest putObjectRequest = createS3Request(fileDataId, fileDto, inputStream);
|
||||
|
||||
Upload upload = transferManager.upload(putObjectRequest);
|
||||
try {
|
||||
@ -243,10 +243,10 @@ public class AmazonS3StorageService implements StorageService {
|
||||
}
|
||||
}
|
||||
|
||||
private PutObjectRequest createS3Request(String id, FileDto fileDto, InputStream inputStream) {
|
||||
private PutObjectRequest createS3Request(String fileDataId, FileDto fileDto, InputStream inputStream) {
|
||||
return new PutObjectRequest(
|
||||
bucketName,
|
||||
id,
|
||||
fileDataId,
|
||||
inputStream,
|
||||
createObjectMetadata(fileDto)
|
||||
);
|
||||
@ -254,7 +254,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
|
||||
private ObjectMetadata createObjectMetadata(FileDto fileDto) {
|
||||
ObjectMetadata objectMetadata = new ObjectMetadata();
|
||||
objectMetadata.addUserMetadata(ID, fileDto.getId());
|
||||
objectMetadata.addUserMetadata(FILE_DATA_ID, fileDto.getFileDataId());
|
||||
objectMetadata.addUserMetadata(FILE_ID, fileDto.getFileId());
|
||||
objectMetadata.addUserMetadata(CREATED_AT, fileDto.getCreatedAt());
|
||||
fileDto.getMetadata().forEach(
|
||||
@ -298,7 +298,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
@Getter
|
||||
private class FileDto {
|
||||
|
||||
private final String id;
|
||||
private final String fileDataId;
|
||||
private final String fileId;
|
||||
private final String createdAt;
|
||||
private final Map<String, Value> metadata;
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
package com.rbkmoney.file.storage.service;
|
||||
|
||||
import com.rbkmoney.damsel.msgpack.Value;
|
||||
import com.rbkmoney.file.storage.FileData;
|
||||
import com.rbkmoney.file.storage.NewFileResult;
|
||||
import com.rbkmoney.file.storage.msgpack.Value;
|
||||
import com.rbkmoney.file.storage.service.exception.StorageException;
|
||||
|
||||
import java.net.URL;
|
||||
@ -13,8 +13,8 @@ public interface StorageService {
|
||||
|
||||
NewFileResult createNewFile(Map<String, Value> metadata, Instant expirationTime) throws StorageException;
|
||||
|
||||
URL generateDownloadUrl(String id, Instant expirationTime) throws StorageException;
|
||||
URL generateDownloadUrl(String fileDataId, Instant expirationTime) throws StorageException;
|
||||
|
||||
FileData getFileData(String id) throws StorageException;
|
||||
FileData getFileData(String fileDataId) throws StorageException;
|
||||
|
||||
}
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
package com.rbkmoney.file.storage;
|
||||
|
||||
import com.rbkmoney.damsel.msgpack.Value;
|
||||
import com.rbkmoney.file.storage.msgpack.Value;
|
||||
import org.apache.thrift.TException;
|
||||
import org.junit.Test;
|
||||
import org.springframework.http.HttpStatus;
|
||||
@ -42,7 +42,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
uploadTestData(fileResult, FILE_NAME, FILE_DATA);
|
||||
|
||||
// генерация url с доступом только для загрузки
|
||||
URL downloadUrl = new URL(client.generateDownloadUrl(fileResult.getId(), expirationTime));
|
||||
URL downloadUrl = new URL(client.generateDownloadUrl(fileResult.getFileDataId(), expirationTime));
|
||||
|
||||
HttpURLConnection downloadUrlConnection = getHttpURLConnection(downloadUrl, "GET", false);
|
||||
InputStream inputStream = downloadUrlConnection.getInputStream();
|
||||
@ -68,7 +68,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult fileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
String fileDataId = fileResult.getId();
|
||||
String fileDataId = fileResult.getFileDataId();
|
||||
|
||||
// ошибка доступа - файла не существует, тк не было upload
|
||||
assertThrows(FileNotFound.class, () -> client.generateDownloadUrl(fileDataId, expirationTime));
|
||||
@ -98,7 +98,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult fileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
String fileDataId = fileResult.getId();
|
||||
String fileDataId = fileResult.getFileDataId();
|
||||
|
||||
// upload тестовых данных в хранилище
|
||||
uploadTestData(fileResult, FILE_NAME, FILE_DATA);
|
||||
@ -119,7 +119,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult validFileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
String validFileDataId = validFileResult.getId();
|
||||
String validFileDataId = validFileResult.getFileDataId();
|
||||
|
||||
// задержка перед upload для теста expiration
|
||||
Thread.sleep(1000);
|
||||
@ -135,7 +135,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
// создание файла с доступом к файлу на секунду
|
||||
NewFileResult throwingFileResult = client.createNewFile(Collections.emptyMap(), generateCurrentTimePlusSecond().toString());
|
||||
|
||||
String throwingFileDataId = throwingFileResult.getId();
|
||||
String throwingFileDataId = throwingFileResult.getFileDataId();
|
||||
|
||||
// ошибка доступа - файла не существует, тк не было upload
|
||||
assertThrows(FileNotFound.class, () -> client.generateDownloadUrl(throwingFileDataId, expirationTime));
|
||||
@ -167,7 +167,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
NewFileResult fileResult = client.createNewFile(metadata, expirationTime);
|
||||
uploadTestData(fileResult, FILE_NAME, FILE_DATA);
|
||||
|
||||
FileData fileData = client.getFileData(fileResult.getId());
|
||||
FileData fileData = client.getFileData(fileResult.getFileDataId());
|
||||
|
||||
assertEquals(fileData.getMetadata(), metadata);
|
||||
}
|
||||
@ -182,7 +182,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
String fileName = "csgo-лучше-чем-1.6";
|
||||
uploadTestData(fileResult, fileName, FILE_DATA);
|
||||
|
||||
FileData fileData = client.getFileData(fileResult.getId());
|
||||
FileData fileData = client.getFileData(fileResult.getFileDataId());
|
||||
|
||||
// тут используется энкодер\декодер, потому что apache http клиент менять кодировку.
|
||||
// при аплоаде напрямую по uploadUrl в ceph такой проблемы нет
|
||||
|
||||
Loading…
Reference in New Issue
Block a user