mirror of
https://github.com/valitydev/file-storage.git
synced 2024-11-06 00:35:22 +00:00
BJ-384: добавление логов (#10)
This commit is contained in:
Anatolii Karlov
GitHub
parent
b3f0c71446
commit
52867f6e81
49
Jenkinsfile
vendored
49
Jenkinsfile
vendored
@ -1,47 +1,16 @@
|
||||
#!groovy
|
||||
build('file-storage', 'java-maven') {
|
||||
checkoutRepo()
|
||||
loadBuildUtils()
|
||||
|
||||
def javaServicePipeline
|
||||
runStage('load JavaService pipeline') {
|
||||
javaServicePipeline = load("build_utils/jenkins_lib/pipeJavaService.groovy")
|
||||
}
|
||||
|
||||
def serviceName = env.REPO_NAME
|
||||
def mvnArgs = '-DjvmArgs="-Xmx256m"'
|
||||
def useJava11 = false
|
||||
|
||||
// Run mvn and generate docker file
|
||||
runStage('Maven package') {
|
||||
withCredentials([[$class: 'FileBinding', credentialsId: 'java-maven-settings.xml', variable: 'SETTINGS_XML']]) {
|
||||
def mvn_command_arguments = ' --batch-mode --settings $SETTINGS_XML -P ci ' +
|
||||
" -Dgit.branch=${env.BRANCH_NAME} " +
|
||||
" ${mvnArgs}"
|
||||
if (env.BRANCH_NAME == 'master') {
|
||||
sh 'mvn deploy' + mvn_command_arguments
|
||||
} else {
|
||||
sh 'mvn package' + mvn_command_arguments
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def serviceImage;
|
||||
def imgShortName = 'rbkmoney/' + "${serviceName}" + ':' + '$COMMIT_ID';
|
||||
getCommitId()
|
||||
runStage('Build Service image') {
|
||||
serviceImage = docker.build(imgShortName, '-f ./target/Dockerfile ./target')
|
||||
}
|
||||
|
||||
try {
|
||||
if (env.BRANCH_NAME == 'master') {
|
||||
runStage('Push Service image') {
|
||||
docker.withRegistry('https://dr.rbkmoney.com/v2/', 'dockerhub-rbkmoneycibot') {
|
||||
serviceImage.push();
|
||||
}
|
||||
// Push under 'withRegistry' generates 2d record with 'long name' in local docker registry.
|
||||
// Untag the long-name
|
||||
sh "docker rmi dr.rbkmoney.com/${imgShortName}"
|
||||
}
|
||||
}
|
||||
}
|
||||
finally {
|
||||
runStage('Remove local image') {
|
||||
// Remove the image to keep Jenkins runner clean.
|
||||
sh "docker rmi ${imgShortName}"
|
||||
}
|
||||
}
|
||||
}
|
||||
javaServicePipeline(serviceName, useJava11, mvnArgs)
|
||||
}
|
||||
|
||||
@ -1 +1 @@
|
||||
Subproject commit 9b664082ddc8ec8cdfbe7513d54e433d24198cc2
|
||||
Subproject commit 1d7072912dbcbb60ece8756895c5bc3aea1b75d2
|
||||
2
pom.xml
2
pom.xml
@ -10,7 +10,7 @@
|
||||
</parent>
|
||||
|
||||
<artifactId>file-storage</artifactId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
<version>0.9</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>file-storage</name>
|
||||
|
||||
@ -54,7 +54,9 @@ public class AmazonS3ClientConfiguration {
|
||||
return new ClientConfiguration()
|
||||
.withProtocol(storageProperties.getClientProtocol())
|
||||
.withSignerOverride("S3SignerType")
|
||||
.withMaxErrorRetry(storageProperties.getClientMaxErrorRetry());
|
||||
.withMaxErrorRetry(storageProperties.getClientMaxErrorRetry())
|
||||
.withConnectionTimeout(storageProperties.getConnectionTimeout())
|
||||
.withSocketTimeout(storageProperties.getSocketTimeout());
|
||||
}
|
||||
|
||||
@Bean
|
||||
|
||||
@ -20,5 +20,7 @@ public class StorageProperties {
|
||||
private Protocol clientProtocol;
|
||||
private Integer clientMaxErrorRetry;
|
||||
private String bucketName;
|
||||
private int socketTimeout;
|
||||
private int connectionTimeout;
|
||||
|
||||
}
|
||||
|
||||
@ -69,6 +69,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
try {
|
||||
// в хранилище записывается неизменяемый фейковый файл с метаданными,
|
||||
// в котором находится ссылка на реальный файл
|
||||
log.info("Upload fake file with metadata in ObjectMetadata, fileDataId='{}', bucketId='{}'", fileDataId, bucketName);
|
||||
uploadRequest(fileDataId, fileDto, emptyContent);
|
||||
|
||||
// генерируем ссылку на выгрузку файла в хранилище напрямую в цеф по ключу fileId
|
||||
@ -91,6 +92,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
|
||||
@Override
|
||||
public URL generateDownloadUrl(String fileDataId, Instant expirationTime) throws StorageException {
|
||||
log.info("Download fake file with metadata in ObjectMetadata, fileDataId='{}', bucketId='{}'", fileDataId, bucketName);
|
||||
String fileId = getFileDto(fileDataId).getFileId();
|
||||
// генерируем ссылку на загрузку файла из хранилища напрямую в цеф по ключу fileId
|
||||
return generatePresignedUrl(fileId, expirationTime, HttpMethod.GET);
|
||||
@ -130,7 +132,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
private S3Object getS3Object(String id) throws StorageException {
|
||||
try {
|
||||
log.info(
|
||||
"Trying to get file from storage, id='{}', bucketId='{}'",
|
||||
"Trying to get fake file with metadata in ObjectMetadata from storage, id='{}', bucketId='{}'",
|
||||
id,
|
||||
bucketName
|
||||
);
|
||||
@ -138,7 +140,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
S3Object object = s3Client.getObject(getObjectRequest);
|
||||
checkNotNull(object, id, "File");
|
||||
log.info(
|
||||
"File have been successfully got from storage, id='{}', bucketId='{}'",
|
||||
"Fake file with metadata in ObjectMetadata have been successfully got from storage, id='{}', bucketId='{}'",
|
||||
id,
|
||||
bucketName
|
||||
);
|
||||
@ -146,7 +148,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
} catch (AmazonClientException ex) {
|
||||
throw new StorageException(
|
||||
String.format(
|
||||
"Failed to get file from storage, fileDataId='%s', bucketId='%s'",
|
||||
"Failed to get fake file with metadata in ObjectMetadata from storage, fileDataId='%s', bucketId='%s'",
|
||||
id,
|
||||
bucketName
|
||||
),
|
||||
@ -156,7 +158,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
}
|
||||
|
||||
private void checkRealFileStatus(S3Object s3Object) throws StorageFileNotFoundException {
|
||||
log.info("Check real file expiration and uploaded status: ETag='{}'", s3Object.getObjectMetadata().getETag());
|
||||
log.info("Check real file expiration and uploaded status by fake file with metadata in ObjectMetadata: ETag='{}'", s3Object.getObjectMetadata().getETag());
|
||||
ObjectMetadata objectMetadata = s3Object.getObjectMetadata();
|
||||
|
||||
String fileId = getFileIdFromObjectMetadata(objectMetadata);
|
||||
@ -170,7 +172,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
}
|
||||
|
||||
private FileDto getFileDtoByFakeFile(ObjectMetadata objectMetadata) {
|
||||
log.info("Trying to extract real file metadata by fake file from storage: ETag='{}'", objectMetadata.getETag());
|
||||
log.info("Trying to extract real file metadata by fake file with metadata in ObjectMetadata: ETag='{}'", objectMetadata.getETag());
|
||||
String id = getUserMetadataParameter(objectMetadata, FILE_DATA_ID);
|
||||
String fileId = getFileIdFromObjectMetadata(objectMetadata);
|
||||
String createdAt = getUserMetadataParameter(objectMetadata, CREATED_AT);
|
||||
@ -183,7 +185,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
)
|
||||
);
|
||||
log.info(
|
||||
"Real file metadata have been successfully extracted by fake file from storage, id='{}', bucketId='{}'",
|
||||
"Real file metadata have been successfully extracted by fake file with metadata in ObjectMetadata, id='{}', bucketId='{}'",
|
||||
id,
|
||||
bucketName
|
||||
);
|
||||
@ -206,7 +208,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
URL url = s3Client.generatePresignedUrl(request);
|
||||
checkNotNull(url, fileId, "Presigned url");
|
||||
log.info(
|
||||
"Presigned url have been successfully generated, url='{}', fileId='{}', bucketId='{}', expirationTime='{}', httpMethod='{}'",
|
||||
"Presigned url for real file have been successfully generated, url='{}', fileId='{}', bucketId='{}', expirationTime='{}', httpMethod='{}'",
|
||||
url,
|
||||
fileId,
|
||||
bucketName,
|
||||
@ -217,7 +219,7 @@ public class AmazonS3StorageService implements StorageService {
|
||||
} catch (AmazonClientException ex) {
|
||||
throw new StorageException(
|
||||
String.format(
|
||||
"Failed to generate presigned url, fileId='%s', bucketId='%s', expirationTime='%s', httpMethod='%s'",
|
||||
"Failed to generate presigned url for real file, fileId='%s', bucketId='%s', expirationTime='%s', httpMethod='%s'",
|
||||
fileId,
|
||||
bucketName,
|
||||
expirationTime,
|
||||
|
||||
@ -1,24 +0,0 @@
|
||||
server.port=@server.port@
|
||||
spring.application.name=@project.name@
|
||||
info.version=@project.version@
|
||||
info.stage=dev
|
||||
management.metrics.export.statsd.flavor=etsy
|
||||
# --
|
||||
spring.servlet.multipart.max-file-size=10MB
|
||||
spring.servlet.multipart.max-request-size=10MB
|
||||
spring.servlet.multipart.enabled=true
|
||||
# --
|
||||
storage.endpoint=localhost
|
||||
storage.signingRegion=RU
|
||||
storage.clientProtocol=HTTP
|
||||
storage.clientMaxErrorRetry=10
|
||||
storage.bucketName=files
|
||||
# ----
|
||||
# for up local test version
|
||||
#storage.endpoint=localhost:32827
|
||||
#storage.signingRegion=RU
|
||||
#storage.accessKey=test
|
||||
#storage.secretKey=test
|
||||
#storage.clientProtocol=HTTP
|
||||
#storage.clientMaxErrorRetry=10
|
||||
#storage.bucketName=TEST
|
||||
27
src/main/resources/application.yml
Normal file
27
src/main/resources/application.yml
Normal file
@ -0,0 +1,27 @@
|
||||
info:
|
||||
version: @project.version@
|
||||
stage: dev
|
||||
server:
|
||||
port: @server.port@
|
||||
management:
|
||||
metrics:
|
||||
export:
|
||||
statsd:
|
||||
flavor: etsy
|
||||
enabled: false
|
||||
security:
|
||||
flag: false
|
||||
spring:
|
||||
application:
|
||||
name: @project.name@
|
||||
output:
|
||||
ansi:
|
||||
enabled: always
|
||||
storage:
|
||||
endpoint: localhost:32827
|
||||
bucketName: files
|
||||
signingRegion: RU
|
||||
clientProtocol: HTTP
|
||||
clientMaxErrorRetry: 10
|
||||
socketTimeout: 50000
|
||||
connectionTimeout: 10000
|
||||
35
src/test/java/com/rbkmoney/TestContainers.java
Normal file
35
src/test/java/com/rbkmoney/TestContainers.java
Normal file
@ -0,0 +1,35 @@
|
||||
package com.rbkmoney;
|
||||
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
@NoArgsConstructor
|
||||
@Setter
|
||||
public class TestContainers {
|
||||
|
||||
private Boolean dockerContainersEnable;
|
||||
private GenericContainer cephTestContainer;
|
||||
|
||||
public Optional<GenericContainer> getCephTestContainer() {
|
||||
return Optional.ofNullable(cephTestContainer);
|
||||
}
|
||||
|
||||
public Boolean isDockerContainersEnable() {
|
||||
return dockerContainersEnable;
|
||||
}
|
||||
|
||||
public void startTestContainers() {
|
||||
if (!isDockerContainersEnable()) {
|
||||
getCephTestContainer().ifPresent(GenericContainer::start);
|
||||
}
|
||||
}
|
||||
|
||||
public void stopTestContainers() {
|
||||
if (!isDockerContainersEnable()) {
|
||||
getCephTestContainer().ifPresent(GenericContainer::stop);
|
||||
}
|
||||
}
|
||||
}
|
||||
68
src/test/java/com/rbkmoney/TestContainersBuilder.java
Normal file
68
src/test/java/com/rbkmoney/TestContainersBuilder.java
Normal file
@ -0,0 +1,68 @@
|
||||
package com.rbkmoney;
|
||||
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.HttpWaitStrategy;
|
||||
import org.testcontainers.containers.wait.strategy.WaitStrategy;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
public class TestContainersBuilder {
|
||||
|
||||
public static final String SIGNING_REGION = "RU";
|
||||
public static final String AWS_ACCESS_KEY = "test";
|
||||
public static final String AWS_SECRET_KEY = "test";
|
||||
public static final String PROTOCOL = "HTTP";
|
||||
public static final String MAX_ERROR_RETRY = "10";
|
||||
public static final String BUCKET_NAME = "TEST";
|
||||
|
||||
private boolean dockerContainersEnable;
|
||||
private boolean cephTestContainerEnable;
|
||||
|
||||
private TestContainersBuilder(boolean dockerContainersEnable) {
|
||||
this.dockerContainersEnable = dockerContainersEnable;
|
||||
}
|
||||
|
||||
public static TestContainersBuilder builder(boolean dockerContainersEnable) {
|
||||
return new TestContainersBuilder(dockerContainersEnable);
|
||||
}
|
||||
|
||||
public TestContainersBuilder addCephTestContainer() {
|
||||
cephTestContainerEnable = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
public TestContainers build() {
|
||||
TestContainers testContainers = new TestContainers();
|
||||
|
||||
if (!dockerContainersEnable) {
|
||||
addTestContainers(testContainers);
|
||||
} else {
|
||||
testContainers.setDockerContainersEnable(true);
|
||||
}
|
||||
return testContainers;
|
||||
}
|
||||
|
||||
private void addTestContainers(TestContainers testContainers) {
|
||||
if (cephTestContainerEnable) {
|
||||
testContainers.setCephTestContainer(
|
||||
new GenericContainer<>("dr.rbkmoney.com/ceph-demo:latest")
|
||||
.withEnv("RGW_NAME", "localhost")
|
||||
.withEnv("NETWORK_AUTO_DETECT", "4")
|
||||
.withEnv("CEPH_DEMO_UID", "ceph-test")
|
||||
.withEnv("CEPH_DEMO_ACCESS_KEY", AWS_ACCESS_KEY)
|
||||
.withEnv("CEPH_DEMO_SECRET_KEY", AWS_SECRET_KEY)
|
||||
.withEnv("CEPH_DEMO_BUCKET", BUCKET_NAME)
|
||||
.withExposedPorts(5000, 80)
|
||||
.waitingFor(getWaitStrategy("/api/v0.1/health"))
|
||||
);
|
||||
}
|
||||
testContainers.setDockerContainersEnable(false);
|
||||
}
|
||||
|
||||
private WaitStrategy getWaitStrategy(String path) {
|
||||
return new HttpWaitStrategy()
|
||||
.forPath(path)
|
||||
.forStatusCode(200)
|
||||
.withStartupTimeout(Duration.ofMinutes(10));
|
||||
}
|
||||
}
|
||||
@ -1,24 +1,24 @@
|
||||
package com.rbkmoney.file.storage;
|
||||
|
||||
import com.rbkmoney.TestContainers;
|
||||
import com.rbkmoney.TestContainersBuilder;
|
||||
import com.rbkmoney.woody.thrift.impl.http.THSpawnClientBuilder;
|
||||
import org.junit.Before;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.util.TestPropertyValues;
|
||||
import org.springframework.boot.web.server.LocalServerPort;
|
||||
import org.springframework.boot.test.context.TestConfiguration;
|
||||
import org.springframework.context.ApplicationContextInitializer;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.HttpWaitStrategy;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
@ -31,61 +31,43 @@ import static org.springframework.boot.test.context.SpringBootTest.WebEnvironmen
|
||||
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
|
||||
public abstract class AbstractIntegrationTest {
|
||||
|
||||
private static final int TIMEOUT = 555000;
|
||||
private static final String SIGNING_REGION = "RU";
|
||||
private static final String AWS_ACCESS_KEY = "test";
|
||||
private static final String AWS_SECRET_KEY = "test";
|
||||
private static final String PROTOCOL = "HTTP";
|
||||
private static final String MAX_ERROR_RETRY = "10";
|
||||
private static final String BUCKET_NAME = "TEST";
|
||||
private static TestContainers testContainers = TestContainersBuilder.builder(false)
|
||||
.addCephTestContainer()
|
||||
.build();
|
||||
|
||||
@LocalServerPort
|
||||
protected int port;
|
||||
@BeforeClass
|
||||
public static void beforeClass() {
|
||||
testContainers.startTestContainers();
|
||||
}
|
||||
|
||||
protected FileStorageSrv.Iface client;
|
||||
@AfterClass
|
||||
public static void afterClass() {
|
||||
testContainers.stopTestContainers();
|
||||
}
|
||||
|
||||
// for up local test version comment this ClassRule
|
||||
@ClassRule
|
||||
public static GenericContainer cephContainer = new GenericContainer("dr.rbkmoney.com/ceph-demo:latest")
|
||||
.withEnv("RGW_NAME", "localhost")
|
||||
.withEnv("NETWORK_AUTO_DETECT", "4")
|
||||
.withEnv("CEPH_DEMO_UID", "ceph-test")
|
||||
.withEnv("CEPH_DEMO_ACCESS_KEY", AWS_ACCESS_KEY)
|
||||
.withEnv("CEPH_DEMO_SECRET_KEY", AWS_SECRET_KEY)
|
||||
.withEnv("CEPH_DEMO_BUCKET", BUCKET_NAME)
|
||||
.withExposedPorts(5000, 80)
|
||||
.waitingFor(
|
||||
new HttpWaitStrategy()
|
||||
.forPath("/api/v0.1/health")
|
||||
.forStatusCode(200)
|
||||
.withStartupTimeout(Duration.ofMinutes(10))
|
||||
);
|
||||
@TestConfiguration
|
||||
public static class TestContextConfiguration {
|
||||
|
||||
public static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
|
||||
@Override
|
||||
public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
|
||||
TestPropertyValues.of(
|
||||
// for up local test version comment this row
|
||||
"storage.endpoint=" + cephContainer.getContainerIpAddress() + ":" + cephContainer.getMappedPort(80),
|
||||
// for up local test version uncomment this row
|
||||
// "storage.endpoint=localhost:32827",
|
||||
"storage.signingRegion=" + SIGNING_REGION,
|
||||
"storage.accessKey=" + AWS_ACCESS_KEY,
|
||||
"storage.secretKey=" + AWS_SECRET_KEY,
|
||||
"storage.clientProtocol=" + PROTOCOL,
|
||||
"storage.clientMaxErrorRetry=" + MAX_ERROR_RETRY,
|
||||
"storage.bucketName=" + BUCKET_NAME
|
||||
)
|
||||
.applyTo(configurableApplicationContext);
|
||||
private static final int TIMEOUT = 555000;
|
||||
|
||||
@Value("${local.server.port}")
|
||||
protected int port;
|
||||
|
||||
@Bean
|
||||
public FileStorageSrv.Iface fileStorageCli() throws URISyntaxException {
|
||||
return new THSpawnClientBuilder()
|
||||
.withAddress(new URI("http://localhost:" + port + "/file_storage"))
|
||||
.withNetworkTimeout(TIMEOUT)
|
||||
.build(FileStorageSrv.Iface.class);
|
||||
}
|
||||
}
|
||||
|
||||
@Before
|
||||
public void before() throws URISyntaxException {
|
||||
client = new THSpawnClientBuilder()
|
||||
.withAddress(new URI("http://localhost:" + port + "/file_storage"))
|
||||
.withNetworkTimeout(TIMEOUT)
|
||||
.build(FileStorageSrv.Iface.class);
|
||||
public static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
|
||||
|
||||
@Override
|
||||
public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
|
||||
FileStorageTestPropertyValuesBuilder.build(testContainers).applyTo(configurableApplicationContext);
|
||||
}
|
||||
}
|
||||
|
||||
protected Instant generateCurrentTimePlusDay() {
|
||||
|
||||
@ -10,6 +10,7 @@ import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.apache.thrift.TException;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.testcontainers.shaded.org.apache.commons.io.IOUtils;
|
||||
|
||||
@ -30,6 +31,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
// все тесты в 1 классе , чтобы сэкономить время на поднятии тест контейнера
|
||||
public class FileStorageTest extends AbstractIntegrationTest {
|
||||
|
||||
@Autowired
|
||||
private FileStorageSrv.Iface fileStorageCli;
|
||||
|
||||
private static final String FILE_DATA = "test";
|
||||
private static final String FILE_NAME = "rainbow-champion";
|
||||
|
||||
@ -38,7 +42,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
HttpClient httpClient = HttpClientBuilder.create().build();
|
||||
|
||||
NewFileResult fileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
NewFileResult fileResult = fileStorageCli.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
Path path = getFileFromResources();
|
||||
|
||||
@ -50,7 +54,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
Assert.assertEquals(response.getStatusLine().getStatusCode(), org.apache.http.HttpStatus.SC_OK);
|
||||
|
||||
// генерация url с доступом только для загрузки
|
||||
String downloadUrl = client.generateDownloadUrl(fileResult.getFileDataId(), expirationTime);
|
||||
String downloadUrl = fileStorageCli.generateDownloadUrl(fileResult.getFileDataId(), expirationTime);
|
||||
|
||||
HttpResponse responseGet = httpClient.execute(new HttpGet(downloadUrl));
|
||||
InputStream content = responseGet.getEntity().getContent();
|
||||
@ -66,11 +70,11 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
try {
|
||||
// создание нового файла
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult fileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
NewFileResult fileResult = fileStorageCli.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
uploadTestData(fileResult, FILE_NAME, FILE_DATA);
|
||||
|
||||
// генерация url с доступом только для загрузки
|
||||
URL downloadUrl = new URL(client.generateDownloadUrl(fileResult.getFileDataId(), expirationTime));
|
||||
URL downloadUrl = new URL(fileStorageCli.generateDownloadUrl(fileResult.getFileDataId(), expirationTime));
|
||||
|
||||
HttpURLConnection downloadUrlConnection = getHttpURLConnection(downloadUrl, "GET", false);
|
||||
InputStream inputStream = downloadUrlConnection.getInputStream();
|
||||
@ -94,20 +98,20 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
public void accessErrorToMethodsTest() throws IOException, TException {
|
||||
// создание файла с доступом к файлу на день
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult fileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
NewFileResult fileResult = fileStorageCli.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
String fileDataId = fileResult.getFileDataId();
|
||||
|
||||
// ошибка доступа - файла не существует, тк не было upload
|
||||
assertThrows(FileNotFound.class, () -> client.generateDownloadUrl(fileDataId, expirationTime));
|
||||
assertThrows(FileNotFound.class, () -> client.getFileData(fileDataId));
|
||||
assertThrows(FileNotFound.class, () -> fileStorageCli.generateDownloadUrl(fileDataId, expirationTime));
|
||||
assertThrows(FileNotFound.class, () -> fileStorageCli.getFileData(fileDataId));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void uploadUrlConnectionAccessTest() throws IOException, TException {
|
||||
// создание файла с доступом к файлу на день
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult fileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
NewFileResult fileResult = fileStorageCli.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
URL uploadUrl = new URL(fileResult.getUploadUrl());
|
||||
|
||||
@ -124,7 +128,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
public void downloadUrlConnectionAccessTest() throws IOException, TException {
|
||||
// создание файла с доступом к файлу на день
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult fileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
NewFileResult fileResult = fileStorageCli.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
String fileDataId = fileResult.getFileDataId();
|
||||
|
||||
@ -132,7 +136,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
uploadTestData(fileResult, FILE_NAME, FILE_DATA);
|
||||
|
||||
// генерация url с доступом только для загрузки
|
||||
URL url = new URL(client.generateDownloadUrl(fileDataId, expirationTime));
|
||||
URL url = new URL(fileStorageCli.generateDownloadUrl(fileDataId, expirationTime));
|
||||
|
||||
// с данной ссылкой нельзя записывать
|
||||
assertEquals(HttpStatus.FORBIDDEN.value(), getHttpURLConnection(url, "PUT", FILE_NAME, true).getResponseCode());
|
||||
@ -145,7 +149,7 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
public void expirationTimeTest() throws TException, InterruptedException, IOException {
|
||||
// создание файла с доступом к файлу на день
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult validFileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
NewFileResult validFileResult = fileStorageCli.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
String validFileDataId = validFileResult.getFileDataId();
|
||||
|
||||
@ -156,18 +160,18 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
uploadTestData(validFileResult, FILE_NAME, FILE_DATA);
|
||||
|
||||
// доступ есть
|
||||
client.getFileData(validFileDataId);
|
||||
client.generateDownloadUrl(validFileDataId, generateCurrentTimePlusDay().toString());
|
||||
fileStorageCli.getFileData(validFileDataId);
|
||||
fileStorageCli.generateDownloadUrl(validFileDataId, generateCurrentTimePlusDay().toString());
|
||||
|
||||
// - - - - - сделаем задержку больше expiration
|
||||
// создание файла с доступом к файлу на секунду
|
||||
NewFileResult throwingFileResult = client.createNewFile(Collections.emptyMap(), generateCurrentTimePlusSecond().toString());
|
||||
NewFileResult throwingFileResult = fileStorageCli.createNewFile(Collections.emptyMap(), generateCurrentTimePlusSecond().toString());
|
||||
|
||||
String throwingFileDataId = throwingFileResult.getFileDataId();
|
||||
|
||||
// ошибка доступа - файла не существует, тк не было upload
|
||||
assertThrows(FileNotFound.class, () -> client.generateDownloadUrl(throwingFileDataId, expirationTime));
|
||||
assertThrows(FileNotFound.class, () -> client.getFileData(throwingFileDataId));
|
||||
assertThrows(FileNotFound.class, () -> fileStorageCli.generateDownloadUrl(throwingFileDataId, expirationTime));
|
||||
assertThrows(FileNotFound.class, () -> fileStorageCli.getFileData(throwingFileDataId));
|
||||
|
||||
// задержка перед upload для теста expiration
|
||||
Thread.sleep(2000);
|
||||
@ -176,8 +180,8 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
assertThrows(AssertionError.class, () -> uploadTestData(throwingFileResult, FILE_NAME, FILE_DATA));
|
||||
|
||||
// ошибка доступа
|
||||
assertThrows(FileNotFound.class, () -> client.getFileData(throwingFileDataId));
|
||||
assertThrows(FileNotFound.class, () -> client.generateDownloadUrl(throwingFileDataId, expirationTime));
|
||||
assertThrows(FileNotFound.class, () -> fileStorageCli.getFileData(throwingFileDataId));
|
||||
assertThrows(FileNotFound.class, () -> fileStorageCli.generateDownloadUrl(throwingFileDataId, expirationTime));
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -192,10 +196,10 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
put("key6", Value.bin(new byte[]{}));
|
||||
}};
|
||||
|
||||
NewFileResult fileResult = client.createNewFile(metadata, expirationTime);
|
||||
NewFileResult fileResult = fileStorageCli.createNewFile(metadata, expirationTime);
|
||||
uploadTestData(fileResult, FILE_NAME, FILE_DATA);
|
||||
|
||||
FileData fileData = client.getFileData(fileResult.getFileDataId());
|
||||
FileData fileData = fileStorageCli.getFileData(fileResult.getFileDataId());
|
||||
|
||||
assertEquals(fileData.getMetadata(), metadata);
|
||||
}
|
||||
@ -204,13 +208,13 @@ public class FileStorageTest extends AbstractIntegrationTest {
|
||||
public void fileNameCyrillicTest() throws TException, IOException {
|
||||
// создание файла с доступом к файлу на день
|
||||
String expirationTime = generateCurrentTimePlusDay().toString();
|
||||
NewFileResult fileResult = client.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
NewFileResult fileResult = fileStorageCli.createNewFile(Collections.emptyMap(), expirationTime);
|
||||
|
||||
// upload тестовых данных в хранилище
|
||||
String fileName = "csgo-лучше-чем-1.6";
|
||||
uploadTestData(fileResult, fileName, FILE_DATA);
|
||||
|
||||
FileData fileData = client.getFileData(fileResult.getFileDataId());
|
||||
FileData fileData = fileStorageCli.getFileData(fileResult.getFileDataId());
|
||||
|
||||
// тут используется энкодер\декодер, потому что apache http клиент менять кодировку.
|
||||
// при аплоаде напрямую по uploadUrl в ceph такой проблемы нет
|
||||
|
||||
@ -0,0 +1,39 @@
|
||||
package com.rbkmoney.file.storage;
|
||||
|
||||
import com.rbkmoney.TestContainers;
|
||||
import org.springframework.boot.test.util.TestPropertyValues;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static com.rbkmoney.TestContainersBuilder.*;
|
||||
|
||||
public class FileStorageTestPropertyValuesBuilder {
|
||||
|
||||
public static TestPropertyValues build(TestContainers testContainers) {
|
||||
List<String> strings = new ArrayList<>();
|
||||
if (!testContainers.isDockerContainersEnable()) {
|
||||
withUsingTestContainers(testContainers, strings);
|
||||
} else {
|
||||
withoutUsingTestContainers(strings);
|
||||
}
|
||||
|
||||
strings.add("storage.signingRegion=" + SIGNING_REGION);
|
||||
strings.add("storage.accessKey=" + AWS_ACCESS_KEY);
|
||||
strings.add("storage.secretKey=" + AWS_SECRET_KEY);
|
||||
strings.add("storage.clientProtocol=" + PROTOCOL);
|
||||
strings.add("storage.clientMaxErrorRetry=" + MAX_ERROR_RETRY);
|
||||
strings.add("storage.bucketName=" + BUCKET_NAME);
|
||||
return TestPropertyValues.of(strings);
|
||||
}
|
||||
|
||||
private static void withUsingTestContainers(TestContainers testContainers, List<String> strings) {
|
||||
testContainers.getCephTestContainer().ifPresent(
|
||||
c -> strings.add("storage.endpoint=" + c.getContainerIpAddress() + ":" + c.getMappedPort(80))
|
||||
);
|
||||
}
|
||||
|
||||
private static void withoutUsingTestContainers(List<String> strings) {
|
||||
strings.add("storage.endpoint=localhost:32827");
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user