valitydev/file-storage
14
0
Fork 0
mirror of https://github.com/valitydev/file-storage.git synced 2024-11-06 00:35:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

disable sdk v1 and ceph tests

Browse Source
This commit is contained in:
Anatoly Karlov 2023-04-18 19:51:12 +03:00
parent 34161553d2
commit 98640fe376
19 changed files with 37 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/build.yml vendored
View File

@ -7,4 +7,4 @@ on:
jobs:
build:
uses: valitydev/java-workflow/.github/workflows/maven-service-build.yml@v1
uses: valitydev/java-workflow/.github/workflows/maven-service-build.yml@v2

2
.github/workflows/deploy.yml vendored
View File

@ -12,7 +12,7 @@ env:
jobs:
deploy:
uses: valitydev/java-workflow/.github/workflows/maven-service-deploy.yml@v1
uses: valitydev/java-workflow/.github/workflows/maven-service-deploy.yml@v2
secrets:
github-token: ${{ secrets.GITHUB_TOKEN }}
mm-webhook-url: ${{ secrets.MATTERMOST_WEBHOOK_URL }}

23
README.md
View File

@ -5,30 +5,10 @@ access an Amazon S3 resource without requiring the user of the URL to know the a
## Параметры запуска
Для работы с 1 версией `AWS SDK S3`
```yaml
s3:
endpoint: 'http://127.0.0.1:32827'
bucket-name: 'files'
signing-region: 'RU'
client-protocol: 'http'
client-max-error-retry: 10
signer-override: 'S3SignerType'
# signer-override: 'AWSS3V4SignerType'
access-key: 'test'
secret-key: 'test'
s3-sdk-v2:
enabled: 'false'
```
дефолтная версия сигнера — `S3SignerType`, для использования более актуальной версии указывается `AWSS3V4SignerType`
Для работы с 2 версией `AWS SDK S3 V2`
```yaml
s3-sdk-v2:
enabled: 'true'
endpoint: 'http://127.0.0.1:9000'
bucket-name: 'files-v2'
region: 'RU'
@ -36,9 +16,6 @@ s3-sdk-v2:
secret-key: 'minio123'
```
Для работы сервиса может использоваться только одна из двух версий `AWS SDK S3`, переключение происходит
параметром `s3-sdk-v2.enabled=false`
## Minio
Если сервисом используется 2 версия `AWS SDK S3 V2`, и в качестве s3 кластера используется `minio`, то для поддержки

14
pom.xml
View File

@ -6,7 +6,7 @@
<parent>
<groupId>dev.vality</groupId>
<artifactId>service-parent-pom</artifactId>
<version>1.0.19</version>
<version>2.1.4</version>
</parent>
<artifactId>file-storage</artifactId>
@ -30,7 +30,7 @@
<dependency>
<groupId>dev.vality</groupId>
<artifactId>file-storage-proto</artifactId>
<version>1.44-1dbc006</version>
<version>1.45-321e0e8</version>
</dependency>
<dependency>
<groupId>dev.vality</groupId>
@ -78,12 +78,12 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.300</version>
<version>1.12.451</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
<version>2.17.271</version>
<version>2.20.48</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
@ -107,7 +107,7 @@
<dependency>
<groupId>dev.vality</groupId>
<artifactId>testcontainers-annotations</artifactId>
<version>1.4.3</version>
<version>2.0.2</version>
<scope>test</scope>
</dependency>
</dependencies>
@ -142,12 +142,12 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-remote-resources-plugin</artifactId>
<version>1.7.0</version>
<version>3.0.0</version>
<dependencies>
<dependency>
<groupId>org.apache.maven.shared</groupId>
<artifactId>maven-filtering</artifactId>
<version>3.3.0</version>
<version>3.3.1</version>
</dependency>
</dependencies>
<configuration>

6
src/main/java/dev/vality/file/storage/configuration/S3ClientConfig.java
View File

@ -12,13 +12,11 @@ import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import dev.vality.file.storage.configuration.properties.S3Properties;
import lombok.RequiredArgsConstructor;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
//@Configuration
@RequiredArgsConstructor
@EnableConfigurationProperties(S3Properties.class)
//@EnableConfigurationProperties(S3Properties.class)
public class S3ClientConfig {
private final S3Properties s3Properties;

6
src/main/java/dev/vality/file/storage/configuration/properties/S3Properties.java
View File

@ -3,13 +3,11 @@ package dev.vality.file.storage.configuration.properties;
import com.amazonaws.Protocol;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Getter
@Setter
@Component
@ConfigurationProperties("s3")
//@Component
//@ConfigurationProperties("s3")
public class S3Properties {
private String endpoint;

31
src/main/java/dev/vality/file/storage/handler/FileStorageHandler.java
View File

@ -6,12 +6,9 @@ import dev.vality.file.storage.FileStorageSrv;
import dev.vality.file.storage.NewFileResult;
import dev.vality.file.storage.service.StorageService;
import dev.vality.file.storage.service.exception.FileNotFoundException;
import dev.vality.file.storage.service.exception.StorageException;
import dev.vality.file.storage.util.CheckerUtil;
import dev.vality.geck.common.util.TypeUtil;
import dev.vality.msgpack.Value;
import dev.vality.woody.api.flow.error.WUnavailableResultException;
import dev.vality.woody.api.flow.error.WUndefinedResultException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.thrift.TException;
@ -30,14 +27,8 @@ public class FileStorageHandler implements FileStorageSrv.Iface {
@Override
public NewFileResult createNewFile(Map<String, Value> metadata, String expiresAt) throws TException {
try {
Instant instant = TypeUtil.stringToInstant(expiresAt);
return storageService.createNewFile(metadata, instant);
} catch (StorageException e) {
throw unavailableResultException(e);
} catch (Exception e) {
throw undefinedResultException("Error when \"createNewFile\"", e);
}
var instant = TypeUtil.stringToInstant(expiresAt);
return storageService.createNewFile(metadata, instant);
}
@Override
@ -50,10 +41,6 @@ public class FileStorageHandler implements FileStorageSrv.Iface {
return url.toString();
} catch (FileNotFoundException e) {
throw fileNotFound(e);
} catch (StorageException e) {
throw unavailableResultException(e);
} catch (Exception e) {
throw undefinedResultException("Error when \"generateDownloadUrl\"", e);
}
}
@ -64,10 +51,6 @@ public class FileStorageHandler implements FileStorageSrv.Iface {
return storageService.getFileData(fileDataId);
} catch (FileNotFoundException e) {
throw fileNotFound(e);
} catch (StorageException e) {
throw unavailableResultException(e);
} catch (Exception e) {
throw undefinedResultException("Error when \"getFileData\"", e);
}
}
@ -75,14 +58,4 @@ public class FileStorageHandler implements FileStorageSrv.Iface {
log.warn("File not found", e);
return new FileNotFound();
}
private WUnavailableResultException unavailableResultException(StorageException e) {
log.error("Error with storage", e);
return new WUnavailableResultException("Error with storage", e);
}
private WUndefinedResultException undefinedResultException(String msg, Exception e) {
log.error(msg, e);
return new WUndefinedResultException(msg, e);
}
}

5
src/main/java/dev/vality/file/storage/service/S3Service.java
View File

@ -19,8 +19,6 @@ import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
@ -34,8 +32,7 @@ import java.util.stream.Collectors;
import static java.lang.String.format;
@Service
@ConditionalOnProperty(value = "s3-sdk-v2.enabled", havingValue = "false")
//@Service
@Slf4j
@RequiredArgsConstructor
public class S3Service implements StorageService {

2
src/main/java/dev/vality/file/storage/service/S3V2Service.java
View File

@ -11,7 +11,6 @@ import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
@ -29,7 +28,6 @@ import java.util.function.Predicate;
import java.util.stream.Collectors;
@Service
@ConditionalOnProperty(value = "s3-sdk-v2.enabled", havingValue = "true")
@Slf4j
@RequiredArgsConstructor
public class S3V2Service implements StorageService {

2
src/main/java/dev/vality/file/storage/servlet/FileStorageServlet.java
View File

@ -8,7 +8,7 @@ import javax.servlet.*;
import javax.servlet.annotation.WebServlet;
import java.io.IOException;
@WebServlet("/file_storage")
@WebServlet("/file_storage/v2")
@RequiredArgsConstructor
public class FileStorageServlet extends GenericServlet {

22
src/main/resources/application.yml
View File

@ -1,19 +1,19 @@
info:
version: '@project.version@'
version: '${project.version}'
stage: dev
server:
port: '@server.port@'
port: '${server.port}'
management:
server:
port: '@management.port@'
port: '${management.port}'
metrics:
export:
prometheus:
enabled: false
tags:
application: '@project.name@'
application: '${project.name}'
endpoint:
health:
probes:
@ -30,24 +30,12 @@ management:
spring:
application:
name: '@project.name@'
name: '${project.name}'
output:
ansi:
enabled: always
s3:
endpoint: 'http://127.0.0.1:9000'
bucket-name: 'files'
signing-region: 'RU'
client-protocol: 'http'
client-max-error-retry: 10
# signer-override: 'S3SignerType'
signer-override: 'AWSS3V4SignerType'
access-key: 'minio'
secret-key: 'minio123'
s3-sdk-v2:
enabled: 'false'
endpoint: 'http://127.0.0.1:9000'
bucket-name: 'files-v2'
region: 'RU'

4
src/test/java/dev/vality/file/storage/CatchExceptionsTest.java
View File

@ -6,6 +6,7 @@ import com.amazonaws.services.s3.transfer.TransferManager;
import dev.vality.woody.api.flow.error.WRuntimeException;
import dev.vality.woody.thrift.impl.http.THSpawnClientBuilder;
import org.apache.thrift.TException;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.boot.test.context.SpringBootTest;
@ -18,6 +19,7 @@ import java.util.UUID;
import static org.junit.jupiter.api.Assertions.assertThrows;
@Disabled
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
public class CatchExceptionsTest {
@ -33,7 +35,7 @@ public class CatchExceptionsTest {
@Test
public void shouldThrowException() throws URISyntaxException, TException {
FileStorageSrv.Iface fileStorageCli = new THSpawnClientBuilder()
.withAddress(new URI("http://localhost:" + port + "/file_storage"))
.withAddress(new URI("http://localhost:" + port + "/file_storage/v2"))
.withNetworkTimeout(555000)
.build(FileStorageSrv.Iface.class);

2
src/test/java/dev/vality/file/storage/FileStorageTest.java
View File

@ -51,7 +51,7 @@ public abstract class FileStorageTest {
@BeforeEach
public void setUp() throws Exception {
fileStorageClient = new THSpawnClientBuilder()
.withAddress(new URI("http://localhost:" + port + "/file_storage"))
.withAddress(new URI("http://localhost:" + port + "/file_storage/v2"))
.withNetworkTimeout(TIMEOUT)
.build(FileStorageSrv.Iface.class);
}

2
src/test/java/dev/vality/file/storage/awssdks3v2/WithCephTest.java
View File

@ -2,7 +2,9 @@ package dev.vality.file.storage.awssdks3v2;
import dev.vality.file.storage.FileStorageTest;
import dev.vality.testcontainers.annotations.ceph.CephTestcontainerSingleton;
import org.junit.jupiter.api.Disabled;
@Disabled
@CephTestcontainerSingleton(
properties = {"s3-sdk-v2.enabled=true", "s3-sdk-v2.region=us-east-1"},
bucketName = "awssdks3v2")

4
src/test/java/dev/vality/file/storage/awssdks3v2/WithMinioTest.java
View File

@ -3,8 +3,6 @@ package dev.vality.file.storage.awssdks3v2;
import dev.vality.file.storage.FileStorageTest;
import dev.vality.testcontainers.annotations.minio.MinioTestcontainerSingleton;
@MinioTestcontainerSingleton(
properties = "s3-sdk-v2.enabled=true",
bucketName = "awssdks3v2")
@MinioTestcontainerSingleton(bucketName = "awssdks3v2")
public class WithMinioTest extends FileStorageTest {
}

2
src/test/java/dev/vality/file/storage/s3signer/WithCephTest.java
View File

@ -2,7 +2,9 @@ package dev.vality.file.storage.s3signer;
import dev.vality.file.storage.FileStorageTest;
import dev.vality.testcontainers.annotations.ceph.CephTestcontainerSingleton;
import org.junit.jupiter.api.Disabled;
@Disabled
@CephTestcontainerSingleton(bucketName = "s3signer")
public class WithCephTest extends FileStorageTest {
}

2
src/test/java/dev/vality/file/storage/s3signer/WithMinioTest.java
View File

@ -2,7 +2,9 @@ package dev.vality.file.storage.s3signer;
import dev.vality.file.storage.FileStorageTest;
import dev.vality.testcontainers.annotations.minio.MinioTestcontainerSingleton;
import org.junit.jupiter.api.Disabled;
@Disabled
@MinioTestcontainerSingleton(bucketName = "s3signer")
public class WithMinioTest extends FileStorageTest {
}

2
src/test/java/dev/vality/file/storage/s3v4signer/WithCephTest.java
View File

@ -2,7 +2,9 @@ package dev.vality.file.storage.s3v4signer;
import dev.vality.file.storage.FileStorageTest;
import dev.vality.testcontainers.annotations.ceph.CephTestcontainerSingleton;
import org.junit.jupiter.api.Disabled;
@Disabled
@CephTestcontainerSingleton(
properties = "s3.signer-override=AWSS3V4SignerType",
bucketName = "s3v4signer")

2
src/test/java/dev/vality/file/storage/s3v4signer/WithMinioTest.java
View File

@ -2,7 +2,9 @@ package dev.vality.file.storage.s3v4signer;
import dev.vality.file.storage.FileStorageTest;
import dev.vality.testcontainers.annotations.minio.MinioTestcontainerSingleton;
import org.junit.jupiter.api.Disabled;
@Disabled
@MinioTestcontainerSingleton(
properties = "s3.signer-override=AWSS3V4SignerType",
bucketName = "s3v4signer")