From 98640fe37687496502869eb7bdc9e88b06d7d26d Mon Sep 17 00:00:00 2001 From: Anatoly Karlov Date: Tue, 18 Apr 2023 19:51:12 +0300 Subject: [PATCH] disable sdk v1 and ceph tests --- .github/workflows/build.yml | 2 +- .github/workflows/deploy.yml | 2 +- README.md | 23 -------------- pom.xml | 14 ++++----- .../storage/configuration/S3ClientConfig.java | 6 ++-- .../properties/S3Properties.java | 6 ++-- .../storage/handler/FileStorageHandler.java | 31 ++----------------- .../file/storage/service/S3Service.java | 5 +-- .../file/storage/service/S3V2Service.java | 2 -- .../storage/servlet/FileStorageServlet.java | 2 +- src/main/resources/application.yml | 22 +++---------- .../file/storage/CatchExceptionsTest.java | 4 ++- .../vality/file/storage/FileStorageTest.java | 2 +- .../file/storage/awssdks3v2/WithCephTest.java | 2 ++ .../storage/awssdks3v2/WithMinioTest.java | 4 +-- .../file/storage/s3signer/WithCephTest.java | 2 ++ .../file/storage/s3signer/WithMinioTest.java | 2 ++ .../file/storage/s3v4signer/WithCephTest.java | 2 ++ .../storage/s3v4signer/WithMinioTest.java | 2 ++ 19 files changed, 37 insertions(+), 98 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a654702..4d5c06c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,4 +7,4 @@ on: jobs: build: - uses: valitydev/java-workflow/.github/workflows/maven-service-build.yml@v1 + uses: valitydev/java-workflow/.github/workflows/maven-service-build.yml@v2 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6a394b0..21c90bb 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -12,7 +12,7 @@ env: jobs: deploy: - uses: valitydev/java-workflow/.github/workflows/maven-service-deploy.yml@v1 + uses: valitydev/java-workflow/.github/workflows/maven-service-deploy.yml@v2 secrets: github-token: ${{ secrets.GITHUB_TOKEN }} mm-webhook-url: ${{ secrets.MATTERMOST_WEBHOOK_URL }} diff --git a/README.md b/README.md index 15ccd7a..db6babc 100644 --- a/README.md +++ b/README.md @@ -5,30 +5,10 @@ access an Amazon S3 resource without requiring the user of the URL to know the a ## Параметры запуска -Для работы с 1 версией `AWS SDK S3` - -```yaml -s3: - endpoint: 'http://127.0.0.1:32827' - bucket-name: 'files' - signing-region: 'RU' - client-protocol: 'http' - client-max-error-retry: 10 - signer-override: 'S3SignerType' - # signer-override: 'AWSS3V4SignerType' - access-key: 'test' - secret-key: 'test' -s3-sdk-v2: - enabled: 'false' -``` - -дефолтная версия сигнера — `S3SignerType`, для использования более актуальной версии указывается `AWSS3V4SignerType` - Для работы с 2 версией `AWS SDK S3 V2` ```yaml s3-sdk-v2: - enabled: 'true' endpoint: 'http://127.0.0.1:9000' bucket-name: 'files-v2' region: 'RU' @@ -36,9 +16,6 @@ s3-sdk-v2: secret-key: 'minio123' ``` -Для работы сервиса может использоваться только одна из двух версий `AWS SDK S3`, переключение происходит -параметром `s3-sdk-v2.enabled=false` - ## Minio Если сервисом используется 2 версия `AWS SDK S3 V2`, и в качестве s3 кластера используется `minio`, то для поддержки diff --git a/pom.xml b/pom.xml index b3acbce..fee6f75 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ dev.vality service-parent-pom - 1.0.19 + 2.1.4 file-storage @@ -30,7 +30,7 @@ dev.vality file-storage-proto - 1.44-1dbc006 + 1.45-321e0e8 dev.vality @@ -78,12 +78,12 @@ com.amazonaws aws-java-sdk-s3 - 1.12.300 + 1.12.451 software.amazon.awssdk s3 - 2.17.271 + 2.20.48 javax.servlet @@ -107,7 +107,7 @@ dev.vality testcontainers-annotations - 1.4.3 + 2.0.2 test @@ -142,12 +142,12 @@ org.apache.maven.plugins maven-remote-resources-plugin - 1.7.0 + 3.0.0 org.apache.maven.shared maven-filtering - 3.3.0 + 3.3.1 diff --git a/src/main/java/dev/vality/file/storage/configuration/S3ClientConfig.java b/src/main/java/dev/vality/file/storage/configuration/S3ClientConfig.java index a0ff064..1f88bbc 100644 --- a/src/main/java/dev/vality/file/storage/configuration/S3ClientConfig.java +++ b/src/main/java/dev/vality/file/storage/configuration/S3ClientConfig.java @@ -12,13 +12,11 @@ import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import dev.vality.file.storage.configuration.properties.S3Properties; import lombok.RequiredArgsConstructor; -import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -@Configuration +//@Configuration @RequiredArgsConstructor -@EnableConfigurationProperties(S3Properties.class) +//@EnableConfigurationProperties(S3Properties.class) public class S3ClientConfig { private final S3Properties s3Properties; diff --git a/src/main/java/dev/vality/file/storage/configuration/properties/S3Properties.java b/src/main/java/dev/vality/file/storage/configuration/properties/S3Properties.java index 0ebcd0f..75b7375 100644 --- a/src/main/java/dev/vality/file/storage/configuration/properties/S3Properties.java +++ b/src/main/java/dev/vality/file/storage/configuration/properties/S3Properties.java @@ -3,13 +3,11 @@ package dev.vality.file.storage.configuration.properties; import com.amazonaws.Protocol; import lombok.Getter; import lombok.Setter; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; @Getter @Setter -@Component -@ConfigurationProperties("s3") +//@Component +//@ConfigurationProperties("s3") public class S3Properties { private String endpoint; diff --git a/src/main/java/dev/vality/file/storage/handler/FileStorageHandler.java b/src/main/java/dev/vality/file/storage/handler/FileStorageHandler.java index f476c95..3977e7d 100644 --- a/src/main/java/dev/vality/file/storage/handler/FileStorageHandler.java +++ b/src/main/java/dev/vality/file/storage/handler/FileStorageHandler.java @@ -6,12 +6,9 @@ import dev.vality.file.storage.FileStorageSrv; import dev.vality.file.storage.NewFileResult; import dev.vality.file.storage.service.StorageService; import dev.vality.file.storage.service.exception.FileNotFoundException; -import dev.vality.file.storage.service.exception.StorageException; import dev.vality.file.storage.util.CheckerUtil; import dev.vality.geck.common.util.TypeUtil; import dev.vality.msgpack.Value; -import dev.vality.woody.api.flow.error.WUnavailableResultException; -import dev.vality.woody.api.flow.error.WUndefinedResultException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.thrift.TException; @@ -30,14 +27,8 @@ public class FileStorageHandler implements FileStorageSrv.Iface { @Override public NewFileResult createNewFile(Map metadata, String expiresAt) throws TException { - try { - Instant instant = TypeUtil.stringToInstant(expiresAt); - return storageService.createNewFile(metadata, instant); - } catch (StorageException e) { - throw unavailableResultException(e); - } catch (Exception e) { - throw undefinedResultException("Error when \"createNewFile\"", e); - } + var instant = TypeUtil.stringToInstant(expiresAt); + return storageService.createNewFile(metadata, instant); } @Override @@ -50,10 +41,6 @@ public class FileStorageHandler implements FileStorageSrv.Iface { return url.toString(); } catch (FileNotFoundException e) { throw fileNotFound(e); - } catch (StorageException e) { - throw unavailableResultException(e); - } catch (Exception e) { - throw undefinedResultException("Error when \"generateDownloadUrl\"", e); } } @@ -64,10 +51,6 @@ public class FileStorageHandler implements FileStorageSrv.Iface { return storageService.getFileData(fileDataId); } catch (FileNotFoundException e) { throw fileNotFound(e); - } catch (StorageException e) { - throw unavailableResultException(e); - } catch (Exception e) { - throw undefinedResultException("Error when \"getFileData\"", e); } } @@ -75,14 +58,4 @@ public class FileStorageHandler implements FileStorageSrv.Iface { log.warn("File not found", e); return new FileNotFound(); } - - private WUnavailableResultException unavailableResultException(StorageException e) { - log.error("Error with storage", e); - return new WUnavailableResultException("Error with storage", e); - } - - private WUndefinedResultException undefinedResultException(String msg, Exception e) { - log.error(msg, e); - return new WUndefinedResultException(msg, e); - } } diff --git a/src/main/java/dev/vality/file/storage/service/S3Service.java b/src/main/java/dev/vality/file/storage/service/S3Service.java index 5ca9e79..e76dcf9 100644 --- a/src/main/java/dev/vality/file/storage/service/S3Service.java +++ b/src/main/java/dev/vality/file/storage/service/S3Service.java @@ -19,8 +19,6 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -34,8 +32,7 @@ import java.util.stream.Collectors; import static java.lang.String.format; -@Service -@ConditionalOnProperty(value = "s3-sdk-v2.enabled", havingValue = "false") +//@Service @Slf4j @RequiredArgsConstructor public class S3Service implements StorageService { diff --git a/src/main/java/dev/vality/file/storage/service/S3V2Service.java b/src/main/java/dev/vality/file/storage/service/S3V2Service.java index 63f45e4..38c3112 100644 --- a/src/main/java/dev/vality/file/storage/service/S3V2Service.java +++ b/src/main/java/dev/vality/file/storage/service/S3V2Service.java @@ -11,7 +11,6 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; @@ -29,7 +28,6 @@ import java.util.function.Predicate; import java.util.stream.Collectors; @Service -@ConditionalOnProperty(value = "s3-sdk-v2.enabled", havingValue = "true") @Slf4j @RequiredArgsConstructor public class S3V2Service implements StorageService { diff --git a/src/main/java/dev/vality/file/storage/servlet/FileStorageServlet.java b/src/main/java/dev/vality/file/storage/servlet/FileStorageServlet.java index 19416f5..09042eb 100644 --- a/src/main/java/dev/vality/file/storage/servlet/FileStorageServlet.java +++ b/src/main/java/dev/vality/file/storage/servlet/FileStorageServlet.java @@ -8,7 +8,7 @@ import javax.servlet.*; import javax.servlet.annotation.WebServlet; import java.io.IOException; -@WebServlet("/file_storage") +@WebServlet("/file_storage/v2") @RequiredArgsConstructor public class FileStorageServlet extends GenericServlet { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 6739b3c..d010113 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,19 +1,19 @@ info: - version: '@project.version@' + version: '${project.version}' stage: dev server: - port: '@server.port@' + port: '${server.port}' management: server: - port: '@management.port@' + port: '${management.port}' metrics: export: prometheus: enabled: false tags: - application: '@project.name@' + application: '${project.name}' endpoint: health: probes: @@ -30,24 +30,12 @@ management: spring: application: - name: '@project.name@' + name: '${project.name}' output: ansi: enabled: always -s3: - endpoint: 'http://127.0.0.1:9000' - bucket-name: 'files' - signing-region: 'RU' - client-protocol: 'http' - client-max-error-retry: 10 - # signer-override: 'S3SignerType' - signer-override: 'AWSS3V4SignerType' - access-key: 'minio' - secret-key: 'minio123' - s3-sdk-v2: - enabled: 'false' endpoint: 'http://127.0.0.1:9000' bucket-name: 'files-v2' region: 'RU' diff --git a/src/test/java/dev/vality/file/storage/CatchExceptionsTest.java b/src/test/java/dev/vality/file/storage/CatchExceptionsTest.java index 9fe739e..7338464 100644 --- a/src/test/java/dev/vality/file/storage/CatchExceptionsTest.java +++ b/src/test/java/dev/vality/file/storage/CatchExceptionsTest.java @@ -6,6 +6,7 @@ import com.amazonaws.services.s3.transfer.TransferManager; import dev.vality.woody.api.flow.error.WRuntimeException; import dev.vality.woody.thrift.impl.http.THSpawnClientBuilder; import org.apache.thrift.TException; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.springframework.boot.test.context.SpringBootTest; @@ -18,6 +19,7 @@ import java.util.UUID; import static org.junit.jupiter.api.Assertions.assertThrows; +@Disabled @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) public class CatchExceptionsTest { @@ -33,7 +35,7 @@ public class CatchExceptionsTest { @Test public void shouldThrowException() throws URISyntaxException, TException { FileStorageSrv.Iface fileStorageCli = new THSpawnClientBuilder() - .withAddress(new URI("http://localhost:" + port + "/file_storage")) + .withAddress(new URI("http://localhost:" + port + "/file_storage/v2")) .withNetworkTimeout(555000) .build(FileStorageSrv.Iface.class); diff --git a/src/test/java/dev/vality/file/storage/FileStorageTest.java b/src/test/java/dev/vality/file/storage/FileStorageTest.java index 4b7ea80..b795ec8 100644 --- a/src/test/java/dev/vality/file/storage/FileStorageTest.java +++ b/src/test/java/dev/vality/file/storage/FileStorageTest.java @@ -51,7 +51,7 @@ public abstract class FileStorageTest { @BeforeEach public void setUp() throws Exception { fileStorageClient = new THSpawnClientBuilder() - .withAddress(new URI("http://localhost:" + port + "/file_storage")) + .withAddress(new URI("http://localhost:" + port + "/file_storage/v2")) .withNetworkTimeout(TIMEOUT) .build(FileStorageSrv.Iface.class); } diff --git a/src/test/java/dev/vality/file/storage/awssdks3v2/WithCephTest.java b/src/test/java/dev/vality/file/storage/awssdks3v2/WithCephTest.java index 064f4b2..9d5546e 100644 --- a/src/test/java/dev/vality/file/storage/awssdks3v2/WithCephTest.java +++ b/src/test/java/dev/vality/file/storage/awssdks3v2/WithCephTest.java @@ -2,7 +2,9 @@ package dev.vality.file.storage.awssdks3v2; import dev.vality.file.storage.FileStorageTest; import dev.vality.testcontainers.annotations.ceph.CephTestcontainerSingleton; +import org.junit.jupiter.api.Disabled; +@Disabled @CephTestcontainerSingleton( properties = {"s3-sdk-v2.enabled=true", "s3-sdk-v2.region=us-east-1"}, bucketName = "awssdks3v2") diff --git a/src/test/java/dev/vality/file/storage/awssdks3v2/WithMinioTest.java b/src/test/java/dev/vality/file/storage/awssdks3v2/WithMinioTest.java index ea51baf..d615023 100644 --- a/src/test/java/dev/vality/file/storage/awssdks3v2/WithMinioTest.java +++ b/src/test/java/dev/vality/file/storage/awssdks3v2/WithMinioTest.java @@ -3,8 +3,6 @@ package dev.vality.file.storage.awssdks3v2; import dev.vality.file.storage.FileStorageTest; import dev.vality.testcontainers.annotations.minio.MinioTestcontainerSingleton; -@MinioTestcontainerSingleton( - properties = "s3-sdk-v2.enabled=true", - bucketName = "awssdks3v2") +@MinioTestcontainerSingleton(bucketName = "awssdks3v2") public class WithMinioTest extends FileStorageTest { } diff --git a/src/test/java/dev/vality/file/storage/s3signer/WithCephTest.java b/src/test/java/dev/vality/file/storage/s3signer/WithCephTest.java index e0afa9a..14cbae8 100644 --- a/src/test/java/dev/vality/file/storage/s3signer/WithCephTest.java +++ b/src/test/java/dev/vality/file/storage/s3signer/WithCephTest.java @@ -2,7 +2,9 @@ package dev.vality.file.storage.s3signer; import dev.vality.file.storage.FileStorageTest; import dev.vality.testcontainers.annotations.ceph.CephTestcontainerSingleton; +import org.junit.jupiter.api.Disabled; +@Disabled @CephTestcontainerSingleton(bucketName = "s3signer") public class WithCephTest extends FileStorageTest { } diff --git a/src/test/java/dev/vality/file/storage/s3signer/WithMinioTest.java b/src/test/java/dev/vality/file/storage/s3signer/WithMinioTest.java index 4f9cfaa..5ba8643 100644 --- a/src/test/java/dev/vality/file/storage/s3signer/WithMinioTest.java +++ b/src/test/java/dev/vality/file/storage/s3signer/WithMinioTest.java @@ -2,7 +2,9 @@ package dev.vality.file.storage.s3signer; import dev.vality.file.storage.FileStorageTest; import dev.vality.testcontainers.annotations.minio.MinioTestcontainerSingleton; +import org.junit.jupiter.api.Disabled; +@Disabled @MinioTestcontainerSingleton(bucketName = "s3signer") public class WithMinioTest extends FileStorageTest { } diff --git a/src/test/java/dev/vality/file/storage/s3v4signer/WithCephTest.java b/src/test/java/dev/vality/file/storage/s3v4signer/WithCephTest.java index b4acc7a..d2bc1cb 100644 --- a/src/test/java/dev/vality/file/storage/s3v4signer/WithCephTest.java +++ b/src/test/java/dev/vality/file/storage/s3v4signer/WithCephTest.java @@ -2,7 +2,9 @@ package dev.vality.file.storage.s3v4signer; import dev.vality.file.storage.FileStorageTest; import dev.vality.testcontainers.annotations.ceph.CephTestcontainerSingleton; +import org.junit.jupiter.api.Disabled; +@Disabled @CephTestcontainerSingleton( properties = "s3.signer-override=AWSS3V4SignerType", bucketName = "s3v4signer") diff --git a/src/test/java/dev/vality/file/storage/s3v4signer/WithMinioTest.java b/src/test/java/dev/vality/file/storage/s3v4signer/WithMinioTest.java index 5590ce5..9adc568 100644 --- a/src/test/java/dev/vality/file/storage/s3v4signer/WithMinioTest.java +++ b/src/test/java/dev/vality/file/storage/s3v4signer/WithMinioTest.java @@ -2,7 +2,9 @@ package dev.vality.file.storage.s3v4signer; import dev.vality.file.storage.FileStorageTest; import dev.vality.testcontainers.annotations.minio.MinioTestcontainerSingleton; +import org.junit.jupiter.api.Disabled; +@Disabled @MinioTestcontainerSingleton( properties = "s3.signer-override=AWSS3V4SignerType", bucketName = "s3v4signer")