diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioBlobStoreGCAlgorithmTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioBlobStoreGCAlgorithmTest.java index 7110e728577..9e02f2120de 100644 --- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioBlobStoreGCAlgorithmTest.java +++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioBlobStoreGCAlgorithmTest.java @@ -31,6 +31,7 @@ import org.apache.james.server.blob.deduplication.BloomFilterGCAlgorithmContract; import org.apache.james.server.blob.deduplication.GenerationAwareBlobId; import org.apache.james.server.blob.deduplication.MinIOGenerationAwareBlobId; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.RegisterExtension; @@ -38,7 +39,7 @@ public class S3MinioBlobStoreGCAlgorithmTest implements BloomFilterGCAlgorithmContract { - private BlobStoreDAO blobStoreDAO; + private S3BlobStoreDAO blobStoreDAO; @RegisterExtension static S3MinioExtension minoExtension = new S3MinioExtension(); @@ -52,6 +53,7 @@ void beforeEach() { .region(DockerAwsS3Container.REGION) .uploadRetrySpec(Optional.of(Retry.backoff(3, java.time.Duration.ofSeconds(1)) .filter(UPLOAD_RETRY_EXCEPTION_PREDICATE))) + .httpConcurrency(Optional.of(200)) .build(); S3ClientFactory s3ClientFactory = new S3ClientFactory(s3Configuration, new RecordingMetricFactory(), new NoopGaugeRegistry()); @@ -61,6 +63,13 @@ void beforeEach() { blobStoreDAO = new S3BlobStoreDAO(s3ClientFactory, s3Configuration, minIOGenerationAwareBlobIdFactory, S3RequestOption.DEFAULT); } + @AfterEach + void tearDown() throws Exception { + blobStoreDAO.deleteAllBuckets().block(); + + Thread.sleep(1000); + } + @Override public BlobStoreDAO blobStoreDAO() { return blobStoreDAO; diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioDocker.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioDocker.java index d53482a39f2..0bfbd2ea46b 100644 --- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioDocker.java +++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioDocker.java @@ -23,7 +23,6 @@ import java.util.UUID; import org.apache.http.client.utils.URIBuilder; -import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.Wait; @@ -34,13 +33,12 @@ public class S3MinioDocker { - public static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse("minio/minio") - .withTag("RELEASE.2025-06-13T11-33-47Z"); + public static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse("chrislusf/seaweedfs") + .withTag("4.07"); - public static final int MINIO_PORT = 9000; - public static final int MINIO_WEB_ADMIN_PORT = 9090; - public static final String MINIO_ROOT_USER = "minio"; - public static final String MINIO_ROOT_PASSWORD = "minio123"; + public static final int S3_PORT = 8333; + public static final String S3_ACCESS_KEY = "testaccesskey"; + public static final String S3_SECRET_KEY = "testsecretkey"; private final GenericContainer container; @@ -55,25 +53,19 @@ public S3MinioDocker(Network network) { private GenericContainer getContainer() { return new GenericContainer<>(DOCKER_IMAGE_NAME) - .withExposedPorts(MINIO_PORT, MINIO_WEB_ADMIN_PORT) - .withEnv("MINIO_ROOT_USER", MINIO_ROOT_USER) - .withEnv("MINIO_ROOT_PASSWORD", MINIO_ROOT_PASSWORD) - .withCommand("server", "--certs-dir", "/opt/minio/certs", "/data", "--console-address", ":" + MINIO_WEB_ADMIN_PORT) - .withClasspathResourceMapping("/minio/private.key", - "/opt/minio/certs/private.key", - BindMode.READ_ONLY) - .withClasspathResourceMapping("/minio/public.crt", - "/opt/minio/certs/public.crt", - BindMode.READ_ONLY) - .waitingFor(Wait.forLogMessage(".*MinIO Object Storage Server.*", 1) + .withExposedPorts(S3_PORT) + .withEnv("AWS_ACCESS_KEY_ID", S3_ACCESS_KEY) + .withEnv("AWS_SECRET_ACCESS_KEY", S3_SECRET_KEY) + .withCommand("mini", + "-dir", "/data") + .waitingFor(Wait.forLogMessage(".*Lock owner changed.*", 1) .withStartupTimeout(Duration.ofMinutes(2))) - .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("james-minio-s3-test-" + UUID.randomUUID())); + .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("james-seaweedfs-s3-test-" + UUID.randomUUID())); } public void start() { if (!container.isRunning()) { container.start(); - setupMC(); } } @@ -85,25 +77,12 @@ public AwsS3AuthConfiguration getAwsS3AuthConfiguration() { Preconditions.checkArgument(container.isRunning(), "Container is not running"); return AwsS3AuthConfiguration.builder() .endpoint(Throwing.supplier(() -> new URIBuilder() - .setScheme("https") + .setScheme("http") .setHost(container.getHost()) - .setPort(container.getMappedPort(MINIO_PORT)) + .setPort(container.getMappedPort(S3_PORT)) .build()).get()) - .accessKeyId(MINIO_ROOT_USER) - .secretKey(MINIO_ROOT_PASSWORD) - .trustAll(true) + .accessKeyId(S3_ACCESS_KEY) + .secretKey(S3_SECRET_KEY) .build(); } - - private void setupMC() { - Preconditions.checkArgument(container.isRunning(), "Container is not running"); - Throwing.runnable(() -> container.execInContainer("mc", "alias", "set", "--insecure", "james", "https://localhost:9000", MINIO_ROOT_USER, MINIO_ROOT_PASSWORD)).run(); - } - - public void flushAll() { - // Remove all objects - Throwing.runnable(() -> container.execInContainer("mc", "--insecure", "rm", "--recursive", "--force", "--dangerous", "james/")).run(); - // Remove all buckets - Throwing.runnable(() -> container.execInContainer("mc", "--insecure", "rb", "--force", "--dangerous", "james/")).run(); - } } diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioExtension.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioExtension.java index 12d95b3170b..833d1889871 100644 --- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioExtension.java +++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioExtension.java @@ -38,11 +38,6 @@ public void afterAll(ExtensionContext extensionContext) { s3MinioDocker.stop(); } - @Override - public void afterEach(ExtensionContext extensionContext) { - s3MinioDocker.flushAll(); - } - @Override public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { return parameterContext.getParameter().getType() == S3MinioDocker.class; diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java index dc4a4e3ba3b..a25ca21c1d4 100644 --- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java +++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java @@ -24,28 +24,22 @@ import static org.apache.james.blob.api.BlobStoreDAOFixture.TEST_BUCKET_NAME; import static org.apache.james.blob.objectstorage.aws.S3BlobStoreConfiguration.UPLOAD_RETRY_EXCEPTION_PREDICATE; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.Optional; -import java.util.concurrent.ExecutionException; -import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BlobStoreDAO; import org.apache.james.blob.api.BlobStoreDAOContract; import org.apache.james.blob.api.TestBlobId; import org.apache.james.metrics.api.NoopGaugeRegistry; import org.apache.james.metrics.tests.RecordingMetricFactory; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.util.retry.Retry; -import software.amazon.awssdk.services.s3.model.S3Exception; public class S3MinioTest implements BlobStoreDAOContract { @@ -55,8 +49,8 @@ public class S3MinioTest implements BlobStoreDAOContract { private static S3BlobStoreDAO testee; private static S3ClientFactory s3ClientFactory; - @BeforeAll - static void setUp() { + @BeforeEach + void setUp() { AwsS3AuthConfiguration awsS3AuthConfiguration = minoExtension.minioDocker().getAwsS3AuthConfiguration(); S3BlobStoreConfiguration s3Configuration = S3BlobStoreConfiguration.builder() @@ -70,29 +64,12 @@ static void setUp() { testee = new S3BlobStoreDAO(s3ClientFactory, s3Configuration, new TestBlobId.Factory(), S3RequestOption.DEFAULT); } - @AfterAll - static void tearDownClass() { + @AfterEach + void tearDown() throws Exception { + testee.deleteAllBuckets().block(); s3ClientFactory.close(); - } - - @BeforeEach - void beforeEach() throws Exception { - // Why? https://github.com/apache/james-project/pull/1981#issuecomment-2380396460 - createBucket(TEST_BUCKET_NAME.asString()); - } - - private void createBucket(String bucketName) throws Exception { - s3ClientFactory.get().createBucket(builder -> builder.bucket(bucketName)) - .get(); - } - private void deleteBucket(String bucketName) { - try { - s3ClientFactory.get().deleteBucket(builder -> builder.bucket(bucketName)) - .get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException("Error while deleting bucket", e); - } + Thread.sleep(1000); } @Override @@ -100,31 +77,12 @@ public BlobStoreDAO testee() { return testee; } - @Test - void saveWillThrowWhenBlobIdHasSlashCharacters() { - BlobId invalidBlobId = new TestBlobId("test-blob//id"); - assertThatThrownBy(() -> Mono.from(testee.save(TEST_BUCKET_NAME, invalidBlobId, SHORT_BYTEARRAY)).block()) - .isInstanceOf(S3Exception.class) - .hasMessageContaining("Object name contains unsupported characters"); - } - @Test void saveShouldWorkWhenValidBlobId() { Mono.from(testee.save(TEST_BUCKET_NAME, TEST_BLOB_ID, SHORT_BYTEARRAY)).block(); assertThat(Mono.from(testee.readBytes(TEST_BUCKET_NAME, TEST_BLOB_ID)).block()).isEqualTo(SHORT_BYTEARRAY); } - @Test - @Override - public void listBucketsShouldReturnEmptyWhenNone() { - deleteBucket(TEST_BUCKET_NAME.asString()); - - BlobStoreDAO store = testee(); - - assertThat(Flux.from(store.listBuckets()).collectList().block()) - .isEmpty(); - } - @Test @Override @Disabled("S3minio return `Connection: close` in header response, https://github.com/apache/james-project/pull/1981#issuecomment-2380396460") diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3WithMinIOGenerationAwareBlobIdTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3WithMinIOGenerationAwareBlobIdTest.java index a13a9e772c7..b7848301db7 100644 --- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3WithMinIOGenerationAwareBlobIdTest.java +++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3WithMinIOGenerationAwareBlobIdTest.java @@ -68,9 +68,6 @@ public class S3WithMinIOGenerationAwareBlobIdTest implements BlobStoreContract { void beforeEach() throws Exception { blobIdFactory = new MinIOGenerationAwareBlobId.Factory(clock, GenerationAwareBlobId.Configuration.DEFAULT, new PlainBlobId.Factory()); testee = createBlobStore(blobIdFactory); - - // Why? https://github.com/apache/james-project/pull/1981#issuecomment-2380396460 - createBucket(testee.getDefaultBucketName().asString()); } @AfterEach diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3BlobStoreDAOWithSSECTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3BlobStoreDAOWithSSECTest.java index 4c55a699fa6..d0048ed7a90 100644 --- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3BlobStoreDAOWithSSECTest.java +++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3BlobStoreDAOWithSSECTest.java @@ -19,13 +19,10 @@ package org.apache.james.blob.objectstorage.aws.sse; -import static org.apache.james.blob.api.BlobStoreDAOFixture.TEST_BUCKET_NAME; import static org.apache.james.blob.objectstorage.aws.JamesS3MetricPublisher.DEFAULT_S3_METRICS_PREFIX; import static org.apache.james.blob.objectstorage.aws.S3BlobStoreConfiguration.UPLOAD_RETRY_EXCEPTION_PREDICATE; -import static org.assertj.core.api.Assertions.assertThat; import java.util.Optional; -import java.util.concurrent.ExecutionException; import org.apache.james.blob.api.BlobStoreDAO; import org.apache.james.blob.api.BlobStoreDAOContract; @@ -39,13 +36,12 @@ import org.apache.james.blob.objectstorage.aws.S3RequestOption; import org.apache.james.metrics.api.NoopGaugeRegistry; import org.apache.james.metrics.tests.RecordingMetricFactory; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import reactor.core.publisher.Flux; import reactor.util.retry.Retry; import software.amazon.awssdk.services.s3.S3AsyncClient; @@ -57,8 +53,8 @@ public class S3BlobStoreDAOWithSSECTest implements BlobStoreDAOContract, S3SSECC private static S3BlobStoreDAO testee; private static S3ClientFactory s3ClientFactory; - @BeforeAll - static void setUp() throws Exception { + @BeforeEach + void setUp() throws Exception { S3BlobStoreConfiguration s3Configuration = S3BlobStoreConfiguration.builder() .authConfiguration(minoExtension.minioDocker().getAwsS3AuthConfiguration()) .region(Region.of(software.amazon.awssdk.regions.Region.EU_WEST_1.id())) @@ -75,13 +71,6 @@ static void setUp() throws Exception { testee = new S3BlobStoreDAO(s3ClientFactory, s3Configuration, new TestBlobId.Factory(), s3RequestOption); } - @BeforeEach - void beforeEach() throws Exception { - // Why? https://github.com/apache/james-project/pull/1981#issuecomment-2380396460 - s3ClientFactory.get().createBucket(builder -> builder.bucket(TEST_BUCKET_NAME.asString())) - .get(); - } - @Override public BlobStoreDAO testee() { return testee; @@ -92,24 +81,12 @@ public S3AsyncClient s3Client() { return s3ClientFactory.get(); } - private void deleteBucket(String bucketName) { - try { - s3ClientFactory.get().deleteBucket(builder -> builder.bucket(bucketName)) - .get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException("Error while deleting bucket", e); - } - } - - @Test - @Override - public void listBucketsShouldReturnEmptyWhenNone() { - deleteBucket(TEST_BUCKET_NAME.asString()); - - BlobStoreDAO store = testee(); + @AfterEach + void tearDown() throws Exception { + testee.deleteAllBuckets().block(); + s3ClientFactory.close(); - assertThat(Flux.from(store.listBuckets()).collectList().block()) - .isEmpty(); + Thread.sleep(1000); } @Test diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECContract.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECContract.java index f18033746fc..4940f740a41 100644 --- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECContract.java +++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECContract.java @@ -49,7 +49,7 @@ default void getObjectShouldFailWhenNotDefineCustomerKeyInHeaderRequest() { .build(), AsyncResponseTransformer.toBytes()) .thenApply(BytesWrapper::asByteArray) .get()) - .hasMessageContaining("The object was stored using a form of Server Side Encryption"); + .hasMessageContaining("Requests specifying Server Side Encryption with Customer provided keys must provide the customer key"); } @Test @@ -64,7 +64,7 @@ default void getObjectShouldFailWhenInvalidCustomerKeyInRequest() { .build(), AsyncResponseTransformer.toBytes()) .thenApply(BytesWrapper::asByteArray) .get()) - .hasMessageContaining("Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm"); + .hasMessageContaining("Invalid Request"); } @Test