diff --git a/multiapps-controller-persistence/pom.xml b/multiapps-controller-persistence/pom.xml index 12bc92d3f7..28afc20036 100644 --- a/multiapps-controller-persistence/pom.xml +++ b/multiapps-controller-persistence/pom.xml @@ -104,12 +104,8 @@ aws-s3 - com.azure - azure-storage-blob - - - com.azure - azure-core-http-okhttp + org.apache.jclouds.provider + azureblob org.apache.jclouds diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index 20b366fbd9..8c3c8226e0 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -28,9 +28,6 @@ requires transitive org.cloudfoundry.multiapps.controller.api; requires aliyun.sdk.oss; - requires com.azure.core; - requires com.azure.core.http.okhttp; - requires com.azure.storage.blob; requires com.fasterxml.jackson.annotation; requires com.fasterxml.jackson.databind; requires com.google.auth; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java index 5bd5769e19..4a2daa7c71 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java @@ -51,8 +51,6 @@ public final class Messages { // ERROR log messages: public static final String UPLOAD_STREAM_FAILED_TO_CLOSE = "Cannot close file upload stream"; - public static final String CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE = "Cannot parse container_uri of object store"; - public static final String MISSING_CONTAINER_URI_IN_THE_CREDENTIALS = "Missing container url in the credentials"; // WARN log messages: public static final String COULD_NOT_CLOSE_RESULT_SET = "Could not close result set."; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java deleted file mode 100644 index 2ceacb0ac5..0000000000 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java +++ /dev/null @@ -1,228 +0,0 @@ -package org.cloudfoundry.multiapps.controller.persistence.services; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URL; -import java.time.LocalDateTime; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import com.azure.core.http.HttpClient; -import com.azure.core.http.okhttp.OkHttpAsyncHttpClientBuilder; -import com.azure.core.http.policy.ExponentialBackoffOptions; -import com.azure.core.http.policy.RetryOptions; -import com.azure.storage.blob.BlobClient; -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.BlobServiceClient; -import com.azure.storage.blob.BlobServiceClientBuilder; -import com.azure.storage.blob.models.BlobItem; -import com.azure.storage.blob.models.BlobListDetails; -import com.azure.storage.blob.models.BlobRange; -import com.azure.storage.blob.models.BlobStorageException; -import com.azure.storage.blob.models.ListBlobsOptions; -import com.azure.storage.blob.models.ParallelTransferOptions; -import com.azure.storage.blob.options.BlobParallelUploadOptions; -import org.cloudfoundry.multiapps.controller.persistence.Messages; -import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; -import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; -import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; - -public class AzureObjectStoreFileStorage extends ObjectStoreFileStorage { - - private static final String SAS_TOKEN = "sas_token"; - private static final String CONTAINER_NAME = "container_name"; - private static final String CONTAINER_URI = "container_uri"; - private static final long MAX_SINGLE_UPLOAD_SIZE = 15L * 1024 * 1024; // 10MB - private static final long BLOCK_SIZE = 15L * 1024 * 1024; // 10MB - private static final int MAX_CONCURRENCY = 30; - private final HttpClient httpClient; - private final BlobContainerClient containerClient; - - public AzureObjectStoreFileStorage(Map credentials) { - this.containerClient = createContainerClient(credentials); - this.httpClient = new OkHttpAsyncHttpClientBuilder().build(); - } - - @Override - public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { - BlobClient blobClient = containerClient.getBlobClient(fileEntry.getId()); - try { - ParallelTransferOptions pto = new ParallelTransferOptions().setMaxSingleUploadSizeLong(MAX_SINGLE_UPLOAD_SIZE) - .setMaxConcurrency(MAX_CONCURRENCY) - .setBlockSizeLong(BLOCK_SIZE); - - BlobParallelUploadOptions blobParallelUploadOptions = new BlobParallelUploadOptions(content); - blobParallelUploadOptions.setParallelTransferOptions(pto); - blobParallelUploadOptions.setMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)); - - blobClient.uploadWithResponse(blobParallelUploadOptions, - ObjectStoreConstants.AZURE_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, null); - } catch (BlobStorageException e) { - throw new FileStorageException(e); - } - } - - @Override - public List getFileEntriesWithoutContent(List fileEntries) throws FileStorageException { - Set existingFiles = getAllEntriesNames(); - return fileEntries.stream() - .filter(fileEntry -> !existingFiles.contains(fileEntry.getId())) - .toList(); - } - - @Override - protected boolean existsInObjectStore(FileEntry fileEntry) { - return containerClient.getBlobClient(fileEntry.getId()) - .exists(); - } - - @Override - public void deleteFile(String id, String space) throws FileStorageException { - BlobClient blobClient = containerClient.getBlobClient(id); - try { - blobClient.deleteIfExists(); - } catch (BlobStorageException e) { - throw new FileStorageException(e); - } - } - - @Override - public void deleteFilesBySpaceIds(List spaceIds) throws FileStorageException { - removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceIds(blob.getMetadata(), spaceIds)); - } - - @Override - public void deleteFilesBySpaceAndNamespace(String space, String namespace) { - removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceAndNamespace(blob.getMetadata(), space, namespace)); - } - - @Override - public int deleteFilesModifiedBefore(LocalDateTime modificationTime) throws FileStorageException { - return removeBlobsByFilter( - blob -> ObjectStoreFilter.filterByModificationTime(blob.getMetadata(), blob.getName(), modificationTime)); - } - - @Override - public T processFileContent(String space, String id, FileContentProcessor fileContentProcessor) throws FileStorageException { - FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); - try (InputStream inputStream = openBlobInputStream(fileEntry)) { - return fileContentProcessor.process(inputStream); - } catch (Exception e) { - throw new FileStorageException(e); - } - } - - private InputStream openBlobInputStream(FileEntry fileEntry) throws FileStorageException { - BlobClient blobClient = containerClient.getBlobClient(fileEntry.getId()); - try { - return blobClient.openInputStream(); - } catch (BlobStorageException e) { - throw new FileStorageException(e); - } - } - - @Override - public InputStream openInputStream(String space, String id) throws FileStorageException { - FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); - return openBlobInputStream(fileEntry); - } - - @Override - public void testConnection() { - containerClient.getBlobClient("test"); - } - - @Override - public void deleteFilesByIds(List fileIds) throws FileStorageException { - removeBlobsByFilter(blob -> fileIds.contains(blob.getName())); - } - - @Override - public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) - throws FileStorageException { - FileEntry fileEntry = ObjectStoreMapper.createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); - BlobClient blobClient = containerClient.getBlobClient(fileEntry.getId()); - long contentSize = fileContentToProcess.getEndOffset() - fileContentToProcess.getStartOffset(); - BlobRange blobRange = new BlobRange(fileContentToProcess.getStartOffset(), contentSize); - - try { - return fileContentProcessor.process(blobClient.openInputStream(blobRange, null)); - } catch (IOException e) { - throw new FileStorageException(e); - } - } - - protected BlobContainerClient createContainerClient(Map credentials) { - BlobServiceClient serviceClient = new BlobServiceClientBuilder().endpoint(getContainerUriEndpoint(credentials)) - .retryOptions(createRetryOptions()) - .httpClient(httpClient) - .sasToken((String) credentials.get(SAS_TOKEN)) - .buildClient(); - - return serviceClient.getBlobContainerClient((String) credentials.get(CONTAINER_NAME)); - } - - public String getContainerUriEndpoint(Map credentials) { - if (!credentials.containsKey(CONTAINER_URI)) { - throw new IllegalStateException(Messages.MISSING_CONTAINER_URI_IN_THE_CREDENTIALS); - } - try { - URL containerUri = new URL((String) credentials.get(CONTAINER_URI)); - return new URL(containerUri.getProtocol(), containerUri.getHost(), containerUri.getPort(), "").toString(); - } catch (MalformedURLException e) { - throw new IllegalStateException(Messages.CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE, e); - } - } - - private RetryOptions createRetryOptions() { - ExponentialBackoffOptions exponentialBackoffOptions = new ExponentialBackoffOptions().setBaseDelay( - ObjectStoreConstants.OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) - .setMaxDelay( - ObjectStoreConstants.OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) - .setMaxRetries( - ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG); - - return new RetryOptions(exponentialBackoffOptions); - } - - private int removeBlobsByFilter(Predicate filter) { - Set blobNames = getEntryNames(filter); - int deletedBlobsResult = 0; - - if (blobNames.isEmpty()) { - return deletedBlobsResult; - } - for (String blobName : blobNames) { - BlobClient blobClient = containerClient.getBlobClient(blobName); - if (blobClient.deleteIfExists()) { - deletedBlobsResult++; - } - } - - return deletedBlobsResult; - } - - protected Set getEntryNames(Predicate filter) { - BlobListDetails blobListDetails = new BlobListDetails().setRetrieveMetadata(true); - ListBlobsOptions listBlobsOptions = new ListBlobsOptions().setDetails(blobListDetails); - - return containerClient.listBlobs(listBlobsOptions, ObjectStoreConstants.AZURE_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) - .stream() - .filter(filter) - .map(BlobItem::getName) - .collect(Collectors.toSet()); - } - - public Set getAllEntriesNames() { - ListBlobsOptions listOptions = new ListBlobsOptions(); - return containerClient.listBlobs(listOptions, ObjectStoreConstants.AZURE_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) - .stream() - .map(BlobItem::getName) - .collect(Collectors.toSet()); - } -} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index a314d15d7b..ea9c127ce2 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -1,5 +1,20 @@ package org.cloudfoundry.multiapps.controller.persistence.services; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.text.MessageFormat; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + import com.google.api.gax.retrying.RetrySettings; import com.google.auth.Credentials; import com.google.auth.oauth2.GoogleCredentials; @@ -13,32 +28,21 @@ import com.google.cloud.storage.StorageRetryStrategy; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.springframework.http.MediaType; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.Channels; -import java.text.MessageFormat; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - public class GcpObjectStoreFileStorage implements FileStorage { private final String bucketName; private final Storage storage; - private static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; private static final String BUCKET = "bucket"; + private static final int OBJECT_STORE_MAX_ATTEMPTS_CONFIG = 6; + private static final double OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; + private static final Duration OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); + private static final Duration OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS = Duration.ofSeconds(10); + private static final Duration OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS = Duration.ofMillis(250); + private static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; public GcpObjectStoreFileStorage(Map credentials) { this.bucketName = (String) credentials.get(BUCKET); @@ -51,12 +55,11 @@ protected Storage createObjectStoreStorage(Map credentials) { .setStorageRetryStrategy(StorageRetryStrategy.getUniformStorageRetryStrategy()) .setRetrySettings( RetrySettings.newBuilder() - .setMaxAttempts(ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG) - .setTotalTimeoutDuration(ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) - .setMaxRetryDelayDuration(ObjectStoreConstants.OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) - .setInitialRetryDelayDuration( - ObjectStoreConstants.OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) - .setRetryDelayMultiplier(ObjectStoreConstants.OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG) + .setMaxAttempts(OBJECT_STORE_MAX_ATTEMPTS_CONFIG) + .setTotalTimeoutDuration(OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) + .setMaxRetryDelayDuration(OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) + .setInitialRetryDelayDuration(OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) + .setRetryDelayMultiplier(OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG) .build()) .build() .getService(); @@ -105,24 +108,6 @@ public List getFileEntriesWithoutContent(List fileEntries) .toList(); } - @Override - public List getExistingFileEntries(List fileEntries) { - if (fileEntries.isEmpty()) { - return List.of(); - } - List blobIds = fileEntries.stream() - .map(fileEntry -> BlobId.of(bucketName, fileEntry.getId())) - .toList(); - List blobs = storage.get(blobIds); - Set existingBlobNames = blobs.stream() - .filter(Objects::nonNull) - .map(Blob::getName) - .collect(Collectors.toSet()); - return fileEntries.stream() - .filter(fileEntry -> existingBlobNames.contains(fileEntry.getId())) - .toList(); - } - @Override public void deleteFile(String id, String space) { deleteFileWithGeneration(id); diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java deleted file mode 100644 index 1721348cac..0000000000 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.cloudfoundry.multiapps.controller.persistence.util; - -import java.time.Duration; - -public class ObjectStoreConstants { - - private ObjectStoreConstants() { - } - - public static final int OBJECT_STORE_MAX_ATTEMPTS_CONFIG = 6; - public static final double OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; - public static final Duration OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); - public static final Duration AZURE_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(30); - public static final Duration OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS = Duration.ofSeconds(10); - public static final Duration OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS = Duration.ofMillis(250); -} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java deleted file mode 100644 index 1200462686..0000000000 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java +++ /dev/null @@ -1,347 +0,0 @@ -package org.cloudfoundry.multiapps.controller.persistence.services; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.math.BigInteger; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Stream; - -import com.azure.core.http.rest.PagedIterable; -import com.azure.storage.blob.BlobClient; -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.models.BlobItem; -import com.azure.storage.blob.models.BlobStorageException; -import com.azure.storage.blob.specialized.BlobInputStream; -import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -class AzureObjectStoreFileStorageTest { - - @Mock - private BlobContainerClient blobContainerClient; - - @Mock - private BlobClient blobClient; - - @Mock - private PagedIterable pagedIterable; - - @Mock - private FileContentProcessor fileContentProcessor; - - @Mock - private BlobInputStream blobInputStream; - - private AzureObjectStoreFileStorage fileStorage; - private InputStream inputStream = new ByteArrayInputStream(new byte[] {}); - private final String TEST_SPACE_ID = UUID.randomUUID() - .toString(); - private final String TEST_SPACE_ID_2 = UUID.randomUUID() - .toString(); - private final String TEST_ID = UUID.randomUUID() - .toString(); - private final String TEST_ID_2 = UUID.randomUUID() - .toString(); - - private final String NAMESPACE = "namespace"; - private final String NAMESPACE_2 = "namespace_2"; - - @BeforeEach - void setUp() throws Exception { - MockitoAnnotations.openMocks(this) - .close(); - - fileStorage = new AzureObjectStoreFileStorage(Map.of()) { - - @Override - protected BlobContainerClient createContainerClient(Map credentials) { - return blobContainerClient; - } - }; - - when(blobContainerClient.getBlobClient(anyString())).thenReturn(blobClient); - } - - @Test - void testAddFileWithSuccessfulUpload() throws FileStorageException { - when(blobClient.uploadWithResponse(any(), any(), any())).thenReturn(null); - FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); - - fileStorage.addFile(fileEntry, inputStream); - - verify(blobClient).uploadWithResponse(any(), any(), any()); - } - - @Test - void testAddFileWithFailedUpload() { - doThrow(BlobStorageException.class).when(blobClient) - .uploadWithResponse(any(), any(), any()); - FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); - - assertThrows(FileStorageException.class, () -> fileStorage.addFile(fileEntry, inputStream)); - } - - @Test - void testGetFileEntriesWithoutContentWithoutMatches() throws FileStorageException { - setupDeleteMethods(createBlobItem(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now()), - createBlobItem(TEST_ID_2, TEST_SPACE_ID_2, NAMESPACE, LocalDateTime.now())); - FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); - - List fileEntries = fileStorage.getFileEntriesWithoutContent(List.of(fileEntry)); - - assertEquals(0, fileEntries.size()); - verify(blobContainerClient).listBlobs(any(), any()); - } - - @Test - void testGetFileEntriesWithoutContent() throws FileStorageException { - setupDeleteMethods(createSecondTestBlobItem()); - - FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); - List fileEntries = fileStorage.getFileEntriesWithoutContent(List.of(fileEntry)); - - assertEquals(TEST_ID, fileEntries.get(0) - .getId()); - assertEquals(TEST_SPACE_ID, fileEntries.get(0) - .getSpace()); - assertEquals(1, fileEntries.size()); - verify(blobContainerClient).listBlobs(any(), any()); - } - - @Test - void testDeleteFile() throws FileStorageException { - when(blobClient.deleteIfExists()).thenReturn(false); - - fileStorage.deleteFile(TEST_ID, TEST_SPACE_ID); - verify(blobClient).deleteIfExists(); - } - - @Test - void testTestConnection() { - fileStorage.testConnection(); - verify(blobContainerClient).getBlobClient("test"); - } - - @Test - void testGetContainerUriEndpointWithEmptyCredentials() { - assertThrows(IllegalStateException.class, () -> fileStorage.getContainerUriEndpoint(Map.of())); - } - - @Test - void testGetContainerUriEndpointWithInvalidContainerUri() { - assertThrows(IllegalStateException.class, () -> fileStorage.getContainerUriEndpoint(Map.of("container_uri", ""))); - } - - @Test - void testGetContainerUriEndpointWithValidContainerUri() { - assertEquals("https://google.com", fileStorage.getContainerUriEndpoint(Map.of("container_uri", "https://google.com"))); - } - - @Test - void testDeleteFileWithException() { - doThrow(new BlobStorageException("", null, null)).when(blobClient) - .deleteIfExists(); - - assertThrows(FileStorageException.class, () -> fileStorage.deleteFile(TEST_ID, TEST_SPACE_ID)); - verify(blobClient).deleteIfExists(); - } - - @Test - void testProcessFileContent() throws FileStorageException, IOException { - when(blobClient.openInputStream()).thenReturn(blobInputStream); - fileStorage.processFileContent(TEST_SPACE_ID, TEST_ID, fileContentProcessor); - - verify(fileContentProcessor).process(blobInputStream); - } - - @Test - void testProcessFileContentWithException() { - doThrow(new BlobStorageException("", null, null)).when(blobClient) - .openInputStream(); - - assertThrows(FileStorageException.class, () -> fileStorage.processFileContent(TEST_SPACE_ID, TEST_ID, fileContentProcessor)); - } - - @Test - void testDeleteFilesBySpaceIdsWithAllMatchingItems() throws FileStorageException { - setupDeleteMethods(createFirstTestBlobItem(), createSecondTestBlobItem()); - - fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID, TEST_SPACE_ID_2)); - - verify(blobClient, times(2)).deleteIfExists(); - } - - @Test - void testDeleteFilesBySpaceIdsWithOneMatchingItem() throws FileStorageException { - setupDeleteMethods(createFirstTestBlobItem(), createSecondTestBlobItem()); - - fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID)); - - verify(blobClient).deleteIfExists(); - } - - @Test - void testDeleteFilesBySpaceIdsWithoutMatchingItem() throws FileStorageException { - setupDeleteMethods(); - - fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID, TEST_SPACE_ID_2)); - - verify(blobClient, times(0)).deleteIfExists(); - } - - @Test - void testDeleteFilesBySpaceAndNamespaceWithOneMatch() { - setupDeleteMethods(createFirstTestBlobItem(), createSecondTestBlobItem()); - when(blobContainerClient.listBlobs()).thenReturn(pagedIterable); - when(blobClient.deleteIfExists()).thenReturn(true); - - fileStorage.deleteFilesBySpaceAndNamespace(TEST_SPACE_ID, NAMESPACE); - - verify(blobClient, times(1)).deleteIfExists(); - } - - @Test - void testDeleteFilesModifiedBefore() throws FileStorageException { - long currentMillis = System.currentTimeMillis(); - long oldFilesTtl = 1000 * 60 * 10; // 10min - setupDeleteMethods(createFirstTestBlobItem(), createSecondTestBlobItem()); - - int deletedFiles = fileStorage.deleteFilesModifiedBefore(LocalDateTime.ofInstant(Instant.ofEpochMilli(currentMillis - oldFilesTtl), - getZoneId())); - - assertEquals(2, deletedFiles); - } - - @Test - void testOpenInputStream() throws FileStorageException { - when(blobClient.openInputStream()).thenReturn(null); - - fileStorage.openInputStream(TEST_SPACE_ID_2, TEST_ID); - - verify(blobContainerClient).getBlobClient(TEST_ID); - verify(blobClient).openInputStream(); - } - - @Test - void testOpenInputStreamWithException() { - doThrow(new BlobStorageException(null, null, null)).when(blobClient) - .openInputStream(); - assertThrows(FileStorageException.class, () -> fileStorage.openInputStream(TEST_SPACE_ID_2, TEST_ID)); - - verify(blobContainerClient).getBlobClient(TEST_ID); - verify(blobClient).openInputStream(); - } - - @Test - void testDeleteFilesByIds() throws FileStorageException { - setupDeleteMethods(createFirstTestBlobItem(), createSecondTestBlobItem()); - - fileStorage.deleteFilesByIds(List.of(TEST_ID)); - - verify(blobClient).deleteIfExists(); - } - - @Test - void getExistingFileEntriesWhenAllEntriesExist() throws FileStorageException { - when(blobClient.exists()).thenReturn(true); - FileEntry firstEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); - FileEntry secondEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); - - List result = fileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); - - assertEquals(2, result.size()); - List returnedIds = result.stream() - .map(FileEntry::getId) - .toList(); - assertTrue(returnedIds.contains(TEST_ID)); - assertTrue(returnedIds.contains(TEST_ID_2)); - } - - @Test - void getExistingFileEntriesWhenNoEntriesExist() throws FileStorageException { - when(blobClient.exists()).thenReturn(false); - FileEntry firstEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); - FileEntry secondEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); - - List result = fileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); - - assertTrue(result.isEmpty()); - } - - @Test - void getExistingFileEntriesWhenSomeEntriesExist() throws FileStorageException { - when(blobContainerClient.getBlobClient(TEST_ID)).thenReturn(blobClient); - BlobClient nonExistingBlobClient = mock(BlobClient.class); - when(blobContainerClient.getBlobClient(TEST_ID_2)).thenReturn(nonExistingBlobClient); - when(blobClient.exists()).thenReturn(true); - when(nonExistingBlobClient.exists()).thenReturn(false); - FileEntry existingEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); - FileEntry nonExistingEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); - - List result = fileStorage.getExistingFileEntries(List.of(existingEntry, nonExistingEntry)); - - assertEquals(1, result.size()); - assertEquals(TEST_ID, result.getFirst().getId()); - } - - private void setupDeleteMethods(BlobItem... blobItems) { - when(pagedIterable.stream()).thenReturn(Stream.of(blobItems)); - when(blobContainerClient.listBlobs(any(), any())).thenReturn(pagedIterable); - when(blobClient.deleteIfExists()).thenReturn(true); - } - - public static FileEntry createFileEntry(String space, String id) { - return ImmutableFileEntry.builder() - .space(space) - .size(BigInteger.TEN) - .modified(LocalDateTime.now()) - .id(id) - .build(); - } - - private BlobItem createFirstTestBlobItem() { - long currentMillis = System.currentTimeMillis(); - long pastMoment = currentMillis - 1000 * 60 * 15; // before 15min - return createBlobItem(TEST_ID, TEST_SPACE_ID, NAMESPACE, - LocalDateTime.ofInstant(Instant.ofEpochMilli(pastMoment), getZoneId())); - } - - private BlobItem createSecondTestBlobItem() { - long currentMillis = System.currentTimeMillis(); - long pastMoment = currentMillis - 1000 * 60 * 15; // before 15min - return createBlobItem(TEST_ID_2, TEST_SPACE_ID_2, NAMESPACE_2, - LocalDateTime.ofInstant(Instant.ofEpochMilli(pastMoment), getZoneId())); - } - - private ZoneId getZoneId() { - return ZoneId.of("UTC"); - } - - private BlobItem createBlobItem(String name, String spaceId, String namespace, LocalDateTime modificationTime) { - BlobItem blobItem = new BlobItem(); - blobItem.setName(name); - blobItem.setMetadata(Map.of("space", spaceId, "namespace", namespace, "modified", modificationTime.toString())); - return blobItem; - } -} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java index 81196ee641..24d2c046d5 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java @@ -25,6 +25,7 @@ public final class Messages { public static final String MISSING_PROPERTIES_FOR_CREATING_THE_SPECIFIC_PROVIDER = "Missing properties for creating the specific provider!"; public static final String DEPLOY_FROM_URL_WRONG_CREDENTIALS_FOR_JOB_WITH_ID = "Credentials to {0} are wrong. Make sure that they are correct. Job id: {1}"; public static final String JOB_NOT_UPDATED_FOR_0_SECONDS = "Job not updated for {0} seconds"; + public static final String FAILED_TO_CREATE_BLOB_STORE_CONTEXT = "Failed to create BlobStoreContext"; // Audit log messages @@ -57,8 +58,6 @@ public final class Messages { // WARN log messages - public static final String NO_OBJECTSTORE_PROVIDER_FOUND_FOR_0 = "No ObjectStore provider found for {0}!"; - // INFO log messages public static final String ALM_SERVICE_ENV_INITIALIZED = "Deploy service environment initialized"; public static final String STORING_TOKEN_FOR_USER_WITH_GUID_0_WHICH_EXPIRES_AT_1 = "Storing token for user with GUID \"{0}\" which expires at: {1}"; diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java index 18020e1a0a..0001e13be1 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java @@ -16,13 +16,13 @@ import org.apache.commons.lang3.StringUtils; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.UriUtil; -import org.cloudfoundry.multiapps.controller.persistence.services.AzureObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; +import org.cloudfoundry.multiapps.controller.web.configuration.service.ImmutableObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator; import org.jclouds.ContextBuilder; @@ -85,6 +85,10 @@ public FileStorage createObjectStoreFromFirstReachableProvider(Map gcpObjectStoreOpt = tryToCreateGcpObjectStore(exceptions); + if (gcpObjectStoreOpt.isPresent()) { + return gcpObjectStoreOpt.get(); + } throw buildNoValidObjectStoreException(exceptions); } @@ -93,11 +97,14 @@ private Optional createObjectStoreBasedOnProvider(String objectStor Map exceptions) { Optional objectStoreServiceInfoOptional = getAppropriateProvider(objectStoreProviderName, providersServiceInfo); - if (objectStoreServiceInfoOptional.isEmpty()) { - LOGGER.warn(MessageFormat.format(Messages.NO_OBJECTSTORE_PROVIDER_FOUND_FOR_0, objectStoreProviderName)); - return Optional.empty(); + Optional createdObjectStore; + if (objectStoreServiceInfoOptional.isPresent()) { + ObjectStoreServiceInfo objectStoreServiceInfo = objectStoreServiceInfoOptional.get(); + createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); + } else { + createdObjectStore = tryToCreateGcpObjectStore(exceptions); } - return tryToCreateObjectStore(objectStoreServiceInfoOptional.get(), exceptions); + return createdObjectStore; } private Optional getAppropriateProvider(String objectStoreProviderName, @@ -108,6 +115,12 @@ private Optional getAppropriateProvider(String objectSto .findFirst(); } + private Optional tryToCreateGcpObjectStore(Map exceptions) { + return tryToCreateObjectStore(ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.GOOGLE_CLOUD_STORAGE) + .build(), exceptions); + } + private Optional tryToCreateObjectStore(ObjectStoreServiceInfo objectStoreServiceInfo, Map exceptions) { try { @@ -122,14 +135,12 @@ private Optional tryToCreateObjectStore(ObjectStoreServiceInfo obje } private FileStorage getFileStorageBasedOnProvider(ObjectStoreServiceInfo objectStoreServiceInfo) { - return switch (objectStoreServiceInfo.getProvider()) { - case Constants.GOOGLE_CLOUD_STORAGE -> createGcpFileStorage(objectStoreServiceInfo); - case Constants.AZUREBLOB -> createAzureFileStorage(objectStoreServiceInfo); - default -> { - BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); - yield createFileStorage(objectStoreServiceInfo, context); - } - }; + if (Constants.GOOGLE_CLOUD_STORAGE.equals(objectStoreServiceInfo.getProvider())) { + return createGcpFileStorage(); + } else { + BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); + return createFileStorage(objectStoreServiceInfo, context); + } } private boolean isObjectStoreEnvValid(String objectStoreProviderName) { @@ -215,12 +226,9 @@ protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo return new JCloudsObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer()); } - protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); - } - - protected AzureObjectStoreFileStorage createAzureFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new AzureObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + protected GcpObjectStoreFileStorage createGcpFileStorage() { + Map credentials = getServiceCredentials(); + return new GcpObjectStoreFileStorage(credentials); } @Override diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java index a822250a15..86c445e306 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java @@ -1,7 +1,5 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import java.util.Map; - import com.google.common.base.Supplier; import org.cloudfoundry.multiapps.common.Nullable; import org.immutables.value.Value; @@ -32,7 +30,4 @@ public interface ObjectStoreServiceInfo { @Nullable String getHost(); - - @Nullable - Map getCredentials(); } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java index f08255dfab..57fd802b88 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java @@ -1,15 +1,18 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; +import java.net.MalformedURLException; +import java.net.URL; import java.util.List; import java.util.Map; import org.cloudfoundry.multiapps.controller.web.Constants; +import org.cloudfoundry.multiapps.controller.web.Messages; public class ObjectStoreServiceInfoCreator { public List getAllProvidersServiceInfo(Map credentials) { return List.of(createServiceInfoForAws(credentials), createServiceInfoForAliCloud(credentials), - createServiceInfoForCcee(credentials), createServiceInfoForAzure(credentials), createServiceInfoForGcp(credentials)); + createServiceInfoForAzure(credentials), createServiceInfoForCcee(credentials)); } private ObjectStoreServiceInfo createServiceInfoForAws(Map credentials) { @@ -42,6 +45,20 @@ private ObjectStoreServiceInfo createServiceInfoForAliCloud(Map .build(); } + private ObjectStoreServiceInfo createServiceInfoForAzure(Map credentials) { + String accountName = (String) credentials.get(Constants.ACCOUNT_NAME); + String sasToken = (String) credentials.get(Constants.SAS_TOKEN); + String containerName = (String) credentials.get(Constants.CONTAINER_NAME); + URL containerUrl = getContainerUriEndpoint(credentials); + return ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.AZUREBLOB) + .identity(accountName) + .credential(sasToken) + .endpoint(containerUrl == null ? null : containerUrl.toString()) + .container(containerName) + .build(); + } + private ObjectStoreServiceInfo createServiceInfoForCcee(Map credentials) { String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID); String containerName = (String) credentials.get(Constants.CONTAINER_NAME_WITH_DASH); @@ -58,18 +75,15 @@ private ObjectStoreServiceInfo createServiceInfoForCcee(Map cred .build(); } - private ObjectStoreServiceInfo createServiceInfoForAzure(Map credentials) { - return createObjectStoreServiceInfo(Constants.AZUREBLOB, credentials); - } - - private ObjectStoreServiceInfo createServiceInfoForGcp(Map credentials) { - return createObjectStoreServiceInfo(Constants.GOOGLE_CLOUD_STORAGE, credentials); - } - - private ObjectStoreServiceInfo createObjectStoreServiceInfo(String provider, Map credentials) { - return ImmutableObjectStoreServiceInfo.builder() - .provider(provider) - .credentials(credentials) - .build(); + private URL getContainerUriEndpoint(Map credentials) { + if (!credentials.containsKey(Constants.CONTAINER_URI)) { + return null; + } + try { + URL containerUri = new URL((String) credentials.get(Constants.CONTAINER_URI)); + return new URL(containerUri.getProtocol(), containerUri.getHost(), containerUri.getPort(), ""); + } catch (MalformedURLException e) { + throw new IllegalStateException(Messages.CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE, e); + } } } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java index dc5fc6bded..5b5617b7a6 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java @@ -8,7 +8,6 @@ import io.pivotal.cfenv.core.CfCredentials; import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; -import org.cloudfoundry.multiapps.controller.persistence.services.AzureObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; @@ -28,7 +27,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.Mockito.doThrow; @@ -56,9 +54,6 @@ class ObjectStoreFileStorageFactoryBeanTest { @Mock private GcpObjectStoreFileStorage gcpObjectStoreFileStorage; - @Mock - private AzureObjectStoreFileStorage azureObjectStoreFileStorage; - @BeforeEach void setUp() throws Exception { MockitoAnnotations.openMocks(this) @@ -84,7 +79,7 @@ void testObjectStoreCreationWithValidServiceInstance() { } @Test - void testObjectStoreCreationWhenEnvIsValidForAws() { + void testObjectStoreCreationWhenEnvIsValid() { mockCfService(); when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.AWS); ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); @@ -97,41 +92,6 @@ void testObjectStoreCreationWhenEnvIsValidForAws() { .createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); verify(jCloudsObjectStoreFileStorage, times(1)) .testConnection(); - assertTrue(createdObjectStoreFileStorage instanceof JCloudsObjectStoreFileStorage); - } - - @Test - void testObjectStoreCreationWhenEnvIsValidForAzure() { - mockCfService(); - when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.AZURE); - ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); - - spy.afterPropertiesSet(); - FileStorage createdObjectStoreFileStorage = spy.getObject(); - - assertNotNull(createdObjectStoreFileStorage); - verify(spy, never()) - .createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); - verify(azureObjectStoreFileStorage, times(1)) - .testConnection(); - assertTrue(createdObjectStoreFileStorage instanceof AzureObjectStoreFileStorage); - } - - @Test - void testObjectStoreCreationWhenEnvIsValid() { - mockCfService(); - when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.GCP); - ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); - - spy.afterPropertiesSet(); - FileStorage createdObjectStoreFileStorage = spy.getObject(); - - assertNotNull(createdObjectStoreFileStorage); - verify(spy, never()) - .createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); - verify(gcpObjectStoreFileStorage, times(1)) - .testConnection(); - assertTrue(createdObjectStoreFileStorage instanceof GcpObjectStoreFileStorage); } @Test @@ -167,8 +127,6 @@ void testObjectStoreCreationWithoutValidServiceInstance() { .testConnection(); doThrow(new IllegalStateException("Cannot create object store")).when(gcpObjectStoreFileStorage) .testConnection(); - doThrow(new IllegalStateException("Cannot create object store")).when(azureObjectStoreFileStorage) - .testConnection(); Exception exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); assertEquals(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND, exception.getMessage()); } @@ -202,15 +160,10 @@ protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo } @Override - protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo credentials) { + protected GcpObjectStoreFileStorage createGcpFileStorage() { return ObjectStoreFileStorageFactoryBeanTest.this.gcpObjectStoreFileStorage; } - @Override - protected AzureObjectStoreFileStorage createAzureFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return ObjectStoreFileStorageFactoryBeanTest.this.azureObjectStoreFileStorage; - } - @Override public List getProvidersServiceInfo() { CfService service = environmentServicesFinder.findService("deploy-service-os"); diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java index 346fc8698e..6439250543 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java @@ -1,5 +1,7 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; +import java.net.MalformedURLException; +import java.net.URL; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -24,7 +26,10 @@ class ObjectStoreServiceInfoCreatorTest { private static final String BUCKET_VALUE = "bucket_value"; private static final String REGION_VALUE = "region_value"; private static final String ENDPOINT_VALUE = "endpoint_value"; - private static final Map CREDENTIALS = Map.of("test", "test1"); + private static final String ACCOUNT_NAME_VALUE = "account_name_value"; + private static final String SAS_TOKEN_VALUE = "sas_token_value"; + private static final String CONTAINER_NAME_VALUE = "container_name_value"; + private static final String CONTAINER_URI_VALUE = "https://container.com:8080"; private ObjectStoreServiceInfoCreator objectStoreServiceInfoCreator; @@ -33,11 +38,10 @@ void setUp() { objectStoreServiceInfoCreator = new ObjectStoreServiceInfoCreatorMock(); } - static Stream testDifferentProviders() { + static Stream testDifferentProviders() throws MalformedURLException { return Stream.of(Arguments.of(buildCfService(buildAliCloudCredentials()), buildAliCloudObjectStoreServiceInfo()), Arguments.of(buildCfService(buildAwsCredentials()), buildAwsObjectStoreServiceInfo()), - Arguments.of(buildCfService(buildSdkCredentials()), buildAzureObjectStoreServiceInfo()), - Arguments.of(buildCfService(buildSdkCredentials()), buildGcpObjectStoreServiceInfo())); + Arguments.of(buildCfService(buildAzureCredentials()), buildAzureObjectStoreServiceInfo())); } @ParameterizedTest @@ -95,21 +99,22 @@ private static ObjectStoreServiceInfo buildAwsObjectStoreServiceInfo() { .build(); } - private static Map buildSdkCredentials() { - return CREDENTIALS; + private static Map buildAzureCredentials() { + Map credentials = new HashMap<>(); + credentials.put(Constants.ACCOUNT_NAME, ACCOUNT_NAME_VALUE); + credentials.put(Constants.SAS_TOKEN, SAS_TOKEN_VALUE); + credentials.put(Constants.CONTAINER_NAME, CONTAINER_NAME_VALUE); + credentials.put(Constants.CONTAINER_URI, CONTAINER_URI_VALUE); + return credentials; } - private static ObjectStoreServiceInfo buildAzureObjectStoreServiceInfo() { + private static ObjectStoreServiceInfo buildAzureObjectStoreServiceInfo() throws MalformedURLException { return ImmutableObjectStoreServiceInfo.builder() .provider(Constants.AZUREBLOB) - .credentials(CREDENTIALS) - .build(); - } - - private static ObjectStoreServiceInfo buildGcpObjectStoreServiceInfo() { - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.GOOGLE_CLOUD_STORAGE) - .credentials(CREDENTIALS) + .identity(ACCOUNT_NAME_VALUE) + .credential(SAS_TOKEN_VALUE) + .endpoint(new URL("https", "container.com", 8080, "").toString()) + .container(CONTAINER_NAME_VALUE) .build(); } diff --git a/pom.xml b/pom.xml index 676c05301a..0c6a3e2d47 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 org.cloudfoundry.multiapps multiapps-controller-parent - 2.45.0-SNAPSHOT + 2.44.0-SNAPSHOT pom MultiApps Controller Parent Multi-Target Application (MTA) deployment service for Cloud Foundry @@ -15,56 +15,52 @@ cloudfoundry ../multiapps-controller-coverage/target/site/jacoco-aggregate/jacoco.xml - 6.0.3 - 5.23.0 + 6.0.2 + 5.21.0 2.4.240 6.1.0 - 2.25.4 + 2.25.3 2.0.17 2.21.0 4.5.0 - 2.6 + 2.5 5.6 - 5.4.2 + 5.4 4.0.9 7.2.0 - 6.2.17 - 6.5.9 + 6.2.16 + 6.5.7 7.0.2 2.21 - 3.1.1 - 2.21.2 + 3.0.4 4.33.0 5.1.0 5.16.0.RELEASE - 1.2.16 + 1.2.15 1.6.16 2.7.0 33.5.0-jre - 2.4.0 + 2.3.0 2.12.1 - 1.16.4 + 1.16.3 3.18.5 1.1.1 1.21.0 - 2.48.0 - 4.0.0 + 2.45.0 + 3.5.1 4.0.5 - 4.0.7 + 4.0.6 2.0.1 3.0.0 2.1.10 1.28.0 - 3.3.0 - 1.3.6 - 2.64.1 - 0.128.14 - 12.33.3 - 1.13.3 + 3.2.3 + 1.3.5 + 2.62.1 + 0.128.11 4.0.1 - 6.2.0 multiapps-controller-client @@ -307,16 +303,6 @@ ${google-cloud-nio.version} test - - com.azure - azure-storage-blob - ${azure-storage-blob.version} - - - com.azure - azure-core-http-okhttp - ${azure-core-http-okhttp.version} - org.immutables @@ -494,7 +480,7 @@ commons-collections4 ${commons-collections4.version} - + org.apache.httpcomponents.core5 httpcore5 @@ -710,18 +696,6 @@ jackson-annotations ${jackson-annotations.version} - - - com.fasterxml.jackson.core - jackson-core - ${fasterxml.jackson.version} - - - - com.fasterxml.jackson.core - jackson-databind - ${fasterxml.jackson.version} - tools.jackson.core @@ -815,13 +789,6 @@ resilience4j-ratelimiter ${resilience4j.version} - - - - com.squareup.wire - wire-runtime - ${wire-runtime.version} - org.bouncycastle