diff --git a/README.md b/README.md index a54f4c0e..e60a53a2 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ You can easily run an instance on your own with the full flexibility and power a * It provides several storage adapters * currently S3 and AzureBlob * It provides several database adapters for the data - * currently Dynamodb (default), Elasticsearch + * currently Dynamodb (default), Elasticsearch, CosmosDb * It provides a REST-API for custom integrations and further automation Tapir is build on [Quarkus](https://quarkus.io/) and [ReactJS](https://reactjs.org/). You can run Tapir wherever you can run Docker images. @@ -66,18 +66,20 @@ There are samples with Terraform in `examples/`. You can configure Tapir passing the following environment variables: -| Variable | Description | Required | Default | -|---------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------|--------------| -| BACKEND_CONFIG | The database to make use of | X | dynamodb | -| BACKEND_ELASTICSEARCH_HOST | Host of the Elasticsearch instance | Yes, if BACKEND_CONFIG is elasticsearch | | -| STORAGE_CONFIG | The blob storage to make use of | X | s3 | -| STORAGE_ACCESS_SESSION_DURATION | Amount of minutes the signed download url is valid | X | 5 | -| AZURE_BLOB_CONNECTION_STRING | [Connection string](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string) to use for authentication | Yes, if STORAGE_CONFIG is azureBlob | | -| AZURE_BLOB_CONTAINER_NAME | Blob container name to be used to store module archives | Yes, if STORAGE_CONFIG is azureBlob | tf-registry | -| S3_STORAGE_BUCKET_NAME | S3 bucket name to be used to store module archives | Yes, if STORAGE_CONFIG is s3 | tf-registry | -| S3_STORAGE_BUCKET_REGION | AWS region of the target S3 bucket | Yes, if STORAGE_CONFIG is s3 | eu-central-1 | -| API_MAX_BODY_SIZE | The maximum payload size for module/providers to be uploaded | X | 100M | -| REGISTRY_GPG_KEYS_0__ID | GPG key ID of the key to be used (eg. D17C807B4156558133A1FB843C7461473EB779BD) | X | | +| Variable | Description | Required | Default | +|----------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------|--------------| +| BACKEND_CONFIG | The database to make use of | X | dynamodb | +| BACKEND_ELASTICSEARCH_HOST | Host of the Elasticsearch instance | Yes, if BACKEND_CONFIG is elasticsearch | | +| BACKEND_AZURE_MASTER_KEY | Master key of your CosmosDb | Yes, if BACKEND_CONFIG is cosmosdb | | +| BACKEND_AZURE_ENDPOINT | Endpoint of your CosmosDb | Yes, if BACKEND_CONFIG is cosmosdb | | +| STORAGE_CONFIG | The blob storage to make use of | X | s3 | +| STORAGE_ACCESS_SESSION_DURATION | Amount of minutes the signed download url is valid | X | 5 | +| AZURE_BLOB_CONNECTION_STRING | [Connection string](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string) to use for authentication | Yes, if STORAGE_CONFIG is azureBlob | | +| AZURE_BLOB_CONTAINER_NAME | Blob container name to be used to store module archives | Yes, if STORAGE_CONFIG is azureBlob | tf-registry | +| S3_STORAGE_BUCKET_NAME | S3 bucket name to be used to store module archives | Yes, if STORAGE_CONFIG is s3 | tf-registry | +| S3_STORAGE_BUCKET_REGION | AWS region of the target S3 bucket | Yes, if STORAGE_CONFIG is s3 | eu-central-1 | +| API_MAX_BODY_SIZE | The maximum payload size for module/providers to be uploaded | X | 100M | +| REGISTRY_GPG_KEYS_0__ID | GPG key ID of the key to be used (eg. D17C807B4156558133A1FB843C7461473EB779BD) | X | | | REGISTRY_GPG_KEYS_0__ASCII_ARMOR | Ascii armored and bas64 encoded GPG public key (only RSA/DSA supported) | X | | :information_source: A note on the GPG configuration. Quarkus (and therefore Tapir) is based on [Smallrye microprofile](https://smallrye.io/smallrye-config/2.9.1/config/indexed-properties/) and supports indexed properties. Hence, you can add one or more key specifying indexed properties. See example below for passing two GPG keys (**Mind the two subsequent underscores after the index**): diff --git a/checkstyle-suppressions.xml b/checkstyle-suppressions.xml index fc4be272..e85cc309 100644 --- a/checkstyle-suppressions.xml +++ b/checkstyle-suppressions.xml @@ -16,6 +16,10 @@ files="Module.java"/> + + 4.0.0 de.euhus tapir - 0.0.1 + 0.2.0 3.11.0 17 @@ -98,6 +98,10 @@ com.azure azure-identity + + com.azure + azure-cosmos + io.quarkus quarkus-junit5 diff --git a/src/main/java/api/dto/PaginationDto.java b/src/main/java/api/dto/PaginationDto.java index 4f2b7ba0..54cbf6f0 100644 --- a/src/main/java/api/dto/PaginationDto.java +++ b/src/main/java/api/dto/PaginationDto.java @@ -17,6 +17,11 @@ public PaginationDto(List entities) { } } + public PaginationDto(List entities, String lastEvaluatedItemId) { + this.entities = entities; + this.lastEvaluatedItemId = lastEvaluatedItemId; + } + Collection entities; String lastEvaluatedItemId; @@ -28,11 +33,11 @@ public void setEntities(Collection entities) { this.entities = entities; } - public String getLastEvaluatedItem() { + public String getLastEvaluatedItemId() { return lastEvaluatedItemId; } - public void setLastEvaluatedItem(String lastEvaluatedItemId) { + public void setLastEvaluatedItemId(String lastEvaluatedItemId) { this.lastEvaluatedItemId = lastEvaluatedItemId; } } diff --git a/src/main/java/core/backend/ISearchService.java b/src/main/java/core/backend/ISearchService.java index e6b4e36c..4e3ca961 100644 --- a/src/main/java/core/backend/ISearchService.java +++ b/src/main/java/core/backend/ISearchService.java @@ -24,7 +24,7 @@ public interface ISearchService { void ingestSecurityScanResult(Report report) throws Exception; - Module increaseDownloadCounter(Module module) throws IOException; + Module increaseDownloadCounter(Module module) throws Exception; Report getReportByModuleVersion(Module module) throws IOException, ReportNotFoundException; diff --git a/src/main/java/core/backend/azure/cosmosdb/CosmosDbRepository.java b/src/main/java/core/backend/azure/cosmosdb/CosmosDbRepository.java new file mode 100644 index 00000000..40f2b381 --- /dev/null +++ b/src/main/java/core/backend/azure/cosmosdb/CosmosDbRepository.java @@ -0,0 +1,219 @@ +package core.backend.azure.cosmosdb; + +import api.dto.PaginationDto; +import com.azure.cosmos.ConsistencyLevel; +import com.azure.cosmos.CosmosClient; +import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.CosmosContainer; +import com.azure.cosmos.CosmosDatabase; +import com.azure.cosmos.implementation.NotFoundException; +import com.azure.cosmos.models.CosmosContainerProperties; +import com.azure.cosmos.models.CosmosItemRequestOptions; +import com.azure.cosmos.models.CosmosQueryRequestOptions; +import com.azure.cosmos.models.FeedResponse; +import com.azure.cosmos.models.PartitionKey; +import com.azure.cosmos.models.SqlParameter; +import com.azure.cosmos.models.SqlQuerySpec; +import core.backend.SearchService; +import core.exceptions.ModuleNotFoundException; +import core.exceptions.ProviderNotFoundException; +import core.exceptions.ReportNotFoundException; +import core.terraform.Module; +import core.terraform.Provider; +import extensions.core.Report; +import io.quarkus.arc.lookup.LookupIfProperty; +import java.util.Collections; +import java.util.List; +import javax.annotation.PostConstruct; +import javax.enterprise.context.ApplicationScoped; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +@LookupIfProperty(name = "registry.search.backend", stringValue = "cosmosdb") +@ApplicationScoped +public class CosmosDbRepository extends SearchService { + + CosmosClient client; + CosmosDatabase database; + CosmosContainer modulesContainer; + CosmosContainer providerContainer; + CosmosContainer reportsContainer; + @ConfigProperty(name = "registry.search.azure.endpoint") + String endpoint; + @ConfigProperty(name = "registry.search.azure.master-key") + String masterKey; + + @PostConstruct + public void initialize() { + this.client = new CosmosClientBuilder() + .endpoint(endpoint) + .key(masterKey) + .consistencyLevel(ConsistencyLevel.EVENTUAL) + .contentResponseOnWriteEnabled(true) + .buildClient(); + this.database = client.getDatabase("tapir"); + this.modulesContainer = database.getContainer("Modules"); + this.providerContainer = database.getContainer("Providers"); + this.reportsContainer = database.getContainer("Reports"); + } + + @Override + public void bootstrap() throws Exception { + client.createDatabaseIfNotExists("tapir"); + createContainerIfNotExists("Modules"); + createContainerIfNotExists("Providers"); + createContainerIfNotExists("Reports"); + } + + private void createContainerIfNotExists(String name) { + CosmosContainerProperties containerProperties = new CosmosContainerProperties(name, "/id"); + database.createContainerIfNotExists(containerProperties); + } + + @Override + public PaginationDto findModules(String identifier, Integer limit, String term) { + List paramList = List.of( + new SqlParameter("@namespace", "%" + term + "%"), + new SqlParameter("@name", "%" + term + "%"), + new SqlParameter("@provider", "%" + term + "%") + ); + SqlQuerySpec querySpec = new SqlQuerySpec( + "SELECT * FROM Modules m WHERE m.namespace " + + "LIKE @namespace OR m.name LIKE @name OR m.provider LIKE @provider", + paramList); + String continuationToken = identifier.isEmpty() ? null : identifier; + FeedResponse feedResponse = modulesContainer + .queryItems( + querySpec, + new CosmosQueryRequestOptions(), + Module.class) + .streamByPage(continuationToken, limit).findFirst().orElse(null); + if (feedResponse == null) { + return new PaginationDto(Collections.EMPTY_LIST); + } + return new PaginationDto(feedResponse.getResults(), feedResponse.getContinuationToken()); + } + + @Override + public PaginationDto findProviders(String identifier, Integer limit, String term) { + List paramList = List.of( + new SqlParameter("@namespace", "%" + term + "%"), + new SqlParameter("@type", "%" + term + "%") + ); + SqlQuerySpec querySpec = new SqlQuerySpec( + "SELECT * FROM Providers p WHERE p.namespace LIKE @namespace OR p.type LIKE @type", + paramList); + String continuationToken = identifier.isEmpty() ? null : identifier; + FeedResponse feedResponse = providerContainer + .queryItems( + querySpec, + new CosmosQueryRequestOptions(), + Provider.class) + .streamByPage(continuationToken, limit).findFirst().orElse(null); + if (feedResponse == null) { + return new PaginationDto(Collections.EMPTY_LIST); + } + return new PaginationDto(feedResponse.getResults(), feedResponse.getContinuationToken()); + } + + @Override + public Module getModuleById(String id) throws ModuleNotFoundException { + try { + return modulesContainer.readItem(id, new PartitionKey(id), Module.class).getItem(); + } catch (NotFoundException cosmosException) { + throw new ModuleNotFoundException(id, cosmosException); + } + } + + @Override + public Module getModuleVersions(Module module) throws ModuleNotFoundException { + return getModuleById(module.getId()); + } + + @Override + public void ingestModuleData(Module module) { + Module moduleToIngest; + try { + Module existingModule = modulesContainer.readItem( + module.getId(), + new PartitionKey(module.getId()), + Module.class + ).getItem(); + existingModule.getVersions().add(module.getVersions().first()); + existingModule.setPublished_at(module.getPublished_at()); + moduleToIngest = existingModule; + } catch (NotFoundException cosmosException) { + moduleToIngest = module; + } + modulesContainer.upsertItem( + moduleToIngest, + new PartitionKey(module.getId()), + new CosmosItemRequestOptions() + ); + } + + @Override + public void ingestProviderData(Provider provider) { + Provider providerToIngest; + try { + Provider existingProvider = providerContainer.readItem( + provider.getId(), + new PartitionKey(provider.getId()), + Provider.class + ).getItem(); + existingProvider + .getVersions() + .put( + provider.getVersions().firstEntry().getKey(), + provider.getVersions().firstEntry().getValue() + ); + providerToIngest = existingProvider; + } catch (NotFoundException cosmosException) { + providerToIngest = provider; + } + providerContainer.createItem( + providerToIngest, + new PartitionKey(provider.getId()), + new CosmosItemRequestOptions() + ); + } + + @Override + public void ingestSecurityScanResult(Report report) { + reportsContainer.upsertItem( + report, + new PartitionKey(report.getId()), + new CosmosItemRequestOptions() + ); + } + + @Override + public Module increaseDownloadCounter(Module module) throws ModuleNotFoundException { + Module existingModule = getModuleById(module.getId()); + Integer downloads = existingModule.getDownloads(); + existingModule.setDownloads(downloads + 1); + modulesContainer.upsertItem(existingModule); + return existingModule; + } + + @Override + public Report getReportByModuleVersion(Module module) throws ReportNotFoundException { + String reportId = module.getId() + + "-" + + module.getCurrentVersion(); + try { + return reportsContainer.readItem(reportId, new PartitionKey(reportId), Report.class) + .getItem(); + } catch (NotFoundException cosmosException) { + throw new ReportNotFoundException(reportId, cosmosException); + } + } + + @Override + public Provider getProviderById(String id) throws ProviderNotFoundException { + try { + return providerContainer.readItem(id, new PartitionKey(id), Provider.class).getItem(); + } catch (NotFoundException cosmosException) { + throw new ProviderNotFoundException(id, cosmosException); + } + } +} diff --git a/src/main/java/core/vertx/event/consumer/DownloadListener.java b/src/main/java/core/vertx/event/consumer/DownloadListener.java index 61cab3af..724e3252 100644 --- a/src/main/java/core/vertx/event/consumer/DownloadListener.java +++ b/src/main/java/core/vertx/event/consumer/DownloadListener.java @@ -3,7 +3,6 @@ import core.backend.SearchService; import core.terraform.Module; import io.quarkus.vertx.ConsumeEvent; -import java.io.IOException; import java.util.logging.Logger; import javax.enterprise.context.ApplicationScoped; import javax.enterprise.inject.Instance; @@ -20,7 +19,7 @@ public DownloadListener(Instance searchServiceInstance) { } @ConsumeEvent("module.download.requested") - public Module handleDownloadRequestedEvent(Module module) throws IOException { + public Module handleDownloadRequestedEvent(Module module) throws Exception { LOGGER.info(String.format("Download was requested for module %s, version %s", module.getName(), module.getCurrentVersion()) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index e6a68882..a1c6ae64 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -7,6 +7,9 @@ registry: backend: ${BACKEND_CONFIG:dynamodb} elasticsearch: host: ${BACKEND_ELASTICSEARCH_HOST:localhost:9200} + azure: + master-key: ${BACKEND_AZURE_MASTER_KEY:C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==} + endpoint: ${BACKEND_AZURE_ENDPOINT:https://localhost:8081} storage: backend: ${STORAGE_CONFIG:s3} access: diff --git a/src/main/webui/src/components/layout/Footer.tsx b/src/main/webui/src/components/layout/Footer.tsx index 2c0816f7..2c17a6cd 100644 --- a/src/main/webui/src/components/layout/Footer.tsx +++ b/src/main/webui/src/components/layout/Footer.tsx @@ -37,7 +37,7 @@ const Footer = () => { > - Terraform Private Registry v0.0.1 + Terraform Private Registry v0.2.0 diff --git a/src/main/webui/src/components/layout/__snapshots__/Footer.test.tsx.snap b/src/main/webui/src/components/layout/__snapshots__/Footer.test.tsx.snap index 1ae737a7..1d81566e 100644 --- a/src/main/webui/src/components/layout/__snapshots__/Footer.test.tsx.snap +++ b/src/main/webui/src/components/layout/__snapshots__/Footer.test.tsx.snap @@ -17,7 +17,7 @@ Object {

- Terraform Private Registry v0.0.1 + Terraform Private Registry v0.2.0

- Terraform Private Registry v0.0.1 + Terraform Private Registry v0.2.0

{ const [loading, setLoading] = useState(false); const [distanceBottom, setDistanceBottom] = useState(0); - const hasMoreData = (lastEvaluatedItem: any) => { - return !!lastEvaluatedItem && modules.at(0)?.id !== lastEvaluatedItem; + const hasMoreData = (lastEvaluatedItemId: string) => { + return !!lastEvaluatedItemId && modules.at(0)?.id !== lastEvaluatedItemId; }; const loadMore = useCallback( () => { setLoading(true); fetchModules( - `search/modules?limit=${fetchDataLimit}&lastKey=${lastEvaluatedItemKey}&q=${searchString}` + `search/modules?limit=${fetchDataLimit}&lastKey=${encodeURIComponent( + lastEvaluatedItemKey + )}&q=${searchString}` ).then((data) => { const allModules = [...modules, ...data.entities]; setLastEvaluatedItemKey( diff --git a/src/test/java/core/vertx/event/consumer/DownloadListenerTest.java b/src/test/java/core/vertx/event/consumer/DownloadListenerTest.java index 7f0c2fa4..9a9ce541 100644 --- a/src/test/java/core/vertx/event/consumer/DownloadListenerTest.java +++ b/src/test/java/core/vertx/event/consumer/DownloadListenerTest.java @@ -5,7 +5,6 @@ import core.backend.aws.dynamodb.repository.DynamodbRepository; import core.terraform.Module; import io.quarkus.test.junit.QuarkusTest; -import java.io.IOException; import javax.inject.Inject; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -41,7 +40,7 @@ void tearDown() { } @Test - void handleDownloadRequestedEvent() throws IOException { + void handleDownloadRequestedEvent() throws Exception { repository.ingestModuleData(fakeModule); assertEquals(fakeModule.getDownloads(), 0); fakeModule = dl.handleDownloadRequestedEvent(fakeModule);