Merge pull request #15 from scale8/configurable-databases

Configurable databases
scale8 · Sep 22, 2021 · 11bdc9f · 11bdc9f
2 parents 1341d63 + f4633fb
commit 11bdc9f
Show file tree

Hide file tree

Showing 99 changed files with 2,642 additions and 857 deletions.
diff --git a/api/src/Types.ts b/api/src/Types.ts
@@ -1,6 +1,8 @@
 import { ValidationType } from '../../common/enums/ValidationType';
 import { InputType } from '../../common/enums/InputType';
 import { TypeIcon } from '../../common/enums/TypeIcon';
+import { AwsRegion } from './enums/AwsRegion';
+import { JWTInput } from 'google-auth-library/build/src/auth/credentials';
 
 export interface PlatformEventConfig {
     persistence_id: string;
@@ -47,3 +49,21 @@ export interface PlatformRevisionConfig {
     events?: PlatformEventConfig[];
     data_containers?: PlatformDataContainerConfig[];
 }
+export interface AwsConfig {
+    access_key_id: string;
+    secret_access_key: string;
+    region: AwsRegion;
+    path_prefix: string;
+    bucket_name: string;
+}
+export interface GCBigQueryStreamConfig {
+    service_account_json: JWTInput;
+    data_set_name: string;
+    data_set_location: 'EU' | 'US';
+    require_partition_filter_in_queries: boolean;
+}
+export interface MongoDbPushConfig {
+    use_api_mongo_server: boolean;
+    connection_string: string;
+    database_name: string;
+}
diff --git a/api/src/UpdateUsage.ts b/api/src/UpdateUsage.ts
@@ -15,6 +15,7 @@ import { AccountType } from './enums/AccountType';
 import { getUsageCycle } from './utils/UsageUtils';
 import BaseDatabase from './backends/databases/abstractions/BaseDatabase';
 import BaseLogger from './backends/logging/abstractions/BaseLogger';
+import { StorageProvider } from './enums/StorageProvider';
 
 //register .env as soon as possible...
 dotenv.config();
@@ -23,7 +24,8 @@ dotenv.config();
 class UpdateUsage {
     @inject(TYPES.RepoFromModelFactory) protected readonly repoFactory!: RepoFromModelFactory;
     @inject(TYPES.BackendLogger) protected readonly logger!: BaseLogger;
-    @inject(TYPES.BackendDatabase) protected readonly backendDatabase!: BaseDatabase;
+    @inject(TYPES.BackendDatabaseFactory)
+    protected readonly backendDatabaseFactory!: (storage_provider: StorageProvider) => BaseDatabase;
 
     public async updateTagManagerUsage() {
         const accounts = await this.repoFactory(TagManagerAccount).find({});
@@ -39,14 +41,17 @@ class UpdateUsage {
                     const dayUsage = (
                         await Promise.all(
                             apps.map((app) =>
-                                this.backendDatabase.eventRequests(app, {
-                                    time_slice: 'DAY',
-                                    filter_options: {
-                                        from: startOfDay(usageCycle.start).getTime(),
-                                        to: endOfDay(usageCycle.end).getTime(),
+                                this.backendDatabaseFactory(app.storageProvider).eventRequests(
+                                    app,
+                                    {
+                                        time_slice: 'DAY',
+                                        filter_options: {
+                                            from: startOfDay(usageCycle.start).getTime(),
+                                            to: endOfDay(usageCycle.end).getTime(),
+                                        },
+                                        limit: 1000,
                                     },
-                                    limit: 1000,
-                                }),
+                                ),
                             ),
                         )
                     ).reduce((v: { [k: string]: number }, appDayUsage) => {
@@ -104,14 +109,17 @@ class UpdateUsage {
                     const dayUsage = (
                         await Promise.all(
                             ingestEndpoints.map((ingestEndpoint) =>
-                                this.backendDatabase.usage(ingestEndpoint, {
-                                    time_slice: 'DAY',
-                                    filter_options: {
-                                        from: startOfDay(usageCycle.start).getTime(),
-                                        to: endOfDay(usageCycle.end).getTime(),
+                                this.backendDatabaseFactory(ingestEndpoint.storageProvider).usage(
+                                    ingestEndpoint,
+                                    {
+                                        time_slice: 'DAY',
+                                        filter_options: {
+                                            from: startOfDay(usageCycle.start).getTime(),
+                                            to: endOfDay(usageCycle.end).getTime(),
+                                        },
+                                        limit: 1000,
                                     },
-                                    limit: 1000,
-                                }),
+                                ),
                             ),
                         )
                     ).reduce(

diff --git a/api/src/backends/configuration/abstractions/BaseConfig.ts b/api/src/backends/configuration/abstractions/BaseConfig.ts
@@ -33,6 +33,10 @@ export default abstract class BaseConfig {
         }
     }
 
+    public getStorageBackend(): string {
+        return BaseConfig.getFromEnvVarOrElse('STORAGE_BACKEND', 'mongodb');
+    }
+
     public getMode(): Mode {
         const mode = BaseConfig.getFromEnvVarOrElse('SERVER_MODE', Mode.SELF_HOSTED);
         return mode === Mode.COMMERCIAL ? Mode.COMMERCIAL : Mode.SELF_HOSTED;
@@ -128,10 +132,6 @@ export default abstract class BaseConfig {
         return await this.getConfigEntryThrows('GC_KEY_FILE');
     }
 
-    public async getGCProjectId(): Promise<string> {
-        return await this.getConfigEntryThrows('GC_PROJECT_ID');
-    }
-
     public async getAssetBucket(): Promise<string> {
         return await this.getConfigEntryOrElse(
             'ASSET_BUCKET',