-
Notifications
You must be signed in to change notification settings - Fork 42
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Kafka Ingestion Spec Submission (#189)
- Implemented functionality to submit Kafka ingestion specs - Fixed integration tests - Updated RBAC rules to include Secret verbs for authenticated API usage
- Loading branch information
1 parent
d9e0f6e
commit bd20e44
Showing
6 changed files
with
213 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| apiVersion: druid.apache.org/v1alpha1 | ||
| kind: DruidIngestion | ||
| metadata: | ||
| labels: | ||
| app.kubernetes.io/name: druidingestion | ||
| app.kubernetes.io/instance: druidingestion-sample | ||
| name: kafka-1 | ||
| spec: | ||
| suspend: false | ||
| druidCluster: tiny-cluster | ||
| ingestion: | ||
| type: kafka | ||
| spec: |- | ||
| { | ||
| "type": "kafka", | ||
| "spec": { | ||
| "dataSchema": { | ||
| "dataSource": "metrics-kafka", | ||
| "timestampSpec": { | ||
| "column": "timestamp", | ||
| "format": "auto" | ||
| }, | ||
| "dimensionsSpec": { | ||
| "dimensions": [], | ||
| "dimensionExclusions": [ | ||
| "timestamp", | ||
| "value" | ||
| ] | ||
| }, | ||
| "metricsSpec": [ | ||
| { | ||
| "name": "count", | ||
| "type": "count" | ||
| }, | ||
| { | ||
| "name": "value_sum", | ||
| "fieldName": "value", | ||
| "type": "doubleSum" | ||
| }, | ||
| { | ||
| "name": "value_min", | ||
| "fieldName": "value", | ||
| "type": "doubleMin" | ||
| }, | ||
| { | ||
| "name": "value_max", | ||
| "fieldName": "value", | ||
| "type": "doubleMax" | ||
| } | ||
| ], | ||
| "granularitySpec": { | ||
| "type": "uniform", | ||
| "segmentGranularity": "HOUR", | ||
| "queryGranularity": "NONE" | ||
| } | ||
| }, | ||
| "ioConfig": { | ||
| "topic": "metrics", | ||
| "inputFormat": { | ||
| "type": "json" | ||
| }, | ||
| "consumerProperties": { | ||
| "bootstrap.servers": "localhost:9092" | ||
| }, | ||
| "taskCount": 1, | ||
| "replicas": 1, | ||
| "taskDuration": "PT1H" | ||
| }, | ||
| "tuningConfig": { | ||
| "type": "kafka", | ||
| "maxRowsPerSegment": 5000000 | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,7 @@ | ||
| tenant: | ||
| pools: | ||
| - servers: 1 | ||
| - name: "minio" | ||
| servers: 1 | ||
| volumesPerServer: 1 | ||
| certificate: | ||
| requestAutoCert: false | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,7 +4,7 @@ set -x | |
| # Get Kind | ||
| go install sigs.k8s.io/[email protected] | ||
| # minio statefulset name | ||
| MINIO_STS_NAME=myminio-ss-0 | ||
| MINIO_STS_NAME=myminio-minio | ||
| # druid namespace | ||
| NAMESPACE=druid | ||
| # fmt code | ||
|
|
@@ -23,8 +23,6 @@ make docker-build-local-test | |
| make docker-push-local-test | ||
| # try to install the CRD with make | ||
| make install | ||
| # delete the crd | ||
| make uninstall | ||
| # install druid-operator | ||
| make helm-install-druid-operator | ||
| # install minio-operator and tenant | ||
|
|
@@ -64,6 +62,19 @@ sleep 30 # wait for the manager to submit the ingestion task | |
| taskId=`kubectl get druidingestion -n druid wikipedia-ingestion --template={{.status.taskId}}` | ||
| make deploy-testingestionjob TASK_ID=$taskId | ||
|
|
||
| # Running a test Kafka DruidIngestion resource and wait for the task to be submitted | ||
| kubectl apply -f e2e/configs/kafka-ingestion.yaml -n ${NAMESPACE} | ||
| sleep 30 # wait for the manager to submit the ingestion task | ||
|
|
||
| # Verify the supervisor task has been created | ||
| supervisorTaskId=`kubectl get druidingestion -n druid kafka-1 --template={{.status.taskId}}` | ||
| if [ -z "$supervisorTaskId" ]; then | ||
| echo "Failed to get supervisor task ID" | ||
| exit 1 | ||
| else | ||
| echo "Supervisor task ID: $supervisorTaskId" | ||
| fi | ||
|
|
||
| # Delete old druid | ||
| kubectl delete -f e2e/configs/druid-cr.yaml -n ${NAMESPACE} | ||
| for d in $(kubectl get pods -n ${NAMESPACE} -l app=druid -l druid_cr=tiny-cluster -o name) | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| apiVersion: druid.apache.org/v1alpha1 | ||
| kind: DruidIngestion | ||
| metadata: | ||
| labels: | ||
| app.kubernetes.io/name: druidingestion | ||
| app.kubernetes.io/instance: druidingestion-sample | ||
| name: kafka-1 | ||
| spec: | ||
| suspend: false | ||
| druidCluster: tiny-cluster | ||
| ingestion: | ||
| type: kafka | ||
| spec: |- | ||
| { | ||
| "type": "kafka", | ||
| "spec": { | ||
| "dataSchema": { | ||
| "dataSource": "metrics-kafka", | ||
| "timestampSpec": { | ||
| "column": "timestamp", | ||
| "format": "auto" | ||
| }, | ||
| "dimensionsSpec": { | ||
| "dimensions": [], | ||
| "dimensionExclusions": [ | ||
| "timestamp", | ||
| "value" | ||
| ] | ||
| }, | ||
| "metricsSpec": [ | ||
| { | ||
| "name": "count", | ||
| "type": "count" | ||
| }, | ||
| { | ||
| "name": "value_sum", | ||
| "fieldName": "value", | ||
| "type": "doubleSum" | ||
| }, | ||
| { | ||
| "name": "value_min", | ||
| "fieldName": "value", | ||
| "type": "doubleMin" | ||
| }, | ||
| { | ||
| "name": "value_max", | ||
| "fieldName": "value", | ||
| "type": "doubleMax" | ||
| } | ||
| ], | ||
| "granularitySpec": { | ||
| "type": "uniform", | ||
| "segmentGranularity": "HOUR", | ||
| "queryGranularity": "NONE" | ||
| } | ||
| }, | ||
| "ioConfig": { | ||
| "topic": "metrics", | ||
| "inputFormat": { | ||
| "type": "json" | ||
| }, | ||
| "consumerProperties": { | ||
| "bootstrap.servers": "localhost:9092" | ||
| }, | ||
| "taskCount": 1, | ||
| "replicas": 1, | ||
| "taskDuration": "PT1H" | ||
| }, | ||
| "tuningConfig": { | ||
| "type": "kafka", | ||
| "maxRowsPerSegment": 5000000 | ||
| } | ||
| } | ||
| } |