Skip to content

Commit

Permalink
Split db and pvc tests (#35)
Browse files Browse the repository at this point in the history
  • Loading branch information
@adolfo-ab
adolfo-ab authored Oct 8, 2024
1 parent 470334e commit 03cbed7
Show file tree
Hide file tree
Showing 3 changed files with 364 additions and 80 deletions.
57 changes: 31 additions & 26 deletions trustyai_tests/tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,38 +203,43 @@ def user_workload_monitoring_config(client: DynamicClient) -> ConfigMap:


@pytest.fixture(scope="class")
def trustyai_service(
request,
def trustyai_service_pvc(
client: DynamicClient,
model_namespace: Namespace,
modelmesh_serviceaccount: Any,
cluster_monitoring_config: ConfigMap,
user_workload_monitoring_config: ConfigMap,
) -> TrustyAIService:
storage_type = request.param["storage_type"]
metrics = {"schedule": "5s"}
if storage_type == "pvc":
with TrustyAIService(
client=client,
name=TRUSTYAI_SERVICE,
namespace=model_namespace.name,
storage={"format": "PVC", "folder": "/inputs", "size": "1Gi"},
data={"filename": "data.csv", "format": "CSV"},
metrics=metrics,
) as trusty:
wait_for_trustyai_pod_running(namespace=model_namespace)
yield trusty
else: # running with db
request.getfixturevalue("mariadb")
with TrustyAIService(
client=client,
name=TRUSTYAI_SERVICE,
namespace=model_namespace.name,
storage={"format": "DATABASE", "databaseConfigurations": "db-credentials"},
metrics=metrics,
) as trusty:
wait_for_trustyai_pod_running(namespace=model_namespace)
yield trusty
with TrustyAIService(
client=client,
name=TRUSTYAI_SERVICE,
namespace=model_namespace.name,
storage={"format": "PVC", "folder": "/inputs", "size": "1Gi"},
data={"filename": "data.csv", "format": "CSV"},
metrics={"schedule": "5s"},
) as trusty:
wait_for_trustyai_pod_running(namespace=model_namespace)
yield trusty


@pytest.fixture(scope="class")
def trustyai_service_db(
client: DynamicClient,
model_namespace: Namespace,
mariadb,
modelmesh_serviceaccount: Any,
cluster_monitoring_config: ConfigMap,
user_workload_monitoring_config: ConfigMap,
) -> TrustyAIService:
with TrustyAIService(
client=client,
name=TRUSTYAI_SERVICE,
namespace=model_namespace.name,
storage={"format": "DATABASE", "databaseConfigurations": "db-credentials"},
metrics={"schedule": "5s"},
) as trusty:
wait_for_trustyai_pod_running(namespace=model_namespace)
yield trusty


@pytest.fixture(scope="class")
Expand Down
217 changes: 184 additions & 33 deletions trustyai_tests/tests/drift/test_drift.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,166 @@
)


@pytest.mark.parametrize(
"trustyai_service",
[pytest.param({"storage_type": "pvc"}, id="pvc"), pytest.param({"storage_type": "db"}, id="db")],
indirect=True,
)
@pytest.mark.openshift
class TestDriftMetrics:
class TestDriftMetricsPVC:
"""
Verifies the different input data drift metrics available in TrustyAI, using PVC storage.
Drift metrics: Meanshift, FourierMMD, KSTest, and ApproxKSTest.
1. Send data to the model (gaussian_credit_model).
2. Upload training data for TrustyAI (used as baseline to calculate the drift metrics).
3. For each metric:
3.1. Send a basic request and verify the response.
3.2. Send a schedule request and verify the response.
3.3. Verify that the metric has reached Prometheus.
"""

def test_gaussian_credit_model_metadata_pvc(
self, model_namespace: Namespace, trustyai_service_pvc: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
wait_for_modelmesh_pods_registered(namespace=model_namespace)

path = f"{MODEL_DATA_PATH}/{gaussian_credit_model.name}"

send_data_to_inference_service(
inference_service=gaussian_credit_model,
namespace=model_namespace,
data_path=f"{path}/data_batches",
)

response = upload_data_to_trustyai_service(
namespace=model_namespace,
data_path=f"{path}/training_data.json",
)
assert response.status_code == http.HTTPStatus.OK

verify_trustyai_model_metadata(
namespace=model_namespace,
model=gaussian_credit_model,
data_path=path,
)

def test_request_meanshift_pvc(
self, model_namespace: Namespace, trustyai_service_pvc: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_request(
namespace=model_namespace,
endpoint=get_metric_endpoint(metric=Metric.MEANSHIFT),
expected_metric_name=Metric.MEANSHIFT.value.upper(),
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_schedule_meanshift_pvc(
self, model_namespace: Namespace, trustyai_service_pvc: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_scheduling(
namespace=model_namespace,
endpoint=get_metric_endpoint(metric=Metric.MEANSHIFT, schedule=True),
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_meanshift_prometheus_query_pvc(
self, model_namespace: Namespace, trustyai_service_pvc: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_trustyai_metric_prometheus(
namespace=model_namespace,
model=gaussian_credit_model,
prometheus_query=f'trustyai_{Metric.MEANSHIFT.value}{{namespace="{model_namespace.name}"}}',
metric_name=Metric.MEANSHIFT.value,
)

def test_request_fouriermmd_pvc(
self, model_namespace: Namespace, trustyai_service_pvc: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_request(
namespace=model_namespace,
endpoint=get_metric_endpoint(metric=Metric.FOURIERMMD),
expected_metric_name=Metric.FOURIERMMD.value.upper(),
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_schedule_fouriermmd_pvc(
self, model_namespace: Namespace, trustyai_service_pvc: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_scheduling(
namespace=model_namespace,
endpoint=get_metric_endpoint(metric=Metric.FOURIERMMD, schedule=True),
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_fouriermmd_prometheus_query_pvc(
self, model_namespace: Namespace, trustyai_service_pvc: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_trustyai_metric_prometheus(
namespace=model_namespace,
model=gaussian_credit_model,
prometheus_query=f'trustyai_{Metric.FOURIERMMD.value}{{namespace="{model_namespace.name}"}}',
metric_name=Metric.FOURIERMMD.value,
)

def test_request_kstest_pvc(
self, model_namespace: Namespace, trustyai_service_pvc: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_request(
namespace=model_namespace,
endpoint=get_metric_endpoint(metric=Metric.KSTEST),
expected_metric_name=Metric.KSTEST.value.upper(),
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_schedule_kstest_scheduling_request_pvc(
self, model_namespace: Namespace, trustyai_service_pvc: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_scheduling(
namespace=model_namespace,
endpoint=get_metric_endpoint(metric=Metric.KSTEST, schedule=True),
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_kstest_prometheus_query_pvc(
self, model_namespace: Namespace, trustyai_service_pvc: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_trustyai_metric_prometheus(
namespace=model_namespace,
model=gaussian_credit_model,
prometheus_query=f'trustyai_{Metric.KSTEST.value}{{namespace="{model_namespace.name}"}}',
metric_name=Metric.KSTEST.value,
)

def test_request_approxkstest_pvc(
self, model_namespace: Namespace, trustyai_service_pvc: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_request(
namespace=model_namespace,
endpoint=get_metric_endpoint(metric=Metric.APPROXKSTEST),
expected_metric_name=Metric.APPROXKSTEST.value.upper(),
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_schedule_approxkstest_pvc(
self, model_namespace: Namespace, trustyai_service_pvc: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_scheduling(
namespace=model_namespace,
endpoint=get_metric_endpoint(metric=Metric.APPROXKSTEST, schedule=True),
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_approxkstest_prometheus_query_pvc(
self, model_namespace: Namespace, trustyai_service_pvc: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_trustyai_metric_prometheus(
namespace=model_namespace,
model=gaussian_credit_model,
prometheus_query=f"trustyai_{Metric.APPROXKSTEST.value}" f'{{namespace="{model_namespace.name}"}}',
metric_name=Metric.APPROXKSTEST.value,
)


@pytest.mark.openshift
class TestDriftMetricsDB:
"""
Verifies the different input data drift metrics available in TrustyAI.
Verifies the different input data drift metrics available in TrustyAI, using DB storage.
Drift metrics: Meanshift, FourierMMD, KSTest, and ApproxKSTest.
1. Send data to the model (gaussian_credit_model).
Expand All @@ -39,8 +190,8 @@ class TestDriftMetrics:
3.3. Verify that the metric has reached Prometheus.
"""

def test_gaussian_credit_model_metadata(
self, model_namespace: Namespace, trustyai_service: TrustyAIService, gaussian_credit_model: InferenceService
def test_gaussian_credit_model_metadata_db(
self, model_namespace: Namespace, trustyai_service_db: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
wait_for_modelmesh_pods_registered(namespace=model_namespace)

Expand All @@ -64,8 +215,8 @@ def test_gaussian_credit_model_metadata(
data_path=path,
)

def test_request_meanshift(
self, model_namespace: Namespace, trustyai_service: TrustyAIService, gaussian_credit_model: InferenceService
def test_request_meanshift_db(
self, model_namespace: Namespace, trustyai_service_db: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_request(
namespace=model_namespace,
Expand All @@ -74,17 +225,17 @@ def test_request_meanshift(
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_schedule_meanshift(
self, model_namespace: Namespace, trustyai_service: TrustyAIService, gaussian_credit_model: InferenceService
def test_schedule_meanshift_db(
self, model_namespace: Namespace, trustyai_service_db: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_scheduling(
namespace=model_namespace,
endpoint=get_metric_endpoint(metric=Metric.MEANSHIFT, schedule=True),
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_meanshift_prometheus_query(
self, model_namespace: Namespace, trustyai_service: TrustyAIService, gaussian_credit_model: InferenceService
def test_meanshift_prometheus_query_db(
self, model_namespace: Namespace, trustyai_service_db: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_trustyai_metric_prometheus(
namespace=model_namespace,
Expand All @@ -93,8 +244,8 @@ def test_meanshift_prometheus_query(
metric_name=Metric.MEANSHIFT.value,
)

def test_request_fouriermmd(
self, model_namespace: Namespace, trustyai_service: TrustyAIService, gaussian_credit_model: InferenceService
def test_request_fouriermmd_db(
self, model_namespace: Namespace, trustyai_service_db: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_request(
namespace=model_namespace,
Expand All @@ -103,17 +254,17 @@ def test_request_fouriermmd(
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_schedule_fouriermmd(
self, model_namespace: Namespace, trustyai_service: TrustyAIService, gaussian_credit_model: InferenceService
def test_schedule_fouriermmd_db(
self, model_namespace: Namespace, trustyai_service_db: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_scheduling(
namespace=model_namespace,
endpoint=get_metric_endpoint(metric=Metric.FOURIERMMD, schedule=True),
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_fouriermmd_prometheus_query(
self, model_namespace: Namespace, trustyai_service: TrustyAIService, gaussian_credit_model: InferenceService
def test_fouriermmd_prometheus_query_db(
self, model_namespace: Namespace, trustyai_service_db: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_trustyai_metric_prometheus(
namespace=model_namespace,
Expand All @@ -122,8 +273,8 @@ def test_fouriermmd_prometheus_query(
metric_name=Metric.FOURIERMMD.value,
)

def test_request_kstest(
self, model_namespace: Namespace, trustyai_service: TrustyAIService, gaussian_credit_model: InferenceService
def test_request_kstest_db(
self, model_namespace: Namespace, trustyai_service_db: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_request(
namespace=model_namespace,
Expand All @@ -132,17 +283,17 @@ def test_request_kstest(
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_schedule_kstest_scheduling_request(
self, model_namespace: Namespace, trustyai_service: TrustyAIService, gaussian_credit_model: InferenceService
def test_schedule_kstest_scheduling_request_db(
self, model_namespace: Namespace, trustyai_service_db: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_scheduling(
namespace=model_namespace,
endpoint=get_metric_endpoint(metric=Metric.KSTEST, schedule=True),
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_kstest_prometheus_query(
self, model_namespace: Namespace, trustyai_service: TrustyAIService, gaussian_credit_model: InferenceService
def test_kstest_prometheus_query_db(
self, model_namespace: Namespace, trustyai_service_db: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_trustyai_metric_prometheus(
namespace=model_namespace,
Expand All @@ -151,8 +302,8 @@ def test_kstest_prometheus_query(
metric_name=Metric.KSTEST.value,
)

def test_request_approxkstest(
self, model_namespace: Namespace, trustyai_service: TrustyAIService, gaussian_credit_model: InferenceService
def test_request_approxkstest_db(
self, model_namespace: Namespace, trustyai_service_db: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_request(
namespace=model_namespace,
Expand All @@ -161,17 +312,17 @@ def test_request_approxkstest(
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_schedule_approxkstest(
self, model_namespace: Namespace, trustyai_service: TrustyAIService, gaussian_credit_model: InferenceService
def test_schedule_approxkstest_db(
self, model_namespace: Namespace, trustyai_service_db: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_metric_scheduling(
namespace=model_namespace,
endpoint=get_metric_endpoint(metric=Metric.APPROXKSTEST, schedule=True),
json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"},
)

def test_approxkstest_prometheus_query(
self, model_namespace: Namespace, trustyai_service: TrustyAIService, gaussian_credit_model: InferenceService
def test_approxkstest_prometheus_query_db(
self, model_namespace: Namespace, trustyai_service_db: TrustyAIService, gaussian_credit_model: InferenceService
) -> None:
verify_trustyai_metric_prometheus(
namespace=model_namespace,
Expand Down
Loading

0 comments on commit 03cbed7

Please sign in to comment.