Include log_level as key in template (#762)

Add possibility for the user to override the the log_level either as a CLI option to gordo-components or as a key under globals runtime in the config file.

.github/workflows/main.yml

@@ -1,6 +1,8 @@
 name: CI
 
-on: [push, pull_request, release]
+on:
+  push:
+  pull_request:
 
 jobs:
   test:
@@ -106,35 +108,3 @@ jobs:
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
         run: make push-prod-images
-
-  publish-to-pypi:
-    if: github.action.event == 'release'
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [3.7]
-    steps:
-      - uses: actions/checkout@v1
-
-      - uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: 'x64'
-
-      - name: Install deps
-        run: |
-          pip install --upgrade pip
-          pip install -r requirements.txt
-
-      - name: Build
-        run: |
-          python setup.py sdist
-          python setup.py bdist_wheel
-
-      - name: Publish
-        env:
-          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-        run: |
-          pip install twine
-          twine upload -u statoil-travis --skip-existing dist/*

.github/workflows/release.yml

@@ -0,0 +1,39 @@
+name: ReleaseCI
+
+on:
+  release:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  publish-to-pypi:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [3.7]
+    steps:
+      - uses: actions/checkout@v1
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: 'x64'
+
+      - name: Install deps
+        run: |
+          pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Build
+        run: |
+          python setup.py sdist
+          python setup.py bdist_wheel
+
+      - name: Publish
+        env:
+          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+        run: |
+          pip install twine
+          twine upload -u statoil-travis --skip-existing dist/*

gordo_components/cli/cli.py

@@ -4,7 +4,6 @@
 CLI interfaces
 """
 
-import os
 import logging
 import sys
 import traceback
@@ -46,19 +45,29 @@
 
 @click.group("gordo-components")
 @click.version_option(version=__version__, message=__version__)
-def gordo():
+@click.option(
+    "--log-level",
+    type=str,
+    default="INFO",
+    help="Run workflow with custom log-level.",
+    envvar="GORDO_LOG_LEVEL",
+)
+@click.pass_context
+def gordo(gordo_ctx: click.Context, **ctx):
     """
     The main entry point for the CLI interface
     """
-
     # Set log level, defaulting to INFO
-    log_level = os.getenv("LOG_LEVEL", "INFO").upper()
-
     logging.basicConfig(
-        level=getattr(logging, log_level),
+        level=getattr(logging, str(gordo_ctx.params.get("log_level")).upper()),
         format="[%(asctime)s] %(levelname)s [%(name)s.%(funcName)s:%(lineno)d] %(message)s",
     )
 
+    # Set matplotlib log level to INFO to avoid noise
+    logging.getLogger("matplotlib").setLevel(logging.INFO)
+
+    gordo_ctx.obj = gordo_ctx.params
+
 
 @click.command()
 @click.argument("machine-config", envvar="MACHINE", type=yaml.safe_load)

gordo_components/cli/workflow_generator.py

@@ -3,6 +3,7 @@
 import pkg_resources
 import json
 import sys
+import os
 
 from typing import Dict, Any
 
@@ -21,7 +22,8 @@
 
 
 @click.group("workflow")
-def workflow_cli():
+@click.pass_context
+def workflow_cli(gordo_ctx):
     pass
 
 
@@ -111,15 +113,23 @@ def workflow_cli():
     help="The docker registry to use for pulling component images from",
     envvar=f"{PREFIX}_DOCKER_REGISTRY",
 )
-def workflow_generator_cli(**ctx: dict):
+@click.pass_context
+def workflow_generator_cli(gordo_ctx, **ctx):
     """
     Machine Configuration to Argo Workflow
     """
 
     context: Dict[Any, Any] = ctx.copy()
-
     yaml_content = wg.get_dict_from_yaml(context["machine_config"])
 
+    try:
+        log_level = yaml_content["globals"]["runtime"]["log_level"]
+    except KeyError:
+        log_level = os.getenv("GORDO_LOG_LEVEL", gordo_ctx.obj["log_level"])
+
+    logging.getLogger("gordo_components").setLevel(log_level.upper())
+    context["log_level"] = log_level.upper()
+
     # Create normalized config
     config = NormalizedConfig(yaml_content, project_name=context["project_name"])
 

gordo_components/workflow/workflow_generator/resources/argo-workflow.yml.template

@@ -79,6 +79,8 @@ spec:
         value: "{{project_name}}"
       - name: GORDO_PROJECT_REVISION
         value: "{{project_revision}}"
+      - name: GORDO_LOG_LEVEL
+        value: "{{log_level}}"
 
   - name: apply-with-retries
     retryStrategy:
@@ -654,6 +656,8 @@ spec:
           secretKeyRef:
             name: dlserviceauth
             key: tenant_id_secret
+      - name: GORDO_LOG_LEVEL
+        value: "{{log_level}}"
       volumeMounts:
       - name: azurefile
         mountPath: /gordo
@@ -876,6 +880,8 @@ spec:
                              env:
                                - name: MODEL_COLLECTION_DIR
                                  value: /gordo/models/{{project_name}}/{{project_revision}}
+                               - name: GORDO_LOG_LEVEL
+                                 value: "{{log_level}}"
 
                              resources:
                                requests:
@@ -1034,6 +1040,8 @@ spec:
           secretKeyRef:
             name: dlserviceauth
             key: tenant_id_secret
+      - name: GORDO_LOG_LEVEL
+        value: "{{log_level}}"
 
   - name: influx-cleanup
     activeDeadlineSeconds: 300

tests/gordo_components/cli/test_cli.py

@@ -1,6 +1,7 @@
 import os
 import pytest
 import logging
+import re
 
 from click.testing import CliRunner
 from unittest import mock
@@ -441,3 +442,31 @@ def test_server_to_sql_cli():
     args = ["workflow", "server-to-sql", "--help"]
     result = runner.invoke(cli.gordo, args)
     assert result.exit_code == 0
+
+
+def test_log_level_cli():
+    """
+    Test that the --log-level option in the CLI sets the correct log-level in the genreated config file.
+    """
+
+    runner = CliRunner()
+    args = [
+        "--log-level",
+        "test_log_level",
+        "workflow",
+        "generate",
+        "--machine-config",
+        "examples/config_crd.yaml",
+        "--project-name",
+        "test",
+    ]
+
+    result = runner.invoke(cli.gordo, args)
+
+    # Find the value on the next line after the key GORDO_LOG_LEVEL
+    gordo_log_levels = re.findall(
+        r"(?<=GORDO_LOG_LEVEL\r|GORDO_LOG_LEVEL\n)[^\r\n]+", result.stdout
+    )
+
+    # Assert all the values to the GORDO_LOG_LEVEL key contains the correct log-level
+    assert all(["TEST_LOG_LEVEL" in value for value in gordo_log_levels])

tests/gordo_components/workflow/test_workflow_generator/data/config-test-with-log-key.yml

@@ -0,0 +1,28 @@
+machines:
+
+  - name: ct-23-0001 #1st machine
+    dataset:
+      tags: #list of tags for 1st machine
+        - GRA-TE  -23-0733.PV
+        - GRA-TT  -23-0719.PV
+      train_start_date: 2016-11-07T09:11:30+01:00
+      train_end_date: 2018-09-15T03:01:00+01:00
+
+  - name: ct-23-0002 #2nd machine
+    dataset:
+      tags: #list of tags for 2nd machine
+        - CT/1
+        - CT/2
+        - CT/3
+      train_start_date: 2016-11-07T09:11:30+01:00
+      train_end_date: 2018-09-15T03:01:00+01:00
+
+globals:
+  model:
+    sklearn.pipeline.Pipeline:
+      steps:
+        - sklearn.preprocessing.data.MinMaxScaler
+        - gordo_components.machine.model.models.KerasAutoEncoder:
+            kind: feedforward_hourglass
+  runtime:
+    log_level: debug

tests/gordo_components/workflow/test_workflow_generator/test_workflow_generator.py

@@ -2,6 +2,7 @@
 
 import logging
 import os
+import re
 
 import docker
 import pytest
@@ -491,3 +492,26 @@ def test_valid_owner_ref(owner_ref_str: str, valid: bool):
     else:
         with pytest.raises(TypeError):
             wg._valid_owner_ref(owner_ref_str)
+
+
+@pytest.mark.parametrize(
+    "test_file, log_level",
+    (
+        ("config-test-with-log-key.yml", "DEBUG"),
+        ("config-test-with-models.yml", "INFO"),
+    ),
+)
+def test_log_level_key(test_file: str, log_level: str, path_to_config_files: str):
+    """
+    Test that GORDO_LOG_LEVEL is set to the correct value if specified in the config file, or default to INFO if not
+    specified.
+    """
+    workflow_str = _generate_test_workflow_str(path_to_config_files, test_file)
+
+    # Find the value on the next line after the key GORDO_LOG_LEVEL
+    gordo_log_levels = re.findall(
+        r"(?<=GORDO_LOG_LEVEL\r|GORDO_LOG_LEVEL\n)[^\r\n]+", workflow_str
+    )
+
+    # Assert all the values to the GORDO_LOG_LEVEL key contains the correct log-level
+    assert all([log_level in value for value in gordo_log_levels])