diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0bf759116..550a0c428 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,8 @@ name: CI -on: [push, pull_request, release] +on: + push: + pull_request: jobs: test: @@ -106,35 +108,3 @@ jobs: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: make push-prod-images - - publish-to-pypi: - if: github.action.event == 'release' - runs-on: ubuntu-latest - strategy: - matrix: - os: [ubuntu-latest] - python-version: [3.7] - steps: - - uses: actions/checkout@v1 - - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - architecture: 'x64' - - - name: Install deps - run: | - pip install --upgrade pip - pip install -r requirements.txt - - - name: Build - run: | - python setup.py sdist - python setup.py bdist_wheel - - - name: Publish - env: - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - run: | - pip install twine - twine upload -u statoil-travis --skip-existing dist/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..20a04472a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,39 @@ +name: ReleaseCI + +on: + release: + push: + tags: + - "*" + +jobs: + publish-to-pypi: + runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.7] + steps: + - uses: actions/checkout@v1 + + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + architecture: 'x64' + + - name: Install deps + run: | + pip install --upgrade pip + pip install -r requirements.txt + + - name: Build + run: | + python setup.py sdist + python setup.py bdist_wheel + + - name: Publish + env: + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + run: | + pip install twine + twine upload -u statoil-travis --skip-existing dist/* diff --git a/gordo_components/cli/cli.py b/gordo_components/cli/cli.py index f2bad81db..314cb77d1 100644 --- a/gordo_components/cli/cli.py +++ b/gordo_components/cli/cli.py @@ -4,7 +4,6 @@ CLI interfaces """ -import os import logging import sys import traceback @@ -46,19 +45,29 @@ @click.group("gordo-components") @click.version_option(version=__version__, message=__version__) -def gordo(): +@click.option( + "--log-level", + type=str, + default="INFO", + help="Run workflow with custom log-level.", + envvar="GORDO_LOG_LEVEL", +) +@click.pass_context +def gordo(gordo_ctx: click.Context, **ctx): """ The main entry point for the CLI interface """ - # Set log level, defaulting to INFO - log_level = os.getenv("LOG_LEVEL", "INFO").upper() - logging.basicConfig( - level=getattr(logging, log_level), + level=getattr(logging, str(gordo_ctx.params.get("log_level")).upper()), format="[%(asctime)s] %(levelname)s [%(name)s.%(funcName)s:%(lineno)d] %(message)s", ) + # Set matplotlib log level to INFO to avoid noise + logging.getLogger("matplotlib").setLevel(logging.INFO) + + gordo_ctx.obj = gordo_ctx.params + @click.command() @click.argument("machine-config", envvar="MACHINE", type=yaml.safe_load) diff --git a/gordo_components/cli/workflow_generator.py b/gordo_components/cli/workflow_generator.py index 3ba21f498..411e5288b 100644 --- a/gordo_components/cli/workflow_generator.py +++ b/gordo_components/cli/workflow_generator.py @@ -3,6 +3,7 @@ import pkg_resources import json import sys +import os from typing import Dict, Any @@ -21,7 +22,8 @@ @click.group("workflow") -def workflow_cli(): +@click.pass_context +def workflow_cli(gordo_ctx): pass @@ -111,15 +113,23 @@ def workflow_cli(): help="The docker registry to use for pulling component images from", envvar=f"{PREFIX}_DOCKER_REGISTRY", ) -def workflow_generator_cli(**ctx: dict): +@click.pass_context +def workflow_generator_cli(gordo_ctx, **ctx): """ Machine Configuration to Argo Workflow """ context: Dict[Any, Any] = ctx.copy() - yaml_content = wg.get_dict_from_yaml(context["machine_config"]) + try: + log_level = yaml_content["globals"]["runtime"]["log_level"] + except KeyError: + log_level = os.getenv("GORDO_LOG_LEVEL", gordo_ctx.obj["log_level"]) + + logging.getLogger("gordo_components").setLevel(log_level.upper()) + context["log_level"] = log_level.upper() + # Create normalized config config = NormalizedConfig(yaml_content, project_name=context["project_name"]) diff --git a/gordo_components/workflow/workflow_generator/resources/argo-workflow.yml.template b/gordo_components/workflow/workflow_generator/resources/argo-workflow.yml.template index 228acbbb9..20e1f6338 100644 --- a/gordo_components/workflow/workflow_generator/resources/argo-workflow.yml.template +++ b/gordo_components/workflow/workflow_generator/resources/argo-workflow.yml.template @@ -79,6 +79,8 @@ spec: value: "{{project_name}}" - name: GORDO_PROJECT_REVISION value: "{{project_revision}}" + - name: GORDO_LOG_LEVEL + value: "{{log_level}}" - name: apply-with-retries retryStrategy: @@ -654,6 +656,8 @@ spec: secretKeyRef: name: dlserviceauth key: tenant_id_secret + - name: GORDO_LOG_LEVEL + value: "{{log_level}}" volumeMounts: - name: azurefile mountPath: /gordo @@ -876,6 +880,8 @@ spec: env: - name: MODEL_COLLECTION_DIR value: /gordo/models/{{project_name}}/{{project_revision}} + - name: GORDO_LOG_LEVEL + value: "{{log_level}}" resources: requests: @@ -1034,6 +1040,8 @@ spec: secretKeyRef: name: dlserviceauth key: tenant_id_secret + - name: GORDO_LOG_LEVEL + value: "{{log_level}}" - name: influx-cleanup activeDeadlineSeconds: 300 diff --git a/tests/gordo_components/cli/test_cli.py b/tests/gordo_components/cli/test_cli.py index 034a7d82b..4bbbc729a 100644 --- a/tests/gordo_components/cli/test_cli.py +++ b/tests/gordo_components/cli/test_cli.py @@ -1,6 +1,7 @@ import os import pytest import logging +import re from click.testing import CliRunner from unittest import mock @@ -441,3 +442,31 @@ def test_server_to_sql_cli(): args = ["workflow", "server-to-sql", "--help"] result = runner.invoke(cli.gordo, args) assert result.exit_code == 0 + + +def test_log_level_cli(): + """ + Test that the --log-level option in the CLI sets the correct log-level in the genreated config file. + """ + + runner = CliRunner() + args = [ + "--log-level", + "test_log_level", + "workflow", + "generate", + "--machine-config", + "examples/config_crd.yaml", + "--project-name", + "test", + ] + + result = runner.invoke(cli.gordo, args) + + # Find the value on the next line after the key GORDO_LOG_LEVEL + gordo_log_levels = re.findall( + r"(?<=GORDO_LOG_LEVEL\r|GORDO_LOG_LEVEL\n)[^\r\n]+", result.stdout + ) + + # Assert all the values to the GORDO_LOG_LEVEL key contains the correct log-level + assert all(["TEST_LOG_LEVEL" in value for value in gordo_log_levels]) diff --git a/tests/gordo_components/workflow/test_workflow_generator/data/config-test-with-log-key.yml b/tests/gordo_components/workflow/test_workflow_generator/data/config-test-with-log-key.yml new file mode 100644 index 000000000..70a67e96a --- /dev/null +++ b/tests/gordo_components/workflow/test_workflow_generator/data/config-test-with-log-key.yml @@ -0,0 +1,28 @@ +machines: + + - name: ct-23-0001 #1st machine + dataset: + tags: #list of tags for 1st machine + - GRA-TE -23-0733.PV + - GRA-TT -23-0719.PV + train_start_date: 2016-11-07T09:11:30+01:00 + train_end_date: 2018-09-15T03:01:00+01:00 + + - name: ct-23-0002 #2nd machine + dataset: + tags: #list of tags for 2nd machine + - CT/1 + - CT/2 + - CT/3 + train_start_date: 2016-11-07T09:11:30+01:00 + train_end_date: 2018-09-15T03:01:00+01:00 + +globals: + model: + sklearn.pipeline.Pipeline: + steps: + - sklearn.preprocessing.data.MinMaxScaler + - gordo_components.machine.model.models.KerasAutoEncoder: + kind: feedforward_hourglass + runtime: + log_level: debug diff --git a/tests/gordo_components/workflow/test_workflow_generator/test_workflow_generator.py b/tests/gordo_components/workflow/test_workflow_generator/test_workflow_generator.py index 5d2788aa8..7ae58c765 100644 --- a/tests/gordo_components/workflow/test_workflow_generator/test_workflow_generator.py +++ b/tests/gordo_components/workflow/test_workflow_generator/test_workflow_generator.py @@ -2,6 +2,7 @@ import logging import os +import re import docker import pytest @@ -491,3 +492,26 @@ def test_valid_owner_ref(owner_ref_str: str, valid: bool): else: with pytest.raises(TypeError): wg._valid_owner_ref(owner_ref_str) + + +@pytest.mark.parametrize( + "test_file, log_level", + ( + ("config-test-with-log-key.yml", "DEBUG"), + ("config-test-with-models.yml", "INFO"), + ), +) +def test_log_level_key(test_file: str, log_level: str, path_to_config_files: str): + """ + Test that GORDO_LOG_LEVEL is set to the correct value if specified in the config file, or default to INFO if not + specified. + """ + workflow_str = _generate_test_workflow_str(path_to_config_files, test_file) + + # Find the value on the next line after the key GORDO_LOG_LEVEL + gordo_log_levels = re.findall( + r"(?<=GORDO_LOG_LEVEL\r|GORDO_LOG_LEVEL\n)[^\r\n]+", workflow_str + ) + + # Assert all the values to the GORDO_LOG_LEVEL key contains the correct log-level + assert all([log_level in value for value in gordo_log_levels])