Add host property tagging

README.md

@@ -1,6 +1,6 @@
 # About
 
-Zabbix-auto-config is an utility that aims to automatically configure hosts, host groups, host inventories and templates in the monitoring software [Zabbix](https://www.zabbix.com/).
+Zabbix-auto-config is an utility that aims to automatically configure hosts, host groups, host inventories, host tags and templates in the monitoring software [Zabbix](https://www.zabbix.com/).
 
 Note: This is only tested with Zabbix 5.0 LTS.
 
@@ -232,9 +232,80 @@ See the [`Host`](https://github.com/unioslo/zabbix-auto-config/blob/2b45f1cb7da0
 
 ## Host inventory
 
-Zac manages only inventory properties configured as `managed_inventory` in `config.toml`. An inventory property will not be removed/blanked from Zabbix even if the inventory property is removed from `managed_inventory` list or from the host in the source e.g:
+ZAC manages only inventory properties configured as `managed_inventory` in `config.toml`. An inventory property will not be removed/blanked from Zabbix even if the inventory property is removed from `managed_inventory` list or from the host in the source e.g:
 
 1. Add "location=x" to a host in a source and wait for sync
 2. Remove the "location" property from the host in the source
 3. "location=x" will remain in Zabbix
 
+## Host property tagging
+
+ZAC can automatically tag hosts with their source properties. Host Properties is metadata/annotations for a host assigned by source collectors or host modifiers. See `collect()` function from `mysource.py` in [Application](#application) and [Source collectors](#source-collectors) for more information on how to assign properties.
+
+By default, property tagging is disabled, but can be enabled in the config file:
+
+```toml
+[zabbix.property_tagging]
+enabled = true
+tag = "property"
+include = []
+exclude = []
+```
+
+Property tagging uses the configured tag prefix to construct the tag name. The default tag prefix is `zac_`, but can be changed in the config file:
+
+```toml
+[zabbix]
+tags_prefix = "zac_"
+```
+
+With this configuration, each of the host's properties are used to create a tag with the format `zac_property:<property_name>`. 
+
+
+For example, if a host has the property `is_dhcp_server`, it will be tagged with `zac_property:is_dhcp_server`. Zabbix hosts can have multiple tags with the same name as long as their values are different. For a host with multiple properties, each property will create a tag named `zac_property` with different values corresponding to the property names.
+
+The name of the tag can be configured with the `tag` option, so if we, for example, want to change the tag format to `zac_role:<property_name>`, we can do so by changing the value of `tag`:
+
+```toml
+tag = "role"
+```
+
+### Filtering
+
+The properties used for tagging can be filtered using the options `include` & `exclude`, which are lists of regular expressions. If no patterns are provided, all properties are used.
+
+If a host has a property called `broken_server` and we want to exclude it from being used as a tag, we can add an exclude pattern:
+
+```toml
+exclude = ["broken_server"]
+```
+
+If we want to exclude all properties starting with `broken_`, we can add a wildcard pattern:
+
+```toml
+exclude = ["broken_.*"]
+```
+
+If we only want to include properties that follow the `is_<type>_server` pattern, we can use an include pattern:
+```toml
+include = ["is_.*_server"]
+```
+
+Given multiple include patterns, properties must match at least one pattern to be included:
+
+```toml
+include = ["is_.*_server", "is_.*_host"]
+```
+
+More advanced include patterns can be used to match multiple properties with a single pattern:
+
+```toml
+include = ["is_(good|bad)_(server|host)"]
+```
+
+We can also combine include and exclude patterns to create more advanced filters:
+
+```toml
+include = ["is_.*_server"]
+exclude = ["is_broken_server", "is_bad_server"]
+```

config.sample.toml

@@ -22,6 +22,14 @@ managed_inventory = ["location"]
 #hostgroup_importance_prefix = "Importance-"
 #extra_siteadmin_hostgroup_prefixes = []
 
+[zabbix.property_tagging]
+enabled = false
+# name of tag without prefix. e.g. "property" becomes "zac_property"
+tag = "property"
+# Regular expressions to include/exclude properties with
+include = []
+exclude = []
+
 [source_collectors.mysource]
 module_name = "mysource"
 update_interval = 60

tests/conftest.py

@@ -3,6 +3,13 @@
 from pathlib import Path
 from typing import Iterable
 import pytest
+from zabbix_auto_config.models import (
+    Host,
+    Settings,
+    ZabbixSettings,
+    ZacSettings,
+    PropertyTaggingSettings,
+)
 
 
 @pytest.fixture(scope="function")
@@ -100,8 +107,14 @@ def sample_config():
         yield config.read()
 
 
+@pytest.fixture(scope="function")
+def map_dir(tmp_path: Path) -> Iterable[Path]:
+    mapdir = tmp_path / "maps"
+    mapdir.mkdir()
+    yield mapdir
+
 @pytest.fixture
-def hostgroup_map_file(tmp_path: Path) -> Iterable[Path]:
+def hostgroup_map_file(map_dir: Path) -> Iterable[Path]:
     contents = """
 # This file defines assosiation between siteadm fetched from Nivlheim and hostsgroups in Zabbix.
 # A siteadm can be assosiated only with one hostgroup or usergroup.
@@ -119,14 +132,89 @@ def hostgroup_map_file(tmp_path: Path) -> Iterable[Path]:
 #
 [email protected]:Hostgroup-user3-primary
 """
-    map_file_path = tmp_path / "siteadmin_hostgroup_map.txt"
+    map_file_path = map_dir / "siteadmin_hostgroup_map.txt"
+    map_file_path.write_text(contents)
+    yield map_file_path
+
+
+@pytest.fixture
+def property_hostgroup_map_file(map_dir: Path) -> Iterable[Path]:
+    contents = """
+is_app_server:Role-app-servers
+is_adfs_server:Role-adfs-servers
+"""
+    map_file_path = map_dir / "property_hostgroup_map.txt"
+    map_file_path.write_text(contents)
+    yield map_file_path
+
+
+@pytest.fixture
+def property_template_map_file(map_dir: Path) -> Iterable[Path]:
+    contents = """
+is_app_server:Template-app-server
+is_adfs_server:Template-adfs-server
+"""
+    map_file_path = map_dir / "property_template_map.txt"
     map_file_path.write_text(contents)
     yield map_file_path
 
 
+@pytest.fixture
+def map_dir_with_files(
+    map_dir: Path,
+    hostgroup_map_file: Path,
+    property_hostgroup_map_file: Path,
+    property_template_map_file: Path,
+) -> Iterable[Path]:
+    """Creates all mapping files and returns the path to their directory."""
+    yield map_dir
+
+
 @pytest.fixture(autouse=True, scope="session")
 def setup_multiprocessing_start_method() -> None:
     # On MacOS we have to set the start mode to fork
     # when using multiprocessing-logging
     if os.uname == "Darwin":
-        multiprocessing.set_start_method("fork", force=True)
+        multiprocessing.set_start_method("fork", force=True)
+
+
+@pytest.fixture(scope="function")
+def minimal_host_obj() -> Host:
+    return Host(
+        enabled=True,
+        hostname="foo.example.com",
+    )
+
+
+@pytest.fixture(name="config", scope="function")
+def _config(map_dir_with_files: Path, tmp_path: Path) -> Settings:
+    # NOTE: consider moving this to conftest, so we can reuse it.
+    modifier_dir = tmp_path / "host_modifiers"
+    modifier_dir.mkdir()
+    source_collector_dir = tmp_path / "source_collectors"
+    source_collector_dir.mkdir()
+    return Settings(
+        zac=ZacSettings(
+            source_collector_dir=str(source_collector_dir),
+            host_modifier_dir=str(modifier_dir),
+            db_uri="dbname='zac' user='zabbix' host='localhost' password='secret' port=5432 connect_timeout=2",
+            health_file=tmp_path / "zac_health.json",
+        ),
+        zabbix=ZabbixSettings(
+            map_dir=str(map_dir_with_files),
+            url="http://localhost",
+            username="Admin",
+            password="zabbix",
+            dryrun=False,
+            failsafe=20,
+            tags_prefix="zac_",
+            managed_inventory=[],
+            property_tagging=PropertyTaggingSettings(
+                enabled=True,
+                tag="property",
+                include=[],
+                exclude=[],
+            ),
+        ),
+        source_collectors={},
+    )

tests/test_processing/test_sourcemerger.py

@@ -0,0 +1,45 @@
+import multiprocessing
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+from zabbix_auto_config.models import Host, Settings
+from zabbix_auto_config.processing import SourceMergerProcess
+
+
+@pytest.mark.parametrize("dryrun", [True, False])
+@patch("psycopg2.connect", MagicMock())
+@patch("pyzabbix.ZabbixAPI", MagicMock())
+def test_set_property_tags(config: Settings, dryrun: bool):
+    """Tests set_property_tags() with a host that has 1 unmanaged tag and 1 managed tag.
+
+    We expect the unamanged tag to be kept, while the managed tag is removed
+    due to no corresponding host property.
+    """
+    config.zabbix.dryrun = dryrun
+
+    config.zabbix.property_tagging.tag = "property"
+    config.zabbix.tags_prefix = "zac_"
+
+    process = SourceMergerProcess(
+        "test-zabbix-host-updater",
+        multiprocessing.Manager().dict(),
+        config,
+    )
+
+    host = Host(
+        hostname="foo.example.com",
+        enabled=True,
+        properties=["is_app_server", "is_adfs_server"],
+        tags=[("zac_tag1", "tag1value")],
+    )
+    host_tags_pre = host.tags.copy()
+    process.set_property_tags(host)
+
+    if not dryrun:
+        assert len(host.tags) == 3
+        assert ("zac_tag1", "tag1value") in host.tags  # existing tag is kept
+        assert ("zac_property", "is_adfs_server") in host.tags
+        assert ("zac_property", "is_app_server") in host.tags
+    else:
+        assert host.tags == host_tags_pre

tests/test_utils.py

@@ -1,6 +1,7 @@
 import logging
 from ipaddress import IPv4Address, IPv6Address
 from pathlib import Path
+import re
 from typing import Dict, List, Set, Tuple, Union
 
 import pytest
@@ -9,6 +10,7 @@
 from hypothesis import strategies as st
 
 from zabbix_auto_config import utils
+from zabbix_auto_config.models import Host
 
 
 @pytest.mark.parametrize(
@@ -230,3 +232,65 @@ def test_mapping_values_with_prefix_no_prefix_separator(
     )
     assert res == {"[email protected]": ["Foouser1-primary", "Foouser1-secondary"]}
     assert caplog.text.count("WARNING") == 2
+
+
+@pytest.mark.parametrize(
+    "properties,include,exclude,expected",
+    [
+        pytest.param(
+            {"is_app_server", "is_adfs_server"},
+            [re.compile(r"is_app_server")],
+            [],
+            {"is_app_server"},
+            id="include (simple pattern)",
+        ),
+        pytest.param(
+            {"is_app_server", "is_adfs_server"},
+            [],
+            [re.compile(r"is_app_server")],
+            {"is_adfs_server"},
+            id="exclude (simple pattern)",
+        ),
+        pytest.param(
+            {"is_app_server", "is_adfs_server"},
+            [],
+            [re.compile(r"is_.*_server")],
+            set(),
+            id="exclude (wildcard pattern)",
+        ),
+        pytest.param(
+            {"is_app_server", "is_adfs_server", "foo"},
+            [re.compile(r"is_.*_server")],
+            [re.compile(r"is_adfs.*")],
+            {"is_app_server"},
+            id="include+exclude (wildcard pattern)",
+        ),
+        pytest.param(
+            {"is_app_server", "is_adfs_server", "foo"},
+            [re.compile(r"is_.*"), re.compile(r".*_server")],
+            [],
+            {"is_app_server", "is_adfs_server"},
+            id="multiple include patterns",
+        ),
+        pytest.param(
+            {"is_app_server", "is_adfs_server", "is_dhcp_server"},
+            [re.compile(r"is_.*"), re.compile(r".*_server")],
+            [re.compile(r".*adfs.*"), re.compile(r".*dhcp.*")],
+            {"is_app_server"},
+            id="multiple include+exclude patterns",
+        ),
+    ],
+)
+def test_match_host_properties(
+    minimal_host_obj: Host,
+    properties: Set[str],
+    include: List[re.Pattern],
+    exclude: List[re.Pattern],
+    expected: Set[str],
+) -> None:
+    host = minimal_host_obj
+    host.properties = properties
+    assert (
+        utils.match_host_properties(host, include, exclude)
+        == expected
+    )

zabbix_auto_config/__init__.py

@@ -138,7 +138,9 @@ def main():
         process = processing.SourceHandlerProcess("source-handler", state_manager.dict(), config.zac.db_uri, source_hosts_queues)
         processes.append(process)
 
-        process = processing.SourceMergerProcess("source-merger", state_manager.dict(), config.zac.db_uri, config.zac.host_modifier_dir)
+        process = processing.SourceMergerProcess(
+            "source-merger", state_manager.dict(), config
+        )
         processes.append(process)
 
         process = processing.ZabbixHostUpdater("zabbix-host-updater", state_manager.dict(), config.zac.db_uri, config.zabbix)