diff --git a/README.md b/README.md index d089214..64df383 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # About -Zabbix-auto-config is an utility that aims to automatically configure hosts, host groups, host inventories and templates in the monitoring software [Zabbix](https://www.zabbix.com/). +Zabbix-auto-config is an utility that aims to automatically configure hosts, host groups, host inventories, host tags and templates in the monitoring software [Zabbix](https://www.zabbix.com/). Note: This is only tested with Zabbix 5.0 LTS. @@ -232,9 +232,80 @@ See the [`Host`](https://github.com/unioslo/zabbix-auto-config/blob/2b45f1cb7da0 ## Host inventory -Zac manages only inventory properties configured as `managed_inventory` in `config.toml`. An inventory property will not be removed/blanked from Zabbix even if the inventory property is removed from `managed_inventory` list or from the host in the source e.g: +ZAC manages only inventory properties configured as `managed_inventory` in `config.toml`. An inventory property will not be removed/blanked from Zabbix even if the inventory property is removed from `managed_inventory` list or from the host in the source e.g: 1. Add "location=x" to a host in a source and wait for sync 2. Remove the "location" property from the host in the source 3. "location=x" will remain in Zabbix +## Host property tagging + +ZAC can automatically tag hosts with their source properties. Host Properties is metadata/annotations for a host assigned by source collectors or host modifiers. See `collect()` function from `mysource.py` in [Application](#application) and [Source collectors](#source-collectors) for more information on how to assign properties. + +By default, property tagging is disabled, but can be enabled in the config file: + +```toml +[zabbix.property_tagging] +enabled = true +tag = "property" +include = [] +exclude = [] +``` + +Property tagging uses the configured tag prefix to construct the tag name. The default tag prefix is `zac_`, but can be changed in the config file: + +```toml +[zabbix] +tags_prefix = "zac_" +``` + +With this configuration, each of the host's properties are used to create a tag with the format `zac_property:`. + + +For example, if a host has the property `is_dhcp_server`, it will be tagged with `zac_property:is_dhcp_server`. Zabbix hosts can have multiple tags with the same name as long as their values are different. For a host with multiple properties, each property will create a tag named `zac_property` with different values corresponding to the property names. + +The name of the tag can be configured with the `tag` option, so if we, for example, want to change the tag format to `zac_role:`, we can do so by changing the value of `tag`: + +```toml +tag = "role" +``` + +### Filtering + +The properties used for tagging can be filtered using the options `include` & `exclude`, which are lists of regular expressions. If no patterns are provided, all properties are used. + +If a host has a property called `broken_server` and we want to exclude it from being used as a tag, we can add an exclude pattern: + +```toml +exclude = ["broken_server"] +``` + +If we want to exclude all properties starting with `broken_`, we can add a wildcard pattern: + +```toml +exclude = ["broken_.*"] +``` + +If we only want to include properties that follow the `is__server` pattern, we can use an include pattern: +```toml +include = ["is_.*_server"] +``` + +Given multiple include patterns, properties must match at least one pattern to be included: + +```toml +include = ["is_.*_server", "is_.*_host"] +``` + +More advanced include patterns can be used to match multiple properties with a single pattern: + +```toml +include = ["is_(good|bad)_(server|host)"] +``` + +We can also combine include and exclude patterns to create more advanced filters: + +```toml +include = ["is_.*_server"] +exclude = ["is_broken_server", "is_bad_server"] +``` diff --git a/config.sample.toml b/config.sample.toml index f949345..62b857f 100644 --- a/config.sample.toml +++ b/config.sample.toml @@ -22,6 +22,14 @@ managed_inventory = ["location"] #hostgroup_importance_prefix = "Importance-" #extra_siteadmin_hostgroup_prefixes = [] +[zabbix.property_tagging] +enabled = false +# name of tag without prefix. e.g. "property" becomes "zac_property" +tag = "property" +# Regular expressions to include/exclude properties with +include = [] +exclude = [] + [source_collectors.mysource] module_name = "mysource" update_interval = 60 diff --git a/tests/conftest.py b/tests/conftest.py index 152a311..5f3fd2d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,13 @@ from pathlib import Path from typing import Iterable import pytest +from zabbix_auto_config.models import ( + Host, + Settings, + ZabbixSettings, + ZacSettings, + PropertyTaggingSettings, +) @pytest.fixture(scope="function") @@ -100,8 +107,14 @@ def sample_config(): yield config.read() +@pytest.fixture(scope="function") +def map_dir(tmp_path: Path) -> Iterable[Path]: + mapdir = tmp_path / "maps" + mapdir.mkdir() + yield mapdir + @pytest.fixture -def hostgroup_map_file(tmp_path: Path) -> Iterable[Path]: +def hostgroup_map_file(map_dir: Path) -> Iterable[Path]: contents = """ # This file defines assosiation between siteadm fetched from Nivlheim and hostsgroups in Zabbix. # A siteadm can be assosiated only with one hostgroup or usergroup. @@ -119,14 +132,89 @@ def hostgroup_map_file(tmp_path: Path) -> Iterable[Path]: # user3@example.com:Hostgroup-user3-primary """ - map_file_path = tmp_path / "siteadmin_hostgroup_map.txt" + map_file_path = map_dir / "siteadmin_hostgroup_map.txt" + map_file_path.write_text(contents) + yield map_file_path + + +@pytest.fixture +def property_hostgroup_map_file(map_dir: Path) -> Iterable[Path]: + contents = """ +is_app_server:Role-app-servers +is_adfs_server:Role-adfs-servers +""" + map_file_path = map_dir / "property_hostgroup_map.txt" + map_file_path.write_text(contents) + yield map_file_path + + +@pytest.fixture +def property_template_map_file(map_dir: Path) -> Iterable[Path]: + contents = """ +is_app_server:Template-app-server +is_adfs_server:Template-adfs-server +""" + map_file_path = map_dir / "property_template_map.txt" map_file_path.write_text(contents) yield map_file_path +@pytest.fixture +def map_dir_with_files( + map_dir: Path, + hostgroup_map_file: Path, + property_hostgroup_map_file: Path, + property_template_map_file: Path, +) -> Iterable[Path]: + """Creates all mapping files and returns the path to their directory.""" + yield map_dir + + @pytest.fixture(autouse=True, scope="session") def setup_multiprocessing_start_method() -> None: # On MacOS we have to set the start mode to fork # when using multiprocessing-logging if os.uname == "Darwin": - multiprocessing.set_start_method("fork", force=True) \ No newline at end of file + multiprocessing.set_start_method("fork", force=True) + + +@pytest.fixture(scope="function") +def minimal_host_obj() -> Host: + return Host( + enabled=True, + hostname="foo.example.com", + ) + + +@pytest.fixture(name="config", scope="function") +def _config(map_dir_with_files: Path, tmp_path: Path) -> Settings: + # NOTE: consider moving this to conftest, so we can reuse it. + modifier_dir = tmp_path / "host_modifiers" + modifier_dir.mkdir() + source_collector_dir = tmp_path / "source_collectors" + source_collector_dir.mkdir() + return Settings( + zac=ZacSettings( + source_collector_dir=str(source_collector_dir), + host_modifier_dir=str(modifier_dir), + db_uri="dbname='zac' user='zabbix' host='localhost' password='secret' port=5432 connect_timeout=2", + health_file=tmp_path / "zac_health.json", + ), + zabbix=ZabbixSettings( + map_dir=str(map_dir_with_files), + url="http://localhost", + username="Admin", + password="zabbix", + dryrun=False, + failsafe=20, + tags_prefix="zac_", + managed_inventory=[], + property_tagging=PropertyTaggingSettings( + enabled=True, + tag="property", + include=[], + exclude=[], + ), + ), + source_collectors={}, + ) diff --git a/tests/test_processing/test_sourcemerger.py b/tests/test_processing/test_sourcemerger.py new file mode 100644 index 0000000..9571ce6 --- /dev/null +++ b/tests/test_processing/test_sourcemerger.py @@ -0,0 +1,45 @@ +import multiprocessing +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from zabbix_auto_config.models import Host, Settings +from zabbix_auto_config.processing import SourceMergerProcess + + +@pytest.mark.parametrize("dryrun", [True, False]) +@patch("psycopg2.connect", MagicMock()) +@patch("pyzabbix.ZabbixAPI", MagicMock()) +def test_set_property_tags(config: Settings, dryrun: bool): + """Tests set_property_tags() with a host that has 1 unmanaged tag and 1 managed tag. + + We expect the unamanged tag to be kept, while the managed tag is removed + due to no corresponding host property. + """ + config.zabbix.dryrun = dryrun + + config.zabbix.property_tagging.tag = "property" + config.zabbix.tags_prefix = "zac_" + + process = SourceMergerProcess( + "test-zabbix-host-updater", + multiprocessing.Manager().dict(), + config, + ) + + host = Host( + hostname="foo.example.com", + enabled=True, + properties=["is_app_server", "is_adfs_server"], + tags=[("zac_tag1", "tag1value")], + ) + host_tags_pre = host.tags.copy() + process.set_property_tags(host) + + if not dryrun: + assert len(host.tags) == 3 + assert ("zac_tag1", "tag1value") in host.tags # existing tag is kept + assert ("zac_property", "is_adfs_server") in host.tags + assert ("zac_property", "is_app_server") in host.tags + else: + assert host.tags == host_tags_pre diff --git a/tests/test_utils.py b/tests/test_utils.py index ccb6bbe..11e6e97 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,7 @@ import logging from ipaddress import IPv4Address, IPv6Address from pathlib import Path +import re from typing import Dict, List, Set, Tuple, Union import pytest @@ -9,6 +10,7 @@ from hypothesis import strategies as st from zabbix_auto_config import utils +from zabbix_auto_config.models import Host @pytest.mark.parametrize( @@ -230,3 +232,65 @@ def test_mapping_values_with_prefix_no_prefix_separator( ) assert res == {"user1@example.com": ["Foouser1-primary", "Foouser1-secondary"]} assert caplog.text.count("WARNING") == 2 + + +@pytest.mark.parametrize( + "properties,include,exclude,expected", + [ + pytest.param( + {"is_app_server", "is_adfs_server"}, + [re.compile(r"is_app_server")], + [], + {"is_app_server"}, + id="include (simple pattern)", + ), + pytest.param( + {"is_app_server", "is_adfs_server"}, + [], + [re.compile(r"is_app_server")], + {"is_adfs_server"}, + id="exclude (simple pattern)", + ), + pytest.param( + {"is_app_server", "is_adfs_server"}, + [], + [re.compile(r"is_.*_server")], + set(), + id="exclude (wildcard pattern)", + ), + pytest.param( + {"is_app_server", "is_adfs_server", "foo"}, + [re.compile(r"is_.*_server")], + [re.compile(r"is_adfs.*")], + {"is_app_server"}, + id="include+exclude (wildcard pattern)", + ), + pytest.param( + {"is_app_server", "is_adfs_server", "foo"}, + [re.compile(r"is_.*"), re.compile(r".*_server")], + [], + {"is_app_server", "is_adfs_server"}, + id="multiple include patterns", + ), + pytest.param( + {"is_app_server", "is_adfs_server", "is_dhcp_server"}, + [re.compile(r"is_.*"), re.compile(r".*_server")], + [re.compile(r".*adfs.*"), re.compile(r".*dhcp.*")], + {"is_app_server"}, + id="multiple include+exclude patterns", + ), + ], +) +def test_match_host_properties( + minimal_host_obj: Host, + properties: Set[str], + include: List[re.Pattern], + exclude: List[re.Pattern], + expected: Set[str], +) -> None: + host = minimal_host_obj + host.properties = properties + assert ( + utils.match_host_properties(host, include, exclude) + == expected + ) diff --git a/zabbix_auto_config/__init__.py b/zabbix_auto_config/__init__.py index ee1ecac..d6acaa9 100644 --- a/zabbix_auto_config/__init__.py +++ b/zabbix_auto_config/__init__.py @@ -138,7 +138,9 @@ def main(): process = processing.SourceHandlerProcess("source-handler", state_manager.dict(), config.zac.db_uri, source_hosts_queues) processes.append(process) - process = processing.SourceMergerProcess("source-merger", state_manager.dict(), config.zac.db_uri, config.zac.host_modifier_dir) + process = processing.SourceMergerProcess( + "source-merger", state_manager.dict(), config + ) processes.append(process) process = processing.ZabbixHostUpdater("zabbix-host-updater", state_manager.dict(), config.zac.db_uri, config.zabbix) diff --git a/zabbix_auto_config/models.py b/zabbix_auto_config/models.py index a0b3e62..d1e8c4c 100644 --- a/zabbix_auto_config/models.py +++ b/zabbix_auto_config/models.py @@ -1,6 +1,6 @@ import logging from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Set, Tuple, Union, Pattern from pydantic import BaseModel from pydantic import BaseModel as PydanticBaseModel @@ -34,16 +34,24 @@ def _check_unknown_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: return values +class PropertyTaggingSettings(ConfigBaseModel): + enabled: bool = False + tag: str = "property" + include: List[Pattern] = [] + exclude: List[Pattern] = [] + + class ZabbixSettings(ConfigBaseModel): map_dir: str url: str username: str password: str dryrun: bool + failsafe: int = 20 tags_prefix: str = "zac_" managed_inventory: List[str] = [] - failsafe: int = 20 + hostgroup_all: str = "All-hosts" hostgroup_manual: str = "All-manual-hosts" @@ -60,6 +68,9 @@ class ZabbixSettings(ConfigBaseModel): # These groups are not managed by ZAC beyond creating them. extra_siteadmin_hostgroup_prefixes: Set[str] = set() + property_tagging: PropertyTaggingSettings = Field(default_factory=PropertyTaggingSettings) + + class ZacSettings(ConfigBaseModel): source_collector_dir: str host_modifier_dir: str diff --git a/zabbix_auto_config/processing.py b/zabbix_auto_config/processing.py index ec99e8d..b2f67f6 100644 --- a/zabbix_auto_config/processing.py +++ b/zabbix_auto_config/processing.py @@ -28,7 +28,6 @@ from ._types import HostModifierDict, SourceCollectorModule, HostModifierModule if TYPE_CHECKING: - from psycopg2.extensions import connection as Connection from psycopg2.extensions import cursor as Cursor class BaseProcess(multiprocessing.Process): @@ -335,13 +334,13 @@ def handle_source_hosts(self, source: str, hosts: List[models.Host]) -> None: class SourceMergerProcess(BaseProcess): - def __init__(self, name, state, db_uri, host_modifier_dir): + def __init__(self, name: str, state: dict, config: models.Settings): super().__init__(name, state) - - self.db_uri = db_uri + self.config = config + self.db_uri = self.config.zac.db_uri self.db_source_table = "hosts_source" self.db_hosts_table = "hosts" - self.host_modifier_dir = host_modifier_dir + self.host_modifier_dir = self.config.zac.host_modifier_dir self.host_modifiers = self.get_host_modifiers() logging.info("Loaded %d host modifiers: %s", len(self.host_modifiers), ", ".join([repr(modifier["name"]) for modifier in self.host_modifiers])) @@ -436,6 +435,10 @@ def handle_host( ) # TODO: Do more? + # Set tags for host based on its properties + if host.properties and self.config.zabbix.property_tagging.enabled: + self.set_property_tags(host) + if current_host: if current_host == host: # logging.debug(f"Host <{host['hostname']}> from source <{source}> is equal to current host") @@ -539,6 +542,46 @@ def merge_sources(self): self.next_update.isoformat(timespec="seconds"), ) + def set_property_tags(self, host: models.Host) -> None: + """Adds tags to a host based on its properties. + + Modifies the Host object in-place.""" + tagging = self.config.zabbix.property_tagging + tag_prefix = self.config.zabbix.tags_prefix + + # Remove current property tags, then set new ones. + # This ensures we always discard old tags for properties that have + # been removed since the last time the host was collected. + tag_name = f"{tag_prefix}{tagging.tag}" + new_tags = set(t for t in host.tags if t[0] != tag_name) + matched_properties = utils.match_host_properties( + host, tagging.include, tagging.exclude + ) + for prop in matched_properties: + new_tags.add((tag_name, prop)) + + # We can't directly compare new and old, because the order of the tags might be different. + if new_tags == host.tags: + logging.debug("No changes to property tags for host '%s'", host.hostname) + return + + if self.config.zabbix.dryrun: + logging.info( + "DRYRUN: Setting property tags on host '%s'. Old: %s. New: %s", + host.hostname, + host.tags, + new_tags, + ) + return + + logging.info( + "Setting property tags on host '%s'. Old: %s. New: %s", + host.hostname, + host.tags, + new_tags, + ) + host.tags = new_tags + class ZabbixUpdater(BaseProcess): def __init__(self, name, state, db_uri, zabbix_config: models.ZabbixSettings): diff --git a/zabbix_auto_config/utils.py b/zabbix_auto_config/utils.py index 02147ef..bc99ddf 100644 --- a/zabbix_auto_config/utils.py +++ b/zabbix_auto_config/utils.py @@ -6,7 +6,20 @@ from pathlib import Path import queue import re -from typing import Dict, Iterable, List, Mapping, MutableMapping, Set, Tuple, Union +from typing import ( + Dict, + Iterable, + List, + MutableMapping, + Optional, + Set, + Tuple, + Union, + TYPE_CHECKING, +) + +if TYPE_CHECKING: + from zabbix_auto_config.models import Host def is_valid_regexp(pattern: str): try: @@ -145,9 +158,7 @@ def mapping_values_with_prefix( try: new_value = with_prefix(text=v, prefix=prefix, separator=separator) except ValueError: - logging.warning( - f"Unable to replace prefix in '%s' with '%s'", v, prefix - ) + logging.warning("Unable to replace prefix in '%s' with '%s'", v, prefix) continue new_values.append(new_value) m[key] = new_values @@ -166,3 +177,59 @@ def drain_queue(q: multiprocessing.Queue) -> None: def timedelta_to_str(td: datetime.timedelta) -> str: """Converts a timedelta to a string of the form HH:MM:SS.""" return str(td).partition(".")[0] + + +def matches_patterns(text: str, patterns: List[re.Pattern]) -> Optional[re.Pattern]: + """Returns the first pattern that matches `text` or None if no pattern matches.""" + for pattern in patterns: + if pattern.match(text): + return pattern + return None + + +def match_host_properties( + host: "Host", include: List[re.Pattern], exclude: List[re.Pattern] +) -> Set[str]: + """Matches a host's properties based on include and exclude patterns. + + All properties are matched if both include and exclude patterns are empty. + + Parameters + ---------- + host : Host + A Zabbix Host object. + include : List[re.Pattern] + List of compiled regexps to match against the host's properties. + If non-empty, at least one include pattern must match. + exclude : List[re.Pattern] + List of compiled regexps to match against the host's properties. + If non-empty, no exclude pattern must match. + + Returns + ------- + Set[str] + Set of matched properties. + """ + matched_properties = set() # type: set[str] + for prop in host.properties: + if exclude: + matched = matches_patterns(prop, exclude) + if matched: + logging.debug( + "Skipping property '%s' for host '%s' (exclude pattern: '%s')", + prop, + host.hostname, + matched.pattern, + ) + continue + if include: + matched = matches_patterns(prop, include) + if not matched: + logging.debug( + "Skipping property '%s' for host '%s'. No include patterns matched.", + prop, + host.hostname, + ) + continue + matched_properties.add(prop) + return matched_properties