-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathkafka2influxdb.example.toml
141 lines (120 loc) · 5.11 KB
/
kafka2influxdb.example.toml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
# kafka2influxdb configuration
# by default, you can put it in "/etc/kafka2influxdb/kafka2influxdb.toml"
# will get "batch_size" metrics as a batch from Kafka and push them to
# influxdb hosts
# default 5000
batch_size = 5000
# send a batch of metrics to InfluxDB after that duration, even if there are
# less than batch_size metrics
# milliseconds, default 60000
batch_max_duration = 60000
# from which Kafka topics to gather metrics. You can specify a glob.
# Default "metrics_*"
topics = ["metrics_*"]
# refresh the matching Kafka topics after that duration
# milliseconds, default 300000
refresh_topics = 300000
# if some error happens while consuming (kafka not available, influxdb not
# available, etc), what to do?
# if retry_delay_ms < 0: abort kafka2influxdb
# if retry_delay_ms = 0: retry without delay
# if retry_delay_ms > 0: pause for retry_delay_ms milliseconds, then retry.
# In next iteration, if some error happens again, the delay will be doubled
# each time.
# default: 30000
retry_delay_ms = 30000
# "mappings" associate Kafka topic names (can be globs) with InfluxDB
# configurations.
# - on the left side, specify a Kafka topic glob
# - on the right side, specify the ID of the matching "topic_conf"
# When a Kafka message will be push to InfluxDB as a measurement, the mappings
# will be tried in the provided order. So you should specify mappings with
# a precise glob first, and the default mapping ("*") last.
# You must specify one mapping per [[mapping]] section (TOML syntax...)
# default mapping: "*" => "default"
[[mapping]]
onetopic = "default"
[[mapping]]
anothertopic = "default"
[[mapping]]
"*webmail*" = "web"
[[mapping]]
"*ldap*" = "directories"
[[mapping]]
"*" = "default"
# The `topic_conf.default` section specify the default influxdb configuration.
# You can override the paramaters in further "topic_conf.any" sections
[topic_conf.default]
# InfluxDB host (default "http://influxdb:8086")
host = "http://influxdb:8086"
# Use authentication for InfluxDB (default false)
# If true, you need to provide at least an admin user or a normal user for InfluxDB connection
auth = true
# InfluxDB "normal" user to write metrics
username = "kafka2influxdb"
# InfluxDB password for "normal" user
password = "CHANGE_ME"
# InfluxDB "admin" user. It is used to create databases in InfluxDB and to grant rights to the normal user.
admin_username = "admin"
# InfluxDB password for the "admin" user
admin_password = "CHANGE_ME_ADMIN"
# Whether to create the needed databases in InfluxDB (default false)
# If true, you need to provide an admin user
# If false, the databases in InfluxDB must be created manually before running kafka2influxdb
create_databases = false
# metrics timestamp precision (default "ns")
precision = "ns"
# the retention policy to use for writes in InfluxDB (default "")
retention_policy = ""
# connection timeout to InfluxDB (ms, default 5000)
timeout = 5000
# InfluxDB database to write to (default "default_db")
dbname = "maindb"
# The format of metrics pulled from that kafka topic (default "json")
# Can be "json" or "influx" (= line protocol). Similar to the same parameter in telegraf.
format = "json"
# Whether to use TLS to connect to InfluxDB (default false)
tls_enable = false
# Path to the CA certificate that's used to verify the InfluxDB server's certificate (optional)
certificate_authority = ""
# Path to the client certificate, if you wish the kafka2influxdb to authenticate to the server (optional)
certificate = ""
# Path to the client private key (optional)
private_key = ""
# if true, do not check the server certificate (eg. for self-signed certificate, default false)
insecure = false
[topic_conf.directories]
dbname = "directories_metrics"
# other params will be inherited from [topic_conf.default]
[topic_conf.web]
dbname = "webapps"
host = "http://anotherhost:8086"
tls_enable = true
certificate_authority = "/path/to/CA.pem"
certificate = "/path/to/client/cert.pem"
private_key = "/path/to/client/private/key"
[kafka]
# the list of kafka brokers to connect to (default: ["kafka1", "kafka2", "kafka3"]
brokers = ["kafka1", "kafka2", "kafka3"]
# Kafka client ID (default "kafka2influxdb")
client_id = "kafka2influxdb"
# Kafka consumer group name (default "kafka2influxdb-cg")
consumer_group = "kafka2influxdb-cg"
# Kafka version (default "0.9.0.0"). We use consumer groups, so 0.9 is required at minimum.
version = "0.10.1.1"
# Whether to use TLS to connect to Kafka (default false)
tls_enable = false
# Path to the CA certificate if you wish to verify the Kafka brokers' certificates (optional)
certificate_authority = ""
# Path to the client certificate, if you wish kafka2influxdb to authentificate to the Kafka brokers (optional)
certificate = ""
# Path to the client private key
private_key = ""
# if true, do not check the Kafka brokers' certificates (default false)
insecure = false
# Whether to use SASL PLAIN to authenticate kafka2influxdb to the Kafka brokers (default false)
sasl_enable = false
# Username for SASL PLAIN
sasl_username = ""
# Password for SASL PLAIN
sasl_password = ""