Merge pull request #48 from signalfx/flake8fixes

Fix flake 8 + spelling errors and add test data for es 5.2 and 5.3

elasticsearch_collectd.py

@@ -435,7 +435,7 @@
     "indices[index={index_name}].primaries.merges.current-size":
         Stat("gauge", "primaries.merges.current_size_in_bytes"),
 
-    # COMPELTION
+    # COMPLETION
     "indices[index={index_name}].primaries.completion.size":
         Stat("gauge", "primaries.completion.size_in_bytes"),
 
@@ -632,7 +632,7 @@ def str_to_bool(value):
     bool() will return true for any string with a length greater than 0.  It
     does not cast a string with the text "true" or "false" to the
     corresponding bool value.  This method is a casting function.  It is
-    insensetive to case and leading/trailing spaces.  An Exception is raised
+    insensitive to case and leading/trailing spaces.  An Exception is raised
     if a cast can not be made.
     """
     if str(value).strip().lower() == "true":
@@ -718,7 +718,7 @@ def configure_callback(conf):
     # determine node information
     load_es_info()
 
-    # intialize stats map based on ES version
+    # initialize stats map based on ES version
     init_stats()
 
     # register the read callback now that we have the complete config
@@ -728,9 +728,9 @@ def configure_callback(conf):
         COLLECTION_INTERVAL)
 
 
-def sanatize_intervals():
-    """Sanatizes the index interval to be greater or equal to and divisible by
-    the colleciton interval
+def sanitize_intervals():
+    """Sanitizes the index interval to be greater or equal to and divisible by
+    the collection interval
     """
     global INDEX_INTERVAL, COLLECTION_INTERVAL, INDEX_SKIP, SKIP_COUNT
     # Sanitize the COLLECTION_INTERVAL and INDEX_INTERVAL
@@ -782,7 +782,7 @@ def remove_deprecated_elements(deprecated, elements, version):
     # Attempt to parse the major, minor, and revision
     (major, minor, revision) = version.split('.')
 
-    # Sanatize alphas and betas from revision number
+    # Sanitize alphas and betas from revision number
     revision = revision.split('-')[0]
 
     # Iterate over deprecation lists and remove any keys that were deprecated
@@ -805,17 +805,16 @@ def remove_deprecated_elements(deprecated, elements, version):
 
 # helper methods
 def init_stats():
-    global ES_HOST, ES_PORT, ES_NODE_URL, ES_URL_SCHEME, ES_CLUSTER_URL, ES_INDEX_URL, \
-        ES_VERSION, NODE_STATS_CUR, INDEX_STATS_CUR, \
+    global ES_HOST, ES_PORT, ES_NODE_URL, ES_URL_SCHEME, ES_CLUSTER_URL, \
+        ES_INDEX_URL, ES_VERSION, NODE_STATS_CUR, INDEX_STATS_CUR, \
         CLUSTER_STATS_CUR, ENABLE_INDEX_STATS, ENABLE_CLUSTER_STATS, \
         INDEX_INTERVAL, INDEX_SKIP, COLLECTION_INTERVAL, SKIP_COUNT, \
         DEPRECATED_NODE_STATS, THREAD_POOLS, CONFIGURED_THREAD_POOLS
 
-    sanatize_intervals()
+    sanitize_intervals()
 
     ES_NODE_URL = ES_URL_SCHEME + "://" + ES_HOST + ":" + str(ES_PORT) + \
-                  "/_nodes/_local/stats/transport,http,process,jvm,indices," \
-                  "thread_pool"
+        "/_nodes/_local/stats/transport,http,process,jvm,indices,thread_pool"
     NODE_STATS_CUR = dict(NODE_STATS.items())
     INDEX_STATS_CUR = dict(INDEX_STATS.items())
     if not ES_VERSION.startswith("1."):
@@ -867,7 +866,7 @@ def init_stats():
     remove_deprecated_threads()
 
     ES_CLUSTER_URL = ES_URL_SCHEME + "://" + ES_HOST + \
-                     ":" + str(ES_PORT) + "/_cluster/health"
+        ":" + str(ES_PORT) + "/_cluster/health"
 
     log.notice('Initialized with version=%s, host=%s, port=%s, url=%s' %
                (ES_VERSION, ES_HOST, ES_PORT, ES_NODE_URL))
@@ -926,11 +925,12 @@ def fetch_stats():
                 log.info('Parsing index stats for _all summary')
                 parse_index_stats(indices['_all'], '_all')
             else:
-	        indexes_json_stats = indices['indices']
+                indexes_json_stats = indices['indices']
                 for index_name in indexes_json_stats.keys():
                     log.info('Parsing index stats for index: %s' % index_name)
-                    parse_index_stats(indexes_json_stats[index_name], index_name)
-    # Incrememnt skip count
+                    parse_index_stats(indexes_json_stats[index_name],
+                                      index_name)
+    # Increment skip count
     SKIP_COUNT += 1
 
 
@@ -1006,8 +1006,8 @@ def detect_es_master():
     ES_CURRENT_MASTER"""
     global ES_CURRENT_MASTER
     # determine current master
-    cluster_state = fetch_url(ES_URL_SCHEME + "://" + ES_HOST + ":" + str(ES_PORT) +
-                              "/_cluster/state/master_node")
+    cluster_state = fetch_url(ES_URL_SCHEME + "://" + ES_HOST + ":"
+                              + str(ES_PORT) + "/_cluster/state/master_node")
     if ES_CURRENT_MASTER is False and cluster_state['master_node'] == NODE_ID:
         ES_CURRENT_MASTER = True
         log.notice('current master: %s' % ES_CURRENT_MASTER)
@@ -1186,7 +1186,7 @@ def __init__(self, plugin="elasticsearch", verbose=False):
 
     def emit(self, record):
         """
-        Emits a log record to the appropraite collectd log function
+        Emits a log record to the appropriate collectd log function
 
         Arguments
         record -- str log record to be emitted
@@ -1254,7 +1254,7 @@ def configure_test():
     global CONFIGURED_THREAD_POOLS, DETAILED_METRICS, INDEX_INTERVAL, \
         ENABLE_INDEX_STATS, ENABLE_CLUSTER_STATS, ES_MASTER_ELIGIBLE
 
-    # Ensure all possible threadpools are elligible for collection
+    # Ensure all possible threadpools are eligible for collection
     CONFIGURED_THREAD_POOLS = set(['generic', 'index', 'get', 'snapshot',
                                    'bulk', 'warmer', 'flush', 'search',
                                    'refresh', 'suggest', 'percolate',

tests/data/5.2.2/_all/_stats/index.json

@@ -0,0 +1 @@
+{"_shards":{"total":10,"successful":5,"failed":0},"_all":{"primaries":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":63,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":13},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":291912,"version_map_memory_in_bytes":106,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":310},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":63,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":13},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":291912,"version_map_memory_in_bytes":106,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":310},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{"twitter":{"primaries":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":63,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":13},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":291912,"version_map_memory_in_bytes":106,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":310},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":63,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":13},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":291912,"version_map_memory_in_bytes":106,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":1,"size_in_bytes":310},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}

tests/data/5.2.2/_cluster/health.json

@@ -0,0 +1 @@
+{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":50.0}

tests/data/5.2.2/_cluster/state/master_node.json

@@ -0,0 +1 @@
+{"cluster_name":"elasticsearch","master_node":"ZgtZbphISoqXoGU34hCDww"}

tests/data/5.2.2/_nodes/_local/index.json

@@ -0,0 +1 @@
+{"_nodes":{"total":1,"successful":1,"failed":0},"cluster_name":"elasticsearch","nodes":{"ZgtZbphISoqXoGU34hCDww":{"name":"ZgtZbph","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1","version":"5.2.2","build_hash":"f9d9b74","total_indexing_buffer":211261849,"roles":["master","data","ingest"],"settings":{"client":{"type":"node"},"cluster":{"name":"elasticsearch"},"http":{"host":"0.0.0.0","type":{"default":"netty4"}},"node":{"name":"ZgtZbph"},"path":{"logs":"/usr/share/elasticsearch/logs","home":"/usr/share/elasticsearch"},"transport":{"type":{"default":"netty4"}}},"os":{"refresh_interval_in_millis":1000,"name":"Linux","arch":"amd64","version":"4.9.12-moby","available_processors":4,"allocated_processors":4},"process":{"refresh_interval_in_millis":1000,"id":1,"mlockall":false},"jvm":{"pid":1,"version":"1.8.0_121","vm_name":"OpenJDK 64-Bit Server VM","vm_version":"25.121-b13","vm_vendor":"Oracle Corporation","start_time_in_millis":1490017996573,"mem":{"heap_init_in_bytes":2147483648,"heap_max_in_bytes":2112618496,"non_heap_init_in_bytes":2555904,"non_heap_max_in_bytes":0,"direct_max_in_bytes":2112618496},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Metaspace","Compressed Class Space","Par Eden Space","Par Survivor Space","CMS Old Gen"],"using_compressed_ordinary_object_pointers":"true"},"thread_pool":{"force_merge":{"type":"fixed","min":1,"max":1,"queue_size":-1},"fetch_shard_started":{"type":"scaling","min":1,"max":8,"keep_alive":"5m","queue_size":-1},"listener":{"type":"fixed","min":2,"max":2,"queue_size":-1},"index":{"type":"fixed","min":4,"max":4,"queue_size":200},"refresh":{"type":"scaling","min":1,"max":2,"keep_alive":"5m","queue_size":-1},"generic":{"type":"scaling","min":4,"max":128,"keep_alive":"30s","queue_size":-1},"warmer":{"type":"scaling","min":1,"max":2,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":7,"max":7,"queue_size":1000},"flush":{"type":"scaling","min":1,"max":2,"keep_alive":"5m","queue_size":-1},"fetch_shard_store":{"type":"scaling","min":1,"max":8,"keep_alive":"5m","queue_size":-1},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":4,"max":4,"queue_size":1000},"bulk":{"type":"fixed","min":4,"max":4,"queue_size":50},"snapshot":{"type":"scaling","min":1,"max":2,"keep_alive":"5m","queue_size":-1}},"transport":{"bound_address":["[::1]:9300","127.0.0.1:9300"],"publish_address":"127.0.0.1:9300","profiles":{}},"http":{"bound_address":["[::]:9200"],"publish_address":"172.17.0.3:9200","max_content_length_in_bytes":104857600},"plugins":[],"modules":[{"name":"aggs-matrix-stats","version":"5.2.2","description":"Adds aggregations whose input are a list of numeric fields and output includes a matrix.","classname":"org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin"},{"name":"ingest-common","version":"5.2.2","description":"Module for ingest processors that do not require additional security permissions or have large dependencies and resources","classname":"org.elasticsearch.ingest.common.IngestCommonPlugin"},{"name":"lang-expression","version":"5.2.2","description":"Lucene expressions integration for Elasticsearch","classname":"org.elasticsearch.script.expression.ExpressionPlugin"},{"name":"lang-groovy","version":"5.2.2","description":"Groovy scripting integration for Elasticsearch","classname":"org.elasticsearch.script.groovy.GroovyPlugin"},{"name":"lang-mustache","version":"5.2.2","description":"Mustache scripting integration for Elasticsearch","classname":"org.elasticsearch.script.mustache.MustachePlugin"},{"name":"lang-painless","version":"5.2.2","description":"An easy, safe and fast scripting language for Elasticsearch","classname":"org.elasticsearch.painless.PainlessPlugin"},{"name":"percolator","version":"5.2.2","description":"Percolator module adds capability to index queries and query these queries by specifying documents","classname":"org.elasticsearch.percolator.PercolatorPlugin"},{"name":"reindex","version":"5.2.2","description":"The Reindex module adds APIs to reindex from one index to another or update documents in place.","classname":"org.elasticsearch.index.reindex.ReindexPlugin"},{"name":"transport-netty3","version":"5.2.2","description":"Netty 3 based transport implementation","classname":"org.elasticsearch.transport.Netty3Plugin"},{"name":"transport-netty4","version":"5.2.2","description":"Netty 4 based transport implementation","classname":"org.elasticsearch.transport.Netty4Plugin"}],"ingest":{"processors":[{"type":"append"},{"type":"convert"},{"type":"date"},{"type":"date_index_name"},{"type":"dot_expander"},{"type":"fail"},{"type":"foreach"},{"type":"grok"},{"type":"gsub"},{"type":"join"},{"type":"json"},{"type":"kv"},{"type":"lowercase"},{"type":"remove"},{"type":"rename"},{"type":"script"},{"type":"set"},{"type":"sort"},{"type":"split"},{"type":"trim"},{"type":"uppercase"}]}}}}