Added tests for filters and fix a bug in PlainValuesFilter

frontera/logger/filters/__init__.py

@@ -1,27 +1,22 @@
 from __future__ import absolute_import
 import logging
 import six
-
-
-def format_str(s):
-    if isinstance(s, six.text_type):
-        return s.encode('ascii', 'ignore')
-    return str(s)
+from w3lib.util import to_native_str
 
 
 class PlainValuesFilter(logging.Filter):
     def __init__(self, separator=None, excluded_fields=None, msg_max_length=0):
         super(PlainValuesFilter, self).__init__()
-        self.separator = separator or " "
+        self.separator = to_native_str(separator or " ")
         self.excluded_fields = excluded_fields or []
         self.msg_max_length = msg_max_length
 
     def filter(self, record):
         if isinstance(record.msg, dict):
             for field_name in self.excluded_fields:
                 setattr(record, field_name, record.msg.get(field_name, ''))
-            record.msg = self.separator.join([format_str(value)
-                                              for key, value in record.msg.items()
+            record.msg = self.separator.join([to_native_str(value)
+                                              for key, value in six.iteritems(record.msg)
                                               if key not in self.excluded_fields])
             if self.msg_max_length and len(record.msg) > self.msg_max_length:
                 record.msg = record.msg[0:self.msg_max_length-3] + "..."

tests/test_filters.py

@@ -0,0 +1,122 @@
+import unittest
+
+from frontera.logger.filters import PLAINVALUES, INCLUDEFIELDS, EXCLUDEFIELDS
+from tests.utils import LoggingCaptureMixin, SetupDefaultLoggingMixin
+
+
+class BaseTestFilters(SetupDefaultLoggingMixin, LoggingCaptureMixin, unittest.TestCase):
+    def tearDown(self):
+        super(BaseTestFilters, self).setUp()
+        self.logger.handlers[0].filters = []
+
+    def addFilter(self, filter):
+        self.logger.handlers[0].addFilter(filter)
+
+
+class TestFilterPlainValues(BaseTestFilters):
+    def test_plain_values_exclude_fields(self):
+        filter = PLAINVALUES(excluded_fields=['event'])
+        self.addFilter(filter)
+        self.logger.debug({'message1': 'logging', 'message2': 'debug', 'event': 'value'})
+        log_msg = self.logger_output.getvalue()
+        assert log_msg == 'logging debug\n' or log_msg == 'debug logging\n'
+
+    def test_plain_values_separator(self):
+        filter = PLAINVALUES(separator=',')
+        self.addFilter(filter)
+        self.logger.debug({'message1': 'logging', 'message2': 'debug'})
+        log_msg = self.logger_output.getvalue()
+        assert log_msg == 'logging,debug\n' or log_msg == 'debug,logging\n'
+
+    def test_plain_values_msg_max_length(self):
+        filter = PLAINVALUES(msg_max_length=10)
+        self.addFilter(filter)
+        self.logger.debug({'message1': '1' * 10, 'message2': '2' * 10})
+        log_msg = self.logger_output.getvalue()
+        assert log_msg == '%s...\n' % ('1' * 7) or log_msg == '%s...\n' % ('2' * 7)
+
+    def test_plain_values_str_msg(self):
+        filter = PLAINVALUES(msg_max_length=10)
+        self.addFilter(filter)
+        self.logger.debug('debug message')
+        self.assertEqual(self.logger_output.getvalue(), 'debug message\n')
+
+
+class TestIncludeFields(BaseTestFilters):
+    def test_include_fields_matching_values(self):
+        filter = INCLUDEFIELDS(field_name='event', included_values=['page_crawled'])
+        self.addFilter(filter)
+        self.logger.debug('crawled page P', extra={'event': 'page_crawled'})
+        self.assertEqual(self.logger_output.getvalue(), 'crawled page P\n')
+
+    def test_include_fields_non_matching_values(self):
+        filter = INCLUDEFIELDS(field_name='event', included_values=['links_extracted'])
+        self.addFilter(filter)
+        self.logger.debug('crawled page P', extra={'event': 'page_crawled'})
+        self.assertEqual(self.logger_output.getvalue(), '')
+
+    def test_include_fields_dict_msg_matching_values(self):
+        filter = INCLUDEFIELDS(field_name='event', included_values=['page_crawled'])
+        self.addFilter(filter)
+        self.logger.debug({'message': 'debug message', 'event': 'page_crawled'})
+        log_msg = self.logger_output.getvalue()
+        assert log_msg == "{'event': 'page_crawled', 'message': 'debug message'}\n" or \
+               log_msg == "{'message': 'debug message', 'event': 'page_crawled'}\n"
+
+    def test_include_fields_dict_msg_non_matching_values(self):
+        filter = INCLUDEFIELDS(field_name='event', included_values=['links_extracted'])
+        self.addFilter(filter)
+        self.logger.debug({'message': 'debug message', 'event': 'page_crawled'})
+        self.assertEqual(self.logger_output.getvalue(), '')
+
+    def test_include_fields_field_name_none(self):
+        filter = INCLUDEFIELDS(field_name=None, included_values=[])
+        self.addFilter(filter)
+        self.logger.debug('debug message')
+        self.assertEqual(self.logger_output.getvalue(), 'debug message\n')
+
+    def test_include_fields_list_message(self):
+        filter = INCLUDEFIELDS(field_name='event', included_values=['page_crawled'])
+        self.addFilter(filter)
+        self.logger.debug(['debug message'])
+        self.assertEqual(self.logger_output.getvalue(), "['debug message']\n")
+
+
+class TestExcludeFields(BaseTestFilters):
+    def test_exclude_fields_matching_values(self):
+        filter = EXCLUDEFIELDS(field_name='event', excluded_fields=['page_crawled'])
+        self.addFilter(filter)
+        self.logger.debug('crawled page P', extra={'event': 'page_crawled'})
+        self.assertEqual(self.logger_output.getvalue(), '')
+
+    def test_exclude_fields_non_matching_values(self):
+        filter = EXCLUDEFIELDS(field_name='event', excluded_fields=['links_extracted'])
+        self.addFilter(filter)
+        self.logger.debug('crawled page P', extra={'event': 'page_crawled'})
+        self.assertEqual(self.logger_output.getvalue(), 'crawled page P\n')
+
+    def test_exclude_fields_dict_msg_matching_values(self):
+        filter = EXCLUDEFIELDS(field_name='event', excluded_fields='page_crawled')
+        self.addFilter(filter)
+        self.logger.debug({'message': 'debug message', 'event': 'page_crawled'})
+        self.assertEqual(self.logger_output.getvalue(), '')
+
+    def test_exclude_fields_dict_msg_non_matching_values(self):
+        filter = EXCLUDEFIELDS(field_name='event', excluded_fields='links_extracted')
+        self.addFilter(filter)
+        self.logger.debug({'message': 'debug message', 'event': 'page_crawled'})
+        log_msg = self.logger_output.getvalue()
+        assert log_msg == "{'event': 'page_crawled', 'message': 'debug message'}\n" or \
+               log_msg == "{'message': 'debug message', 'event': 'page_crawled'}\n"
+
+    def test_include_fields_field_name_none(self):
+        filter = EXCLUDEFIELDS(field_name=None, excluded_fields=[])
+        self.addFilter(filter)
+        self.logger.debug('debug message')
+        self.assertEqual(self.logger_output.getvalue(), 'debug message\n')
+
+    def test_include_fields_list_message(self):
+        filter = EXCLUDEFIELDS(field_name='event', excluded_fields=['page_crawled'])
+        self.addFilter(filter)
+        self.logger.debug(['debug message'])
+        self.assertEqual(self.logger_output.getvalue(), "['debug message']\n")