Configure logging (#20)

#19

* Update construct_dpdb_visu.py

Added main()
Moved import argparse to toplevel
in construct rename generalGraph to general_gr

* Update logging

* Update main docstring

+ Renamed ARGS to _args
+ argparse on toplevel import
+ sorted imports

* Update visualization.py

* Create logging.yml

* Remove logging.basicConfig

* Explicit logger names

Same as in logging.yml #19

* Update logging.yml

Update formatter

* Add utilities.py

Factor out common  helper methods.
Makes files easier to read, imports easier.
#7

* Move static methods to utilities

The static methods were generic enough to be a utility.
Other than that it helps to keep visualization.py more slim to just import these methods.
Some Format.

* Update read_cfg with read_yml_or_cfg

#19

Using utility method to read different cfg formats.

* Update test_bag_creation.py

Use utility

* Update logging Setup

Setup with utilities DefaultLoggingCfg

* Factored logging config dpdb into utilities

#19

* Some type refactoring

using int/float correctly

* Favtored logging out in visualization.py

Other main application that needs logging config.
#19

* loglevel default to None

loglevel=None means take what the config says.
The commandline parameter takes precedence over config.
If not None: update 'root' logger and it's handlers to that level. #19 #20

* Added logging.ini for comparison

Much more verbose, handlers+qualname needed most likely.
Updated logging.yml with WARNING default and all loggers use the root handler https://docs.python.org/3/howto/logging.html#logging-flow
#19

* Comments, Changelog

Comment basicConfig (provide information until logging is set up).

* Update svgjoin.py

Fixed wrong conversion to int.
Added better typing for f_transform

CHANGELOG.md

@@ -7,7 +7,30 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html ).
 
 ## [Unreleased]
 
-No unreleased changes yet.
+### Added
+- Added file utilities.py with several static or shared things like
+    - Constants: CFG_EXT, LOGLEVEL_EPILOG, DEFAULT_LOGGING_CFG 
+    - Methods: 
+        - flatten
+        - read_yml_or_cfg combining yaml, json, cfg reader in one
+        - logging_cfg configure logging with file or DEFAULT_LOGGING_CFG 
+        - helper convert_to_adj from dijkstra.py
+        - add_edge_to (edges and adj list)
+        - gen_arg infinite Generator
+    - Styles:
+        - base_style, emphasise_node, style_hide_node, style_hide_edge
+    - Graph manipulation:
+        - bag_node
+        - solution_node
+
+- Added file logging.yml (and .ini) with logging configuration for the module
+
+### Changed
+- Changed path of image SharpSatExample to the absolute URL for [PyPI].
+- Changed names of loggers to absolute name. Should be easy to adjust if needed.
+- Changed logging defaults and config in tdvisu/visualization.py and construct_dpdb_visu.py
+- Updated ArgumentParser help
+- Some fixes of code-style or variable names. 
 
 ## [1.0.1] - 2020-06-04
 ### Added

tdvisu/construct_dpdb_visu.py

@@ -23,14 +23,13 @@
     If not, see https://www.gnu.org/licenses/gpl-3.0.html
 
 """
-
-import json
 import abc
+import argparse
+import json
 import logging
-import pathlib
+
 
 from time import sleep
-from configparser import ConfigParser
 
 import psycopg2 as pg
 
@@ -39,16 +38,17 @@
 from tdvisu.reader import TwReader
 from tdvisu.visualization import flatten
 from tdvisu.version import __date__, __version__ as version
+from tdvisu.utilities import read_yml_or_cfg, logging_cfg, LOGLEVEL_EPILOG
 
+LOGGER = logging.getLogger('construct_dpdb_visu.py')
 
-logging.basicConfig(
-    format="%(asctime)s,%(msecs)d %(levelname)-8s"
-    "[%(filename)s:%(lineno)d] %(message)s",
-    datefmt='%Y-%m-%d %H:%M:%S',
-    level=logging.WARNING)
-
-LOGGER = logging.getLogger(__name__)
-
+DEFAULT_DBCONFIG = {
+    "host": "localhost",
+    "port": 5432,
+    "database": "logicsem",
+    "user": "postgres",
+    "application_name": "dpdb-admin"
+}
 
 PSYCOPG2_8_5_TASTATUS = {
     pg.extensions.TRANSACTION_STATUS_IDLE:
@@ -79,40 +79,25 @@ def good_db_status() -> tuple:
             pg.extensions.TRANSACTION_STATUS_INTRANS)
 
 
-def read_cfg(cfg_file, section) -> dict:
-    """Read the config file and return the result.
-
-    Works for both .ini and .json files but
-    assumes json-format if the ending is NOT .ini
-    """
-    if pathlib.Path(cfg_file).suffix.lower() == '.ini':
-        iniconfig = ConfigParser()
-        iniconfig.read(cfg_file)
-        result = dict()
-        result['host'] = iniconfig.get(section, 'host', fallback='localhost')
-        result['port'] = iniconfig.get(section, 'port', fallback='5432')
-        result['database'] = iniconfig.get(
-            section, 'database', fallback='logicsem')
-        result['user'] = iniconfig.get(section, 'user', fallback='postgres')
-        result['password'] = iniconfig.get(section, 'password')
-        result['application_name'] = iniconfig.get(
-            section, 'application_name', fallback='dpdb-admin')
-        return {section: result}
-
-    # default behaviour
-    with open(cfg_file) as jsonfile:
-        return json.load(jsonfile)
-
-
-def config(filename='database.ini', section='postgresql') -> dict:
+def read_cfg(cfg_file, section, prefer_cfg=False) -> dict:
+    """Read the config file and return the result of one section."""
+    try:
+        file_content = read_yml_or_cfg(cfg_file, prefer_cfg=prefer_cfg)
+        content = dict(file_content[section])
+        LOGGER.debug(
+            "Found keys %s in %s[%s]", content.keys(), cfg_file, section)
+    except (OSError, AttributeError, TypeError) as err:
+        LOGGER.warning("Encountered %s while reading config '%s' section '%s'",
+                       err, cfg_file, section, exc_info=True)
+        content = {}
+    return content
+
+
+def db_config(filename='database.ini', section='postgresql') -> dict:
     """Return the database config as JSON"""
-    cfg = read_cfg(filename, section)
-    if section in cfg:
-        db_config = cfg[section]
-    else:
-        raise Exception(f'Section {section} not found in the {filename} file')
     LOGGER.info("Read db_config['%s'] from '%s'", section, filename)
-    return db_config
+    cfg = read_cfg(filename, section)
+    return {**DEFAULT_DBCONFIG, **cfg}
 
 
 class IDpdbVisuConstruct(metaclass=abc.ABCMeta):
@@ -439,11 +424,11 @@ def construct(self):
             'labeldict': labeldict,
             'num_vars': self.read_num_vars()}
 
-        generalGraph = {'edges': self.read_twfile()} if self.tw_file else False
+        general_gr = {'edges': self.read_twfile()} if self.tw_file else False
 
         timeline = self.read_timeline(edgearray)
         return {'incidenceGraph': False,
-                'generalGraph': generalGraph,
+                'generalGraph': general_gr,
                 'tdTimeline': timeline,
                 'treeDecJson': tree_dec_json}
 
@@ -454,7 +439,7 @@ def connect() -> pg.extensions.connection:
     conn = None
     try:
         # read connection parameters
-        params = config()
+        params = db_config(filename='Archive/database.json')
         db_name = params['database']
         LOGGER.info("Connecting to the PostgreSQL database '%s'...", db_name)
         conn = pg.connect(**params)
@@ -501,9 +486,55 @@ def create_json(problem: int, tw_file=None, intermed_nodes=False) -> dict:
     return {}
 
 
-if __name__ == "__main__":
+def main(args: argparse.Namespace) -> None:
+    """
+    Main method running construct_dpdb_visu for arguments in 'args'
 
-    import argparse
+    Parameters
+    ----------
+    args : argparse.Namespace
+        The namespace containing all (command-line) parameters.
+
+    Returns
+    -------
+    None
+    """
+
+    loglevel = None  # passed to the configuration of the root-logger
+    if args.loglevel is not None:
+        try:
+            loglevel = int(float(args.loglevel))
+        except ValueError:
+            loglevel = args.loglevel.upper()
+    logging.basicConfig(level=loglevel)  # Output logging for setup
+    logging_cfg(filename='logging.yml', loglevel=loglevel)
+    LOGGER.info("Called with '%s'", args)
+
+    problem_ = args.problemnumber
+    # get twfile if supplied
+    try:
+        tw_file_ = args.twfile
+    except AttributeError:
+        tw_file_ = None
+    result_json = create_json(problem=problem_, tw_file=tw_file_)
+    # build json filename, can be supplied with problem-number
+    try:
+        outfile = args.outfile % problem_
+    except TypeError:
+        outfile = args.outfile
+    LOGGER.info("Output file-name: %s", outfile)
+    with open(outfile, 'w') as file:
+        json.dump(
+            result_json,
+            file,
+            sort_keys=True,
+            indent=2 if args.pretty else None,
+            ensure_ascii=False)
+        LOGGER.debug("Wrote to %s", file)
+
+
+if __name__ == "__main__":
+    # Parse args, call main
 
     PARSER = argparse.ArgumentParser(
         description="""
@@ -514,14 +545,7 @@ def create_json(problem: int, tw_file=None, intermed_nodes=False) -> dict:
 
         Extracts Information from https://github.com/hmarkus/dp_on_dbs runs
         for further visualization.""",
-        epilog="""Logging levels for python 3.8.2:
-            CRITICAL: 50
-            ERROR:    40
-            WARNING:  30
-            INFO:     20
-            DEBUG:    10
-            NOTSET:    0 (will traverse the logging hierarchy until a value is found)
-            """,
+        epilog=LOGLEVEL_EPILOG,
         formatter_class=argparse.RawDescriptionHelpFormatter
     )
 
@@ -531,47 +555,17 @@ def create_json(problem: int, tw_file=None, intermed_nodes=False) -> dict:
                         type=argparse.FileType('r', encoding='UTF-8'),
                         help="tw-File containing the edges of the graph - "
                         "obtained from dpdb with option --gr-file GR_FILE.")
-    PARSER.add_argument('--loglevel', default='INFO', help="default:'INFO'")
-    PARSER.add_argument(
-        '--outfile',
-        default='dbjson%d.json',
-        help="default:'dbjson%%d.json'")
+    PARSER.add_argument('--loglevel', help="set the minimal loglevel for root")
+    PARSER.add_argument('--outfile', default='dbjson%d.json',
+                        help="default:'dbjson%%d.json'")
     PARSER.add_argument('--pretty', action='store_true',
                         help="Pretty-print the JSON.")
-    PARSER.add_argument(
-        '--inter-nodes',
-        action='store_true',
-        help="Calculate path between successive nodes during the evaluation order.")
+    PARSER.add_argument('--inter-nodes', action='store_true',
+                        help="Calculate path between successive nodes "
+                        "during the evaluation order.")
     PARSER.add_argument('--version', action='version',
                         version='%(prog)s ' + version + ', ' + __date__)
 
     # get cmd-arguments
-    args = PARSER.parse_args()
-    LOGGER.info('%s', args)
-    # get loglevel
-    try:
-        loglevel = int(float(args.loglevel))
-    except ValueError:
-        loglevel = args.loglevel.upper()
-    LOGGER.setLevel(loglevel)
-    problem_ = args.problemnumber
-    # get twfile if supplied
-    try:
-        tw_file_ = args.twfile
-    except AttributeError:
-        tw_file_ = None
-    RESULTJSON = create_json(problem=problem_, tw_file=tw_file_)
-    # build json filename, can be supplied with problem-number
-    try:
-        outfile = args.outfile % problem_
-    except TypeError:
-        outfile = args.outfile
-    LOGGER.info("Output file-name: %s", outfile)
-    with open(outfile, 'w') as file:
-        json.dump(
-            RESULTJSON,
-            file,
-            sort_keys=True,
-            indent=2 if args.pretty else None,
-            ensure_ascii=False)
-        LOGGER.debug("Wrote to %s", file)
+    _args = PARSER.parse_args()
+    main(_args)

tdvisu/dijkstra.py

@@ -47,6 +47,8 @@
 from heapq import heappush, heappop
 from itertools import count
 
+from tdvisu.utilities import convert_to_adj
+
 
 def bidirectional_dijkstra(edges, source, target, weight='weight'):
     r"""Dijkstra's algorithm for shortest paths using bidirectional search.
@@ -229,39 +231,6 @@ def _weight_function(weight, multigraph=False):
     return lambda u, v, data: data.get(weight, 1)
 
 
-def convert_to_adj(edgelist, directed=False):
-    """
-    Helper function to convert the edgelist into the adj-format from NetworkX.
-
-    Parameters
-    ----------
-    edgelist : array-like of pairs of vertices.
-        Simple edgelist. Example:
-            [(2, 1), (3, 2), (4, 2), (5, 4)]
-    directed : bool, optional
-        Whether to add the backward edges too. The default is False.
-
-    Returns
-    -------
-    adj : dict of edges with empty attributes
-        See Docs » Module code » networkx.classes.graph.adj(self)
-        for detailed structure.
-        Basically: dict of {source1:{target1:{'attr1':value,},},...}
-        https://networkx.github.io/documentation/networkx-2.1/_modules/networkx/classes/graph.html
-    """
-    adj = dict()
-    for (u, v) in edgelist:
-        if u not in adj:
-            adj[u] = {}
-        adj[u][v] = {}
-        if not directed:
-            # add reversed edge
-            if v not in adj:
-                adj[v] = {}
-            adj[v][u] = {}
-    return adj
-
-
 if __name__ == "__main__":
     # Show one example and print to console
     EDGELIST = [(2, 1), (3, 2), (4, 2), (5, 4)]

tdvisu/logging.ini

@@ -0,0 +1,52 @@
+[loggers]
+keys=root,visualization.py,svgjoin.py,reader.py,construct_dpdb_visu.py,utilities.py
+
+[handlers]
+keys=console
+
+[formatters]
+keys=simple,full
+
+[handler_console]
+class=StreamHandler
+level=WARNING
+formatter=full
+args=(sys.stdout,)
+
+[formatter_simple]
+format=%(asctime)s %(levelname)s %(message)s
+datefmt=%H:%M:%S
+
+[formatter_full]
+format=%(asctime)s,%(msecs)d %(levelname)s[%(filename)s:%(lineno)d] %(message)s
+datefmt=%Y-%m-%d %H:%M:%S
+
+[logger_root]
+level=WARNING
+handlers=console
+
+[logger_visualization.py]
+level=NOTSET
+qualname=visualization.py
+handlers=console
+
+[logger_svgjoin.py]
+level=NOTSET
+qualname=svgjoin.py
+handlers=console
+
+[logger_reader.py]
+level=NOTSET
+qualname=reader.py
+handlers=console
+
+[logger_construct_dpdb_visu.py]
+level=NOTSET
+qualname=construct_dpdb_visu.py
+handlers=console
+
+[logger_utilities.py]
+level=NOTSET
+qualname=utilities.py
+handlers=console
+