From a0eafdf380f8b693f045ebb1e6f6b943ecf71727 Mon Sep 17 00:00:00 2001
From: Malte Tashiro <malte@iij.ad.jp>
Date: Tue, 3 Sep 2024 02:23:50 +0000
Subject: [PATCH] Do not print all crawler names in case of errors.

Also move the error notification to the end so that we can print the
exceptions in a nicer format.
---
 autodeploy/autodeploy.py |  2 +-
 create_db.py             | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/autodeploy/autodeploy.py b/autodeploy/autodeploy.py
index 68d82c1..b1decba 100644
--- a/autodeploy/autodeploy.py
+++ b/autodeploy/autodeploy.py
@@ -115,7 +115,7 @@ def check_log(config: dict, date: datetime):
         return False
     body = r.content
     last_line = body.decode().split('\n')[-1]
-    if 'Errors:' in last_line:
+    if 'errors' in last_line:
         logging.error(f'There were errors from create_db found in logs for {log_url}')
         sys.exit(1)
     return True
diff --git a/create_db.py b/create_db.py
index 6b62403..8910274 100644
--- a/create_db.py
+++ b/create_db.py
@@ -15,6 +15,8 @@
 
 NEO4J_VERSION = '5.16.0'
 
+STATUS_OK = 'OK'
+
 
 def main():
     parser = argparse.ArgumentParser()
@@ -131,7 +133,7 @@ def __init__(self, message):
             if not passed:
                 error_message = f'Did not receive data from crawler {name}'
                 raise RelationCountError(error_message)
-            status[module_name] = 'OK'
+            status[module_name] = STATUS_OK
             logging.info(f'end {module}')
         except RelationCountError as relation_count_error:
             no_error = False
@@ -155,7 +157,7 @@ def __init__(self, message):
             post = module.PostProcess()
             post.run()
             post.close()
-            status[module_name] = 'OK'
+            status[module_name] = STATUS_OK
             logging.info(f'end {module}')
 
         except Exception as e:
@@ -200,14 +202,17 @@ def __init__(self, message):
         # TODO send an email
 
         # Add the log line to indicate to autodeploy that there were errors
-        final_words = f'\nErrors: {" ".join((k for k in status))}'
-        logging.error('There were errors!')
+        final_words = '\nErrors: '
+        for module, status in status.items():
+            if status != STATUS_OK:
+                final_words += f'\n{module}: {status}'
     else:
         final_words = 'No error :)'
     # Delete tmp file in cron job
     #    shutil.rmtree(tmp_dir)
 
     logging.info(f'Finished: {sys.argv} {final_words}')
+    logging.error('There were errors!')
 
     if args.archive:
         # Push the dump and log to ihr archive