hide excessive numbers of resource_data log entries, ckan#187

- Default to showing the first 50 and last 50 entries, with links to display more - This dramatically improves page load times for very large resources
qld-gov-au · May 19, 2023 · 4111f3f · 4111f3f
1 parent c062f54
commit 4111f3f
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 9 deletions.
diff --git a/ckanext/xloader/templates/xloader/resource_data.html b/ckanext/xloader/templates/xloader/resource_data.html
@@ -61,7 +61,25 @@
   {% if status.status and status.task_info and show_table %}
     <h3>{{ _('Upload Log') }}</h3>
     <ul class="activity">
-      {% for item in status.task_info.logs %}
+      {% set items = status.task_info.logs %}
+      {% set rows = rows or 50 %}
+      {% set skipped_rows = (items | length) - (rows * 2) %}
+      {% if skipped_rows > 1 %}
+        <li class="item no-avatar">
+          <i class="fa icon fa-exclamation"></i>
+          <p>
+            {{ skipped_rows }} out of {{ items | length }} logs will be hidden.
+            <br>
+            <span class="date">
+              <a href="?rows={{rows * 2}}">Show more</a>&nbsp; &nbsp;<a href="?rows={{ items | length }}">Show all</a>
+            </span>
+          </p>
+        </li>
+      {% endif %}
+      {% for item in items %}
+        {# Truncate very long loops, showing just the start and end #}
+        {% if loop.index <= rows or loop.revindex <= rows
+           or (loop.index == rows + 1 and loop.revindex == rows + 1) %}
         {% set icon = 'ok' if item.level == 'INFO' else 'exclamation' %}
         {% set class = ' failure' if icon == 'exclamation' else ' success' %}
         {% set popover_content = 'test' %}
@@ -77,6 +95,18 @@ <h3>{{ _('Upload Log') }}</h3>
             </span>
           </p>
         </li>
+        {% elif loop.index == rows + 1 %}
+          <li class="item no-avatar">
+            <i class="fa icon fa-exclamation"></i>
+            <p>
+              Skipping {{ skipped_rows }} logs...
+              <br>
+              <span class="date">
+                <a href="?rows={{rows * 2}}">Show more</a>&nbsp; &nbsp;<a href="?rows={{ items | length }}">Show all</a>
+              </span>
+            </p>
+          </li>
+        {% endif %}
       {% endfor %}
       <li class="item no-avatar">
         <i class="fa icon fa-info"></i>

diff --git a/ckanext/xloader/utils.py b/ckanext/xloader/utils.py
@@ -11,7 +11,7 @@
 import ckan.plugins as p
 
 
-def resource_data(id, resource_id):
+def resource_data(id, resource_id, rows=None):
 
     if p.toolkit.request.method == "POST":
         try:
@@ -44,13 +44,16 @@ def resource_data(id, resource_id):
     except p.toolkit.NotAuthorized:
         return p.toolkit.abort(403, p.toolkit._("Not authorized to see this page"))
 
+    extra_vars = {
+        "status": xloader_status,
+        "resource": resource,
+        "pkg_dict": pkg_dict,
+    }
+    if rows:
+        extra_vars["rows"] = rows
     return p.toolkit.render(
         "xloader/resource_data.html",
-        extra_vars={
-            "status": xloader_status,
-            "resource": resource,
-            "pkg_dict": pkg_dict,
-        },
+        extra_vars=extra_vars,
     )
 
 

diff --git a/ckanext/xloader/views.py b/ckanext/xloader/views.py
@@ -1,4 +1,4 @@
-from flask import Blueprint
+from flask import Blueprint, request
 
 import ckanext.xloader.utils as utils
 
@@ -12,4 +12,12 @@ def get_blueprints():
 
 @xloader.route("/dataset/<id>/resource_data/<resource_id>", methods=("GET", "POST"))
 def resource_data(id, resource_id):
-    return utils.resource_data(id, resource_id)
+    rows = request.args.get('rows')
+    if rows:
+        try:
+            rows = int(rows)
+            if rows < 0:
+                rows = None
+        except ValueError:
+            rows = None
+    return utils.resource_data(id, resource_id, rows)