Cloud mirror overview page (#2176)

This adds a page to the console which gives an overview of the active
*cloud mirrors* (Google Cloud Storage and Amazon S3).

Currently this is just a table so that we can easily see which projects
are uploaded and which aren't. Since this stuff has to be managed by
hand I'd like to have an easy way to see the status of all projects (and
grouped by open vs restricted.) Eventually it might be nice to add
management functions to this page as well.

In order to view this page, you will need to have the
`project.change_publishedproject` permission (the same permission as for
manage_published_project.) I think that there should be a separate
permission for managing mirrors, but this will work for now.

The implementation is more fragile than I'd like, in a couple of ways.

- I want to retrieve the GCP/AWS information in *one SQL query*, and
furthermore want to select only the relevant fields. Using only() and
select_related() lets us do that (and you can verify for yourself: this
code is only making a single SQL query). But this could easily be broken
by code changes elsewhere, and suddenly this page would become much
slower to load.

- I don't just want to display whether or not file access is allowed, I
want to indicate the reason (deprecated, forbidden, embargoed). There's
no way to do this via `project.authorization.access` and I can't
immediately think of a clean way to do so.

That said, I think this code works and should be useful.

physionet-django/console/navbar.py

@@ -154,6 +154,10 @@ def get_menu_items(self, request):
 
     NavLink(_('Storage'), 'storage_requests', 'cube'),
 
+    NavSubmenu(_('Cloud'), 'cloud', 'cloud', [
+        NavLink(_('Mirrors'), 'cloud_mirrors'),
+    ]),
+
     NavSubmenu(_('Identity check'), 'identity', 'hand-paper', [
         NavLink(_('Processing'), 'credential_processing'),
         NavLink(_('All Applications'), 'credential_applications',

physionet-django/console/static/console/css/cloud-mirrors.css

@@ -0,0 +1,91 @@
+.table-cloud-status {
+    table-layout: fixed;
+    width: 100%;
+}
+
+.table-cloud-status .col-project-version {
+    width: 7rem;
+}
+.table-cloud-status .col-project-site-status,
+.table-cloud-status .col-project-cloud-status {
+    width: 3rem;
+    text-align: center;
+    overflow-x: hidden;
+}
+
+.project-site-status-title,
+.project-cloud-status-title {
+    font-size: 0;
+}
+.project-site-status-title::before,
+.project-cloud-status-title::before {
+    font-size: 1rem;
+    display: inline-block;
+    width: 1.25em;
+    font-family: "Font Awesome 5 Free";
+    font-weight: 900;
+}
+.project-site-status-title::before {
+    content: "\f019";           /* download */
+}
+.project-cloud-status-title::before {
+    content: "\f381";           /* cloud-download-alt */
+}
+.col-gcp .project-cloud-status-title::before {
+    font-family: "Font Awesome 5 Brands";
+    font-weight: 400;
+    content: "\f1a0";           /* google */
+}
+.col-aws .project-cloud-status-title::before {
+    font-family: "Font Awesome 5 Brands";
+    font-weight: 400;
+    content: "\f375";           /* aws */
+}
+
+.project-site-status-open,
+.project-site-status-restricted,
+.project-site-status-embargo,
+.project-site-status-forbidden,
+.project-cloud-status-public,
+.project-cloud-status-private,
+.project-cloud-status-pending,
+.project-cloud-status-none {
+    font-size: 0;
+}
+.project-site-status-open::before,
+.project-site-status-restricted::before,
+.project-site-status-embargo::before,
+.project-site-status-forbidden::before,
+.project-cloud-status-public::before,
+.project-cloud-status-private::before,
+.project-cloud-status-pending::before,
+.project-cloud-status-none::before {
+    font-size: 1rem;
+    display: inline-block;
+    width: 1.25em;
+    font-family: "Font Awesome 5 Free";
+    font-weight: 900;
+}
+.project-site-status-open::before,
+.project-cloud-status-public::before {
+    content: "\f058";           /* check-circle */
+    color: #0a0;
+}
+.project-site-status-restricted::before,
+.project-cloud-status-private::before {
+    content: "\f2bd";           /* user-circle */
+    color: #50f;
+}
+.project-site-status-embargo::before {
+    content: "\f28b";           /* pause-circle */
+    color: #fa0;
+}
+.project-site-status-forbidden::before {
+    content: "\f057";           /* times-circle */
+    color: #c00;
+}
+.project-cloud-status-pending::before {
+    content: "\f017";           /* clock */
+    font-weight: 400;
+    color: #888;
+}

physionet-django/console/templates/console/cloud_mirrors.html

@@ -0,0 +1,96 @@
+{% extends "console/base_console.html" %}
+
+{% load static %}
+
+{% block title %}Cloud Mirrors{% endblock %}
+
+{% block local_css %}
+  <link rel="stylesheet" type="text/css" href="{% static 'console/css/cloud-mirrors.css' %}"/>
+{% endblock %}
+
+{% block content %}
+  <div class="card">
+    <div class="card-header">
+      <ul class="nav nav-tabs card-header-tabs">
+        <li class="nav-item">
+          <a class="nav-link {% if group == 'open' %}active{% endif %}"
+             href="?group=open">Open</a>
+        </li>
+        <li class="nav-item">
+          <a class="nav-link {% if group == 'restricted' %}active{% endif %}"
+             href="?group=restricted">Restricted</a>
+        </li>
+      </ul>
+    </div>
+    <div class="card-body">
+      <table class="table table-cloud-status table-bordered">
+        <thead>
+          <tr>
+            <th class="col-project-title">Project</th>
+            <th class="col-project-version">Version</th>
+            <th class="col-project-site-status">
+              <span class="project-site-status-title"
+                    title="File acccess via {{ SITE_NAME }}">
+                {{ SITE_NAME }}
+              </span>
+            </th>
+            {% for platform in cloud_platforms %}
+              <th class="col-project-cloud-status col-{{ platform.id }}">
+                <span class="project-cloud-status-title"
+                      title="File access via {{ platform.long_name }}">
+                  {{ platform.name }}
+                </span>
+              </th>
+            {% endfor %}
+          </tr>
+        </thead>
+        <tbody>
+          {% for project, mirrors in project_mirrors.items %}
+            <tr>
+              <td class="col-project-title">
+                <a href="{% url 'manage_published_project' project.slug project.version %}">
+                  {{ project.title }}
+                </a>
+              </td>
+              <td class="col-project-version">{{ project.version }}</td>
+              <td class="col-project-site-status">
+                {% if project.deprecated_files %}
+                  <span class="project-site-status-forbidden"
+                        title="Access is deprecated">Deprecated</span>
+                {% elif not project.allow_file_downloads %}
+                  <span class="project-site-status-forbidden"
+                        title="Access is forbidden">Forbidden</span>
+                {% elif project.embargo_active %}
+                  <span class="project-site-status-embargo"
+                        title="Under embargo">Embargo</span>
+                {% elif project.access_policy != AccessPolicy.OPEN %}
+                  <span class="project-site-status-restricted"
+                        title="Restricted by access policy">Restricted</span>
+                {% else %}
+                  <span class="project-site-status-open"
+                        title="Open to the public">Open</span>
+                {% endif %}
+              </td>
+              {% for platform_mirror in mirrors %}
+                <td class="col-project-cloud-status">
+                  {% if not platform_mirror %}
+                    <span class="project-cloud-status-none"></span>
+                  {% elif not platform_mirror.sent_files %}
+                    <span class="project-cloud-status-pending"
+                          title="Upload pending">Pending</span>
+                  {% elif platform_mirror.is_private %}
+                    <span class="project-cloud-status-private"
+                          title="Private">Private</span>
+                  {% else %}
+                    <span class="project-cloud-status-public"
+                          title="Public">Public</span>
+                  {% endif %}
+                </td>
+              {% endfor %}
+            </tr>
+          {% endfor %}
+        </tbody>
+      </table>
+    </div>
+  </div>
+{% endblock %}

physionet-django/console/urls.py

@@ -19,6 +19,8 @@
     path('published-projects/<project_slug>/<version>/',
         views.manage_published_project, name='manage_published_project'),
     path('data-access-request/<int:pk>/', views.access_request, name='access_request'),
+    path('cloud/mirrors/', views.cloud_mirrors,
+         name='cloud_mirrors'),
 
     # Logs
     path('data-access-logs/', views.project_access_requests_list, name='project_access_requests_list'),

physionet-django/console/views.py

@@ -1145,6 +1145,61 @@ def aws_bucket_management(request, project, user):
     send_files_to_aws(project.id, verbose_name='AWS - {}'.format(project), creator=user)
 
 
+@console_permission_required('project.change_publishedproject')
+def cloud_mirrors(request):
+    """
+    Page for viewing the status of cloud mirrors.
+    """
+    projects = PublishedProject.objects.order_by('-publish_datetime')
+
+    group = request.GET.get('group', 'open')
+    if group == 'open':
+        projects = projects.filter(access_policy=AccessPolicy.OPEN)
+    else:
+        projects = projects.exclude(access_policy=AccessPolicy.OPEN)
+
+    cloud_platforms = []
+    if settings.GOOGLE_APPLICATION_CREDENTIALS:
+        cloud_platforms.append({
+            'field_name': 'gcp',
+            'name': 'GCP',
+            'long_name': 'Google Cloud Platform',
+        })
+    if has_s3_credentials():
+        cloud_platforms.append({
+            'field_name': 'aws',
+            'name': 'AWS',
+            'long_name': 'Amazon Web Services',
+        })
+
+    # Relevant fields for the status table (see
+    # templates/console/cloud_mirrors.html)
+    field_names = [platform['field_name'] for platform in cloud_platforms]
+    projects = projects.select_related(*field_names).only(
+        'slug',
+        'title',
+        'version',
+        'access_policy',
+        'allow_file_downloads',
+        'deprecated_files',
+        'embargo_files_days',
+        *(f'{field}__is_private' for field in field_names),
+        *(f'{field}__sent_files' for field in field_names),
+    )
+
+    project_mirrors = {
+        project: [
+            getattr(project, field, None) for field in field_names
+        ] for project in projects
+    }
+
+    return render(request, 'console/cloud_mirrors.html', {
+        'group': group,
+        'cloud_platforms': cloud_platforms,
+        'project_mirrors': project_mirrors,
+    })
+
+
 @console_permission_required('project.change_activeproject')
 def archived_submissions(request):
     """

physionet-django/project/modelcomponents/storage.py

@@ -42,6 +42,9 @@ class GCP(models.Model):
     class Meta:
         default_permissions = ()
 
+    def __str__(self):
+        return self.bucket_name
+
 
 class AWS(models.Model):
     """
@@ -65,3 +68,6 @@ def s3_uri(self):
             return f's3://{self.bucket_name}/{self.project.version}/'
         else:
             return f's3://{self.bucket_name}/{self.project.slug}/{self.project.version}/'
+
+    def __str__(self):
+        return self.s3_uri()