From 079da4475eeeafe2b2edbeb1c753c54909bdb80e Mon Sep 17 00:00:00 2001
From: chrystinne <chrystinne@gmail.com>
Date: Thu, 19 Oct 2023 16:30:53 -0400
Subject: [PATCH] Configure server access logging for an S3 bucket. This
 enables access logging, directing logs to the specified target bucket and
 prefix.

---
 .env.example                                  |   3 +
 .../console/manage_published_project.html     |   2 +-
 physionet-django/console/views.py             |  10 +-
 physionet-django/physionet/settings/base.py   |   4 +
 physionet-django/project/cloud/s3.py          | 498 ++++++++++--------
 5 files changed, 288 insertions(+), 229 deletions(-)

diff --git a/.env.example b/.env.example
index 3ebac56023..12c08093a8 100644
--- a/.env.example
+++ b/.env.example
@@ -62,11 +62,14 @@ AWS_CLOUD_FORMATION=url
 
 # AWS credentials (Access Key and Secret Key): Configure AWS credentials in the AWS CLI profile using the 'aws configure' command.
 AWS_PROFILE=
+AWS_ACCOUNT_ID=
 # Path to the file containing credentials for AWS
 # (https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#shared-credentials-file)
 AWS_SHARED_CREDENTIALS_FILE=
 # The default bucket name to store projects with an 'OPEN' access policy.
 S3_OPEN_ACCESS_BUCKET=
+# The default bucket name to store logs and metrics related to project usage.
+S3_SERVER_ACCESS_LOGS=
 
 # Datacite
 # Used to assign the DOIs
diff --git a/physionet-django/console/templates/console/manage_published_project.html b/physionet-django/console/templates/console/manage_published_project.html
index 7700cdda73..fda1899ba0 100644
--- a/physionet-django/console/templates/console/manage_published_project.html
+++ b/physionet-django/console/templates/console/manage_published_project.html
@@ -417,7 +417,7 @@ <h5 class="card-title mt-3 mb-1">AWS</h5>
             are set.)
           </p>
         {% elif not project.aws.bucket_name %}
-          <p>Create a bucket on AWS to store the files associated with this project. dsjkbjak {{project.id}} {{project.aws.bucket_name}} {{project.aws.sent_files}}</p>
+          <p>Create a bucket on AWS to store the files associated with this project.</p>
           <form action="" method="post">
             {% csrf_token %}
             <button class="btn btn-primary" name="aws-bucket" value='{{project.slug}}'
diff --git a/physionet-django/console/views.py b/physionet-django/console/views.py
index 0e17841566..b21d08de02 100644
--- a/physionet-django/console/views.py
+++ b/physionet-django/console/views.py
@@ -797,13 +797,13 @@ def send_files_to_aws(pid):
     """
     project = PublishedProject.objects.get(id=pid)
     # Create or get the associated AWS object
-    aws_instance = AWS.objects.get_or_create(project=project)
+    AWS.objects.get_or_create(project=project)
     upload_project_to_S3(project)
-    aws_instance.sent_files = True
-    aws_instance.finished_datetime = timezone.now()
+    project.aws.sent_files = True
+    project.aws.finished_datetime = timezone.now()
     if project.compressed_storage_size:
-        aws_instance.sent_zip = True
-    aws_instance.save()
+        project.aws.sent_zip = True
+    project.aws.save()
 
 
 @associated_task(PublishedProject, "pid", read_only=True)
diff --git a/physionet-django/physionet/settings/base.py b/physionet-django/physionet/settings/base.py
index 134afc5d37..ba154e735a 100644
--- a/physionet-django/physionet/settings/base.py
+++ b/physionet-django/physionet/settings/base.py
@@ -233,10 +233,14 @@
     AWS_SHARED_CREDENTIALS_FILE = None
 
 AWS_PROFILE = config('AWS_PROFILE', default=False)
+AWS_ACCOUNT_ID = config('AWS_ACCOUNT_ID', default=False)
 
 # Bucket name for the S3 bucket containing the open access data
 S3_OPEN_ACCESS_BUCKET = config('S3_OPEN_ACCESS_BUCKET', default=False)
 
+# Bucket name to store logs and metrics related to project usage.
+S3_SERVER_ACCESS_LOGS = config('S3_SERVER_ACCESS_LOGS', default=False)
+
 # Header tags for the AWS lambda function that grants access to S3 storage
 AWS_HEADER_KEY = config('AWS_KEY', default=False)
 AWS_HEADER_VALUE = config('AWS_VALUE', default=False)
diff --git a/physionet-django/project/cloud/s3.py b/physionet-django/project/cloud/s3.py
index 165195d6d9..5be0b68936 100644
--- a/physionet-django/project/cloud/s3.py
+++ b/physionet-django/project/cloud/s3.py
@@ -30,8 +30,10 @@ def has_s3_credentials():
         bool: True if AWS_PROFILE is set, False otherwise.
     """
     return all([
+        settings.AWS_ACCOUNT_ID,
         settings.AWS_PROFILE,
         settings.S3_OPEN_ACCESS_BUCKET,
+        settings.S3_SERVER_ACCESS_LOGS,
     ])
 
 
@@ -225,7 +227,7 @@ def get_prefix_open_project(project):
     if project.access_policy != AccessPolicy.OPEN:
         return None
     else:
-        target_prefix = project.slug + "-" + project.version + "/"
+        target_prefix = project.slug + "/"
         matching_prefix = find_matching_prefix(get_all_prefixes(project), target_prefix)
     return matching_prefix
 
@@ -370,6 +372,43 @@ def create_s3_bucket(s3, bucket_name):
     s3.create_bucket(Bucket=bucket_name)
 
 
+def put_bucket_logging(s3, bucket_name, target_bucket, target_prefix):
+    """
+    Configures server access logging for an Amazon S3 bucket.
+
+    Args:
+        s3 (boto3.client): The Amazon S3 client.
+        bucket_name (str): The name of the source bucket.
+        target_bucket (str): The name of the bucket where log
+        files will be stored.
+        target_prefix (str): The prefix for log file names within the
+        target bucket.
+
+    Returns:
+        None
+
+    Note:
+        This method utilizes the `put_bucket_logging` operation to enable
+        server access logging for the specified source bucket, directing
+        logs to the specified target bucket and prefix.
+
+    Example:
+        put_bucket_logging(s3_client, 'source_bucket', 'log_bucket', 'logs/')
+    """
+    logging_config = {
+        'LoggingEnabled': {
+            'TargetBucket': target_bucket,
+            'TargetPrefix': target_prefix,
+        },
+    }
+
+    # Enable bucket logging
+    s3.put_bucket_logging(
+        Bucket=bucket_name,
+        BucketLoggingStatus=logging_config
+    )
+
+
 def send_files_to_s3(folder_path, s3_prefix, bucket_name):
     """
     Upload files from a local folder to an AWS S3 bucket with
@@ -454,9 +493,16 @@ def get_aws_accounts_for_dataset(dataset_name):
     return aws_accounts
 
 
-def get_initial_bucket_policy(bucket_name):
+def get_initial_bucket_policy(bucket_name, aws_account_id):
     """
     Create an initial bucket policy for an AWS S3 bucket.
+
+    Args:
+        bucket_name (str): The name of the S3 bucket.
+        aws_account_id (str): The AWS account ID to allow access.
+
+    Returns:
+        str: The JSON string representing the bucket policy.
     """
     bucket_policy = {
         "Version": "2012-10-17",
@@ -471,7 +517,9 @@ def get_initial_bucket_policy(bucket_name):
                     f"arn:aws:s3:::{bucket_name}/*",
                 ],
                 "Condition": {
-                    "StringNotLike": {"aws:PrincipalArn": "arn:aws:iam::724665945834:*"}
+                    "StringNotLike": {
+                        "aws:PrincipalArn": f"arn:aws:iam::{aws_account_id}:*"
+                    }
                 },
             }
         ],
@@ -667,7 +715,7 @@ def update_bucket_policy(project, bucket_name):
         or project.access_policy == AccessPolicy.CONTRIBUTOR_REVIEW
     ):
         if aws_ids == []:
-            bucket_policy = get_initial_bucket_policy(bucket_name)
+            bucket_policy = get_initial_bucket_policy(bucket_name, settings.AWS_ACCOUNT_ID)
         else:
             bucket_policy = create_bucket_policy(bucket_name, aws_ids, False)
 
@@ -724,6 +772,10 @@ def upload_project_to_S3(project):
         s3_prefix = f"{project.slug}/"
     send_files_to_s3(folder_path, s3_prefix, bucket_name)
 
+    put_bucket_logging(
+        s3, bucket_name, settings.S3_SERVER_ACCESS_LOGS, get_prefix_open_project(project) + "/logs/"
+    )
+
 
 def upload_list_of_projects(projects):
     """
@@ -776,222 +828,222 @@ def upload_all_projects():
         upload_project_to_S3(project)
 
 
-def empty_s3_bucket(bucket):
-    """
-    Delete all objects (files) within an AWS S3 bucket.
-
-    This function iterates through all objects within the specified
-    'bucket' and deletes each object, effectively emptying
-    the bucket of its contents.
-
-    Args:
-        bucket (boto3.resources.factory.s3.Bucket): The AWS S3
-        bucket to empty.
-
-    Returns:
-        None
-
-    Note:
-    - Ensure that AWS credentials (Access Key and Secret Key) with
-    sufficient permissions are properly configured for the S3
-    client used in this function.
-    """
-    for key in bucket.objects.all():
-        key.delete()
-
-
-def empty_project_from_S3(project):
-    """
-    Empty an AWS S3 bucket associated with a project if
-    AWS_PROFILE is not None.
-
-    This function removes all files within the AWS S3 bucket
-    associated with the specified 'project', effectively emptying
-    the bucket. It leverages the 'empty_s3_bucket' function
-    for this purpose.
-
-    Args:
-        project (project.models.Project): The project for
-        which to empty the associated AWS S3 bucket.
-
-    Returns:
-        None
-
-    Note:
-    - The function ensures that the AWS S3 bucket remains intact
-    but contains no files after execution.
-    - Ensure that AWS credentials (Access Key and Secret Key)
-    with sufficient permissions are properly configured for the
-    S3 client used in this function.
-    """
-    if not check_s3_bucket_exists(project):
-        return
-    s3 = create_s3_resource()
-    bucket_name = get_bucket_name(project)
-    bucket = s3.Bucket(bucket_name)
-    empty_s3_bucket(bucket)
-
-
-def empty_list_of_projects(projects):
-    """
-    Bulk empty AWS S3 buckets associated with a list of projects.
-
-    This function takes a list of 'projects' and empties the
-    AWS S3 buckets associated with each project, effectively
-    removing all files within them. It delegates the emptying
-    process to the 'empty_project_from_S3' function.
-
-    Args:
-        projects (list of project.models.Project): A list of
-        projects for which to empty the associated AWS S3 buckets.
-
-    Returns:
-        None
-
-    Note:
-    - The function ensures that AWS S3 buckets remain intact
-    but contain no files after execution.
-    - Ensure that AWS credentials (Access Key and Secret Key)
-    with sufficient permissions are properly configured for
-    the S3 client used in this function.
-    """
-    for project in projects:
-        empty_project_from_S3(project)
-
-
-def empty_all_projects_from_S3():
-    """
-    Bulk empty all AWS S3 buckets associated with published
-    projects.
-
-    This function iterates through all published projects in the
-    database and empties the AWS S3 buckets associated with each
-    project, effectively removing all files within them. It
-    leverages the 'empty_project_from_S3' function for this purpose.
-
-    Args:
-        None
-
-    Returns:
-        None
-
-    Note:
-    - The function ensures that AWS S3 buckets remain intact
-    but contain no files after execution.
-    - Ensure that AWS credentials (Access Key and Secret Key)
-    with sufficient permissions are properly configured for the
-    S3 client used in this function.
-    """
-
-    # Retrieve all published projects from the database
-    published_projects = PublishedProject.objects.all()
-    # Empty the AWS S3 buckets associated with each published project
-    for project in published_projects:
-        empty_project_from_S3(project)
-
-
-def delete_project_from_S3(project):
-    """
-    Delete a project's files and associated AWS S3 bucket
-    if AWS_PROFILE is not None.
-
-    This function cleans up and removes all files associated
-    with the specified 'project' from an AWS S3 bucket.
-    It then deletes the empty S3 bucket itself.
-
-    Args:
-        project (project.models.Project): The project for which
-        to delete files and the associated bucket.
-
-    Returns:
-        None
-
-    Note:
-    - The function leverages the 'delete_s3_bucket' method
-    to perform the deletion.
-    - Ensure that AWS credentials (Access Key and Secret Key)
-    with sufficient permissions are properly configured for
-    the S3 client used in this function.
-    """
-    if not check_s3_bucket_exists(project):
-        return
-    s3 = create_s3_resource()
-    bucket_name = get_bucket_name(project)
-    bucket = s3.Bucket(bucket_name)
-    # Empty the specified AWS S3 'bucket'
-    # by deleting all objects (files) within it
-    empty_s3_bucket(bucket)
-    # Delete the empty bucket itself
-    bucket.delete()
-
-
-def delete_list_of_projects_from_s3(projects):
-    """ "
-    Bulk delete a list of projects from AWS S3.
-
-    This function deletes a list of projects, including
-    their files and associated AWS S3 buckets.
-    If the list includes open projects, the
-    'delete_project_from_S3' function will be called only
-    once for the first open project encountered.
-
-    Args:
-        projects (list of project.models.Project): A list of
-        projects to delete from AWS S3.
-
-    Returns:
-        None
-
-    Note:
-    - The function leverages the 'delete_project_from_S3' method
-    to perform project deletions.
-    - Ensure that AWS credentials (Access Key and Secret Key)
-    with sufficient permissions are properly configured for
-    the S3 client used in this function.
-    """
-    for project in projects:
-        delete_project_from_S3(project)
-        if project.access_policy == AccessPolicy.OPEN:
-            break
-
-
-def delete_all_projects_from_S3():
-    """
-    Bulk delete all published projects from AWS S3.
-
-    This function deletes all published projects, including
-    their files and associated AWS S3 buckets.
-    It distinguishes between open and non-open projects,
-    calling the 'delete_project_from_S3' function
-    accordingly. Non-open projects, such as 'RESTRICTED',
-    'CREDENTIALED', or 'CONTRIBUTOR_REVIEW', are
-    deleted individually. Open projects are stored in the same
-    S3 bucket, so only one delete operation is needed, which is
-    performed by passing the first open project.
-
-    Args:
-        None
-
-    Returns:
-        None
-
-    Note:
-    - The function leverages the 'delete_project_from_S3' method
-    to perform project deletions.
-    - Ensure that AWS credentials (Access Key and Secret Key)
-    with sufficient permissions are properly configured for the
-    S3 client used in this function.
-    """
-    not_open_published_projects = (
-        PublishedProject.objects.filter(access_policy=AccessPolicy.RESTRICTED)
-        | PublishedProject.objects.filter(access_policy=AccessPolicy.CREDENTIALED)
-        | PublishedProject.objects.filter(access_policy=AccessPolicy.CONTRIBUTOR_REVIEW)
-    )
-    for project in not_open_published_projects:
-        delete_project_from_S3(project)
-    # since all open projects are stored in the same bucket, we only
-    # need to delete the bucket once by passing
-    # the first open project
-    delete_project_from_S3(
-        PublishedProject.objects.filter(access_policy=AccessPolicy.OPEN)[0]
-    )
+# def empty_s3_bucket(bucket):
+#     """
+#     Delete all objects (files) within an AWS S3 bucket.
+
+#     This function iterates through all objects within the specified
+#     'bucket' and deletes each object, effectively emptying
+#     the bucket of its contents.
+
+#     Args:
+#         bucket (boto3.resources.factory.s3.Bucket): The AWS S3
+#         bucket to empty.
+
+#     Returns:
+#         None
+
+#     Note:
+#     - Ensure that AWS credentials (Access Key and Secret Key) with
+#     sufficient permissions are properly configured for the S3
+#     client used in this function.
+#     """
+#     for key in bucket.objects.all():
+#         key.delete()
+
+
+# def empty_project_from_S3(project):
+#     """
+#     Empty an AWS S3 bucket associated with a project if
+#     AWS_PROFILE is not None.
+
+#     This function removes all files within the AWS S3 bucket
+#     associated with the specified 'project', effectively emptying
+#     the bucket. It leverages the 'empty_s3_bucket' function
+#     for this purpose.
+
+#     Args:
+#         project (project.models.Project): The project for
+#         which to empty the associated AWS S3 bucket.
+
+#     Returns:
+#         None
+
+#     Note:
+#     - The function ensures that the AWS S3 bucket remains intact
+#     but contains no files after execution.
+#     - Ensure that AWS credentials (Access Key and Secret Key)
+#     with sufficient permissions are properly configured for the
+#     S3 client used in this function.
+#     """
+#     if not check_s3_bucket_exists(project):
+#         return
+#     s3 = create_s3_resource()
+#     bucket_name = get_bucket_name(project)
+#     bucket = s3.Bucket(bucket_name)
+#     empty_s3_bucket(bucket)
+
+
+# def empty_list_of_projects(projects):
+#     """
+#     Bulk empty AWS S3 buckets associated with a list of projects.
+
+#     This function takes a list of 'projects' and empties the
+#     AWS S3 buckets associated with each project, effectively
+#     removing all files within them. It delegates the emptying
+#     process to the 'empty_project_from_S3' function.
+
+#     Args:
+#         projects (list of project.models.Project): A list of
+#         projects for which to empty the associated AWS S3 buckets.
+
+#     Returns:
+#         None
+
+#     Note:
+#     - The function ensures that AWS S3 buckets remain intact
+#     but contain no files after execution.
+#     - Ensure that AWS credentials (Access Key and Secret Key)
+#     with sufficient permissions are properly configured for
+#     the S3 client used in this function.
+#     """
+#     for project in projects:
+#         empty_project_from_S3(project)
+
+
+# def empty_all_projects_from_S3():
+#     """
+#     Bulk empty all AWS S3 buckets associated with published
+#     projects.
+
+#     This function iterates through all published projects in the
+#     database and empties the AWS S3 buckets associated with each
+#     project, effectively removing all files within them. It
+#     leverages the 'empty_project_from_S3' function for this purpose.
+
+#     Args:
+#         None
+
+#     Returns:
+#         None
+
+#     Note:
+#     - The function ensures that AWS S3 buckets remain intact
+#     but contain no files after execution.
+#     - Ensure that AWS credentials (Access Key and Secret Key)
+#     with sufficient permissions are properly configured for the
+#     S3 client used in this function.
+#     """
+
+#     # Retrieve all published projects from the database
+#     published_projects = PublishedProject.objects.all()
+#     # Empty the AWS S3 buckets associated with each published project
+#     for project in published_projects:
+#         empty_project_from_S3(project)
+
+
+# def delete_project_from_S3(project):
+#     """
+#     Delete a project's files and associated AWS S3 bucket
+#     if AWS_PROFILE is not None.
+
+#     This function cleans up and removes all files associated
+#     with the specified 'project' from an AWS S3 bucket.
+#     It then deletes the empty S3 bucket itself.
+
+#     Args:
+#         project (project.models.Project): The project for which
+#         to delete files and the associated bucket.
+
+#     Returns:
+#         None
+
+#     Note:
+#     - The function leverages the 'delete_s3_bucket' method
+#     to perform the deletion.
+#     - Ensure that AWS credentials (Access Key and Secret Key)
+#     with sufficient permissions are properly configured for
+#     the S3 client used in this function.
+#     """
+#     if not check_s3_bucket_exists(project):
+#         return
+#     s3 = create_s3_resource()
+#     bucket_name = get_bucket_name(project)
+#     bucket = s3.Bucket(bucket_name)
+#     # Empty the specified AWS S3 'bucket'
+#     # by deleting all objects (files) within it
+#     empty_s3_bucket(bucket)
+#     # Delete the empty bucket itself
+#     bucket.delete()
+
+
+# def delete_list_of_projects_from_s3(projects):
+#     """ "
+#     Bulk delete a list of projects from AWS S3.
+
+#     This function deletes a list of projects, including
+#     their files and associated AWS S3 buckets.
+#     If the list includes open projects, the
+#     'delete_project_from_S3' function will be called only
+#     once for the first open project encountered.
+
+#     Args:
+#         projects (list of project.models.Project): A list of
+#         projects to delete from AWS S3.
+
+#     Returns:
+#         None
+
+#     Note:
+#     - The function leverages the 'delete_project_from_S3' method
+#     to perform project deletions.
+#     - Ensure that AWS credentials (Access Key and Secret Key)
+#     with sufficient permissions are properly configured for
+#     the S3 client used in this function.
+#     """
+#     for project in projects:
+#         delete_project_from_S3(project)
+#         if project.access_policy == AccessPolicy.OPEN:
+#             break
+
+
+# def delete_all_projects_from_S3():
+#     """
+#     Bulk delete all published projects from AWS S3.
+
+#     This function deletes all published projects, including
+#     their files and associated AWS S3 buckets.
+#     It distinguishes between open and non-open projects,
+#     calling the 'delete_project_from_S3' function
+#     accordingly. Non-open projects, such as 'RESTRICTED',
+#     'CREDENTIALED', or 'CONTRIBUTOR_REVIEW', are
+#     deleted individually. Open projects are stored in the same
+#     S3 bucket, so only one delete operation is needed, which is
+#     performed by passing the first open project.
+
+#     Args:
+#         None
+
+#     Returns:
+#         None
+
+#     Note:
+#     - The function leverages the 'delete_project_from_S3' method
+#     to perform project deletions.
+#     - Ensure that AWS credentials (Access Key and Secret Key)
+#     with sufficient permissions are properly configured for the
+#     S3 client used in this function.
+#     """
+#     not_open_published_projects = (
+#         PublishedProject.objects.filter(access_policy=AccessPolicy.RESTRICTED)
+#         | PublishedProject.objects.filter(access_policy=AccessPolicy.CREDENTIALED)
+#         | PublishedProject.objects.filter(access_policy=AccessPolicy.CONTRIBUTOR_REVIEW)
+#     )
+#     for project in not_open_published_projects:
+#         delete_project_from_S3(project)
+#     # since all open projects are stored in the same bucket, we only
+#     # need to delete the bucket once by passing
+#     # the first open project
+#     delete_project_from_S3(
+#         PublishedProject.objects.filter(access_policy=AccessPolicy.OPEN)[0]
+#     )