[ENG-6876] Merge develop into feature/gravy_valet_intergration

CHANGELOG

@@ -2,6 +2,30 @@
 
 We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO.
 
+24.11.0 (2024-12-11)
+====================
+- Institutional Dashboard Project Bugfix Release
+
+24.10.0 (2024-12-05)
+====================
+
+- Migrate Preprint Affilations
+- Add OOPSpam and Akismet metrics to spam report
+- Add PrivateSpamMetricsReport
+- Update PrivateSpamMetricsReporter to work with refactored MonthlyReporter
+- Fix duplicate reports when run for past years
+- Fix counted-usage clobbers
+
+24.09.0 (2024-11-14)
+====================
+
+- Institutional Dashboard Project BE Release
+
+24.08.0 (2024-10-30)
+====================
+
+- Fix admin confirmation link generation and handling
+
 24.07.0 (2024-09-19)
 ====================
 

Dockerfile

@@ -52,6 +52,10 @@ RUN set -ex \
         libffi-dev
 
 WORKDIR /code
+
+# Policies
+ADD https://github.com/CenterForOpenScience/cos.io.git#master ./COS_POLICIES/
+
 COPY pyproject.toml .
 COPY poetry.lock .
 # Fix: https://github.com/CenterForOpenScience/osf.io/pull/6783

admin/management/views.py

@@ -1,4 +1,3 @@
-import datetime
 from dateutil.parser import isoparse
 from django.views.generic import TemplateView, View
 from django.contrib import messages
@@ -13,6 +12,7 @@
 from scripts.find_spammy_content import manage_spammy_content
 from django.urls import reverse
 from django.shortcuts import redirect
+from osf.metrics.utils import YearMonth
 from osf.models import Preprint, Node, Registration
 
 
@@ -120,11 +120,14 @@ def post(self, request, *args, **kwargs):
         if monthly_report_date:
             report_date = isoparse(monthly_report_date).date()
         else:
-            report_date = datetime.datetime.now().date()
+            report_date = None
 
         errors = monthly_reporters_go(
-            report_month=report_date.month,
-            report_year=report_date.year
+            yearmonth=(
+                str(YearMonth.from_date(report_date))
+                if report_date is not None
+                else ''
+            ),
         )
 
         if errors:

admin/nodes/urls.py

@@ -38,4 +38,5 @@
     re_path(r'^(?P<guid>[a-z0-9]+)/make_private/$', views.NodeMakePrivate.as_view(), name='make-private'),
     re_path(r'^(?P<guid>[a-z0-9]+)/make_public/$', views.NodeMakePublic.as_view(), name='make-public'),
     re_path(r'^(?P<guid>[a-z0-9]+)/remove_notifications/$', views.NodeRemoveNotificationView.as_view(), name='node-remove-notifications'),
+    re_path(r'^(?P<guid>[a-z0-9]+)/update_moderation_state/$', views.NodeUpdateModerationStateView.as_view(), name='node-update-mod-state'),
 ]

admin/nodes/views.py

@@ -118,6 +118,17 @@ def post(self, request, *args, **kwargs):
 
         return redirect('nodes:node', guid=kwargs.get('guid'))
 
+
+class NodeUpdateModerationStateView(View):
+    def post(self, request, *args, **kwargs):
+        guid = kwargs.get('guid')
+        node = AbstractNode.load(guid)
+        node.update_moderation_state()
+        messages.success(request, 'Moderation state successfully updated.')
+
+        return redirect('nodes:node', guid=kwargs.get('guid'))
+
+
 class NodeSearchView(PermissionRequiredMixin, FormView):
     """ Allows authorized users to search for a node by it's guid.
     """

admin/templates/nodes/node.html

@@ -64,7 +64,12 @@ <h2>{{ node.type|cut:'osf.'|title }}: <b>{{ node.title }}</b> <a href="{{ node.a
                     </tr>
                     <tr>
                         <td>Moderation State</td>
-                        <td>{{ node.moderation_state }}</td>
+                        <td>{{ node.moderation_state }}
+                        <form method="post" action="{% url 'nodes:node-update-mod-state' node.guid %}">
+                            {% csrf_token %}
+                            <button type="submit" class="btn btn-primary">Update Moderation State</button>
+                        </form>
+                        </td>
                     </tr>
                     <tr>
                         <td>Creator</td>

admin/users/views.py

@@ -16,6 +16,7 @@
 from django.core.mail import send_mail
 from django.shortcuts import redirect
 from django.core.paginator import Paginator
+from django.core.exceptions import ValidationError
 
 from osf.exceptions import UserStateError
 from osf.models.base import Guid
@@ -456,10 +457,19 @@ def get_context_data(self, **kwargs):
 
 class GetUserConfirmationLink(GetUserLink):
     def get_link(self, user):
+        if user.is_confirmed:
+            return f'User {user._id} is already confirmed'
+
+        if user.deleted or user.is_merged:
+            return f'User {user._id} is deleted or merged'
+
         try:
-            return user.get_confirmation_url(user.username, force=True)
-        except KeyError as e:
-            return str(e)
+            confirmation_link = user.get_or_create_confirmation_url(user.username, force=True, renew=True)
+            return confirmation_link
+        except ValidationError:
+            return f'Invalid email for user {user._id}'
+        except KeyError:
+            return 'Could not generate or refresh confirmation link'
 
     def get_link_type(self):
         return 'User Confirmation'

admin_tests/users/test_views.py

@@ -486,10 +486,15 @@ def test_get_user_confirmation_link(self):
         view = views.GetUserConfirmationLink()
         view = setup_view(view, request, guid=user._id)
 
+        link = view.get_link(user)
+
+        user.refresh_from_db()
+
         user_token = list(user.email_verifications.keys())[0]
+
         ideal_link_path = f'/confirm/{user._id}/{user_token}/'
-        link = view.get_link(user)
-        link_path = str(furl(link).path)
+
+        link_path = str(furl(link).path).rstrip('/') + '/'
 
         assert link_path == ideal_link_path
 
@@ -511,6 +516,45 @@ def test_get_user_confirmation_link_with_expired_token(self):
 
         assert link_path == ideal_link_path
 
+    def test_get_user_confirmation_link_generates_new_token_if_expired(self):
+        user = UnconfirmedUserFactory()
+        request = RequestFactory().get('/fake_path')
+        view = views.GetUserConfirmationLink()
+        view = setup_view(view, request, guid=user._id)
+
+        old_user_token = list(user.email_verifications.keys())[0]
+        user.email_verifications[old_user_token]['expiration'] = datetime.utcnow().replace(tzinfo=pytz.utc) - timedelta(hours=24)
+        user.save()
+
+        link = view.get_link(user)
+        user.refresh_from_db()
+
+        new_user_token = list(user.email_verifications.keys())[0]
+
+        assert new_user_token != old_user_token
+
+        link_path = str(furl(link).path)
+        ideal_link_path = f'/confirm/{user._id}/{new_user_token}/'
+        assert link_path == ideal_link_path
+
+    def test_get_user_confirmation_link_does_not_change_unexpired_token(self):
+        user = UnconfirmedUserFactory()
+        request = RequestFactory().get('/fake_path')
+        view = views.GetUserConfirmationLink()
+        view = setup_view(view, request, guid=user._id)
+
+        user_token_before = list(user.email_verifications.keys())[0]
+
+        user.email_verifications[user_token_before]['expiration'] = datetime.utcnow().replace(tzinfo=pytz.utc) + timedelta(hours=24)
+        user.save()
+
+        with mock.patch('osf.models.user.OSFUser.get_or_create_confirmation_url') as mock_method:
+            mock_method.return_value = user.get_confirmation_url(user.username, force=False, renew=False)
+
+        user_token_after = list(user.email_verifications.keys())[0]
+
+        assert user_token_before == user_token_after
+
     def test_get_password_reset_link(self):
         user = UnconfirmedUserFactory()
         request = RequestFactory().get('/fake_path')

api/base/elasticsearch_dsl_views.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+import abc
+import datetime
+import typing
+
+import elasticsearch_dsl as edsl
+from rest_framework import generics, exceptions as drf_exceptions
+from rest_framework.settings import api_settings as drf_settings
+from api.base.settings.defaults import REPORT_FILENAME_FORMAT
+
+if typing.TYPE_CHECKING:
+    from rest_framework import serializers
+
+from api.base.filters import FilterMixin
+from api.base.views import JSONAPIBaseView
+from api.metrics.renderers import (
+    MetricsReportsCsvRenderer,
+    MetricsReportsTsvRenderer,
+    MetricsReportsJsonRenderer,
+)
+from api.base.pagination import ElasticsearchQuerySizeMaximumPagination, JSONAPIPagination
+from api.base.renderers import JSONAPIRenderer
+
+
+class ElasticsearchListView(FilterMixin, JSONAPIBaseView, generics.ListAPIView, abc.ABC):
+    '''abstract view class using `elasticsearch_dsl.Search` as a queryset-analogue
+
+    builds a `Search` based on `self.get_default_search()` and the request's
+    query parameters for filtering, sorting, and pagination -- fetches only
+    the data required for the response, just like with a queryset!
+    '''
+    serializer_class: type[serializers.BaseSerializer]  # required on subclasses
+
+    default_ordering: str | None = None  # name of a serializer field, prepended with "-" for descending sort
+    ordering_fields: frozenset[str] = frozenset()  # serializer field names
+
+    @abc.abstractmethod
+    def get_default_search(self) -> edsl.Search | None:
+        '''the base `elasticsearch_dsl.Search` for this list, based on url path
+
+        (common jsonapi query parameters will be considered automatically)
+        '''
+        ...
+
+    FILE_RENDERER_CLASSES = {
+        MetricsReportsCsvRenderer,
+        MetricsReportsTsvRenderer,
+        MetricsReportsJsonRenderer,
+    }
+
+    def set_content_disposition(self, response, renderer: str):
+        """Set the Content-Disposition header to prompt a file download with the appropriate filename.
+
+        Args:
+            response: The HTTP response object to modify.
+            renderer: The renderer instance used for the response, which determines the file extension.
+        """
+        current_date = datetime.datetime.now().strftime('%Y-%m')
+
+        if isinstance(renderer, JSONAPIRenderer):
+            extension = 'json'
+        else:
+            extension = getattr(renderer, 'extension', renderer.format)
+
+        filename = REPORT_FILENAME_FORMAT.format(
+            view_name=self.view_name,
+            date_created=current_date,
+            extension=extension,
+        )
+
+        response['Content-Disposition'] = f'attachment; filename="{filename}"'
+
+    def finalize_response(self, request, response, *args, **kwargs):
+        # Call the parent method to finalize the response first
+        response = super().finalize_response(request, response, *args, **kwargs)
+        # Check if this is a direct download request or file renderer classes, set to the Content-Disposition header
+        # so filename and attachment for browser download
+        if isinstance(request.accepted_renderer, tuple(self.FILE_RENDERER_CLASSES)):
+            self.set_content_disposition(response, request.accepted_renderer)
+
+        return response
+
+    ###
+    # beware! inheritance shenanigans below
+
+    # override FilterMixin to disable all operators besides 'eq' and 'ne'
+    MATCHABLE_FIELDS = ()
+    COMPARABLE_FIELDS = ()
+    DEFAULT_OPERATOR_OVERRIDES = {}
+    # (if you want to add fulltext-search or range-filter support, remove the override
+    #  and update `__add_search_filter` to handle those operators -- tho note that the
+    #  underlying elasticsearch field mapping will need to be compatible with the query)
+
+    # override DEFAULT_FILTER_BACKENDS rest_framework setting
+    # (filtering handled in-view to reuse logic from FilterMixin)
+    filter_backends = ()
+
+    # note: because elasticsearch_dsl.Search supports slicing and gives results when iterated on,
+    #       it works fine with default pagination
+
+    # override rest_framework.generics.GenericAPIView
+    @property
+    def pagination_class(self):
+        """
+        When downloading a file assume no pagination is necessary unless the user specifies
+        """
+        is_file_download = any(
+            self.request.accepted_renderer.format == renderer.format
+            for renderer in self.FILE_RENDERER_CLASSES
+        )
+        # if it's a file download of the JSON respect default page size
+        if is_file_download:
+            return ElasticsearchQuerySizeMaximumPagination
+        return JSONAPIPagination
+
+    def get_queryset(self):
+        _search = self.get_default_search()
+        if _search is None:
+            return []
+        # using parsing logic from FilterMixin (oddly nested dict and all)
+        for _parsed_param in self.parse_query_params(self.request.query_params).values():
+            for _parsed_filter in _parsed_param.values():
+                _search = self.__add_search_filter(
+                    _search,
+                    elastic_field_name=_parsed_filter['source_field_name'],
+                    operator=_parsed_filter['op'],
+                    value=_parsed_filter['value'],
+                )
+        return self.__add_sort(_search)
+
+    ###
+    # private methods
+
+    def __add_sort(self, search: edsl.Search) -> edsl.Search:
+        _elastic_sort = self.__get_elastic_sort()
+        return (search if _elastic_sort is None else search.sort(_elastic_sort))
+
+    def __get_elastic_sort(self) -> str | None:
+        _sort_param = self.request.query_params.get(drf_settings.ORDERING_PARAM, self.default_ordering)
+        if not _sort_param:
+            return None
+        _sort_field, _ascending = (
+            (_sort_param[1:], False)
+            if _sort_param.startswith('-')
+            else (_sort_param, True)
+        )
+        if _sort_field not in self.ordering_fields:
+            raise drf_exceptions.ValidationError(
+                f'invalid value for {drf_settings.ORDERING_PARAM} query param (valid values: {", ".join(self.ordering_fields)})',
+            )
+        _serializer_field = self.get_serializer().fields[_sort_field]
+        _elastic_sort_field = _serializer_field.source
+        return (_elastic_sort_field if _ascending else f'-{_elastic_sort_field}')
+
+    def __add_search_filter(
+        self,
+        search: edsl.Search,
+        elastic_field_name: str,
+        operator: str,
+        value: str,
+    ) -> edsl.Search:
+        match operator:  # operators from FilterMixin
+            case 'eq':
+                if value == '':
+                    return search.exclude('exists', field=elastic_field_name)
+                return search.filter('term', **{elastic_field_name: value})
+            case 'ne':
+                if value == '':
+                    return search.filter('exists', field=elastic_field_name)
+                return search.exclude('term', **{elastic_field_name: value})
+            case _:
+                raise NotImplementedError(f'unsupported filter operator "{operator}"')