fix flaky email tests

changelog/_4567.md

@@ -0,0 +1,8 @@
+### Added
+
+- test helper for testing emails
+- initial doc on testing
+
+### Fixed
+
+- fixed the flaky test_notify_creator_exclude_moderator test

docs/tests.md

@@ -0,0 +1,14 @@
+# Tests
+
+## Backend Tests
+
+We use pytest in combination with pytest-django and factoryboy to write backend
+tests.
+
+### Notes
+
+#### Testing Emails
+
+Emails are not guaranteed to be sent in a deterministic order, so checking
+against an index will result in flaky tests. If you want to assert
+that a mail was sent use one of the helper functions in `tests/helpers.py`.

tests/helpers.py

@@ -0,0 +1,7 @@
+from django.core import mail
+
+
+def get_emails_for_address(email_address):
+    """Return all mails send to email_address"""
+    mails = list(filter(lambda mails: mails.to[0] == email_address, mail.outbox))
+    return mails

tests/notifications/test_creator_notifications.py

@@ -3,6 +3,7 @@
 from django.urls import reverse
 
 from adhocracy4.test.helpers import redirect_target
+from tests.helpers import get_emails_for_address
 
 
 @pytest.mark.django_db
@@ -33,10 +34,11 @@ def test_notify_creator_exclude_moderator(idea, comment_factory, user):
     comment_factory(content_object=idea)
 
     assert len(mail.outbox) == 3
+    mails = get_emails_for_address(creator_moderator.email)
+    assert len(mails) == 1
     # moderator notification instead of creator notification
-    assert mail.outbox[1].to[0] == creator_moderator.email
-    assert not mail.outbox[1].subject.startswith("Reaction to your contribution")
-    assert mail.outbox[1].subject.startswith("A comment was added to the project")
+    assert not mails[0].subject.startswith("Reaction to your contribution")
+    assert mails[0].subject.startswith("A comment was added to the project")
 
 
 @pytest.mark.django_db