Merge pull request #933 from uktrade/release/3.4.0

Release 3.4.0

Makefile

@@ -12,6 +12,9 @@ docker-cleanup:
 migrate:
 	docker-compose run leeloo python manage.py migrate
 
+init-es:
+	docker-compose run leeloo python manage.py init_es
+
 makemigrations:
 	docker-compose run leeloo python manage.py makemigrations
 

Procfile

@@ -1,4 +1,4 @@
-web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && gunicorn config.wsgi --config config/gunicorn.py
+web: python manage.py collectstatic --noinput && python manage.py distributed_migrate --noinput && python manage.py init_es && gunicorn config.wsgi --config config/gunicorn.py
 init_rev: python manage.py createinitialrevisions
-celeryworker: celery worker -A config -l info
+celeryworker: celery worker -A config -l info -O fair --prefetch-multiplier 1 -Q celery,long-running
 celerybeat: celery beat -A config -l info

README.md

@@ -24,10 +24,11 @@ Leeloo uses Docker compose to setup and run all the necessary components. The do
     docker-compose build
     ```
 
-3.  Populate the db:
+3.  Populate the database and initialise Elasticsearch:
 
     ```shell
     docker-compose run leeloo ./manage.py migrate
+    docker-compose run leeloo ./manage.py init_es
     docker-compose run leeloo ./manage.py loadinitialmetadata
     docker-compose run leeloo ./manage.py createinitialrevisions
     ```
@@ -108,11 +109,11 @@ Dependencies:
 
 9. Make sure you have redis running locally and that the REDIS_BASE_URL in your `.env` is up-to-date.
 
-10.  Configure and populate the db:
+10.  Populate the database and initialise Elasticsearch:
 
     ```shell
     ./manage.py migrate
-    ./manage.py createsuperuser
+    ./manage.py init_es
 
     ./manage.py loadinitialmetadata
     ./manage.py createinitialrevisions
@@ -126,17 +127,26 @@ Dependencies:
     ./manage.py sync_es
     ```
 
-12. Start the server:
+12.  Create a superuser:
+
+    ```shell
+    ./manage.py createsuperuser
+    ```
+    
+13. Start the server:
 
     ```shell
     ./manage.py runserver
     ```
 
-13. Start celery:
+14. Start celery:
 
     ```shell
-    celery worker -A config -l info -B
+    celery worker -A config -l info -Q celery,long-running -B
     ```
+    
+    Note that in production the `-O fair --prefetch-multiplier 1` arguments are also used for better fairness when
+    long-running tasks are running or pending execution.
 
 ## Local development
 
@@ -198,6 +208,8 @@ Leeloo can run on any Heroku-style platform. Configuration is performed via the
 
 | Variable name | Required | Description |
 | ------------- | ------------- | ------------- |
+| `ALLOWED_ADMIN_IPS` | No | IP addresses (comma-separated) that can access the admin site when RESTRICT_ADMIN is True. |
+| `ALLOWED_ADMIN_IP_RANGES` | No | IP address ranges (comma-separated) that can access the admin site when RESTRICT_ADMIN is True. |
 | `AV_SERVICE_URL` | Yes | URL for ClamAV service. If not configured, virus scanning will fail. |
 | `AWS_ACCESS_KEY_ID` | No | Used as part of [boto3 auto-configuration](http://boto3.readthedocs.io/en/latest/guide/configuration.html#configuring-credentials). |
 | `AWS_DEFAULT_REGION` | No | [Default region used by boto3.](http://boto3.readthedocs.io/en/latest/guide/configuration.html#environment-variable-configuration) |
@@ -229,6 +241,7 @@ Leeloo can run on any Heroku-style platform. Configuration is performed via the
 | `REDIS_CELERY_DB`  | No | redis db for celery (default 1) |
 | `RESOURCE_SERVER_INTROSPECTION_URL` | If SSO enabled | RFC 7662 token introspection URL used for signle sign-on |
 | `RESOURCE_SERVER_AUTH_TOKEN` | If SSO enabled | Access token for RFC 7662 token introspection server |
+| `RESTRICT_ADMIN` | No | Whether to restrict access to the admin site by IP address. |
 | `SENTRY_ENVIRONMENT`  | Yes | Value for the environment tag in Sentry. |
 | `SSO_ENABLED` | Yes | Whether single sign-on via RFC 7662 token introspection is enabled |
 | `WEB_CONCURRENCY` | No | Number of Gunicorn workers (set automatically by Heroku, otherwise defaults to 1). |
@@ -266,6 +279,12 @@ These commands are generally only intended to be used on a blank database.
 
 ### Elasticsearch
 
+Create the Elasticsearch index (if it doesn't exist) and update the mapping:
+
+```shell
+./manage.py init_es
+```
+
 Resync all Elasticsearch records:
 
 ```shell

config/api_urls.py

@@ -6,6 +6,7 @@
 from datahub.company import urls as company_urls
 from datahub.company import views as company_views
 from datahub.event import urls as event_urls
+from datahub.feature_flag import urls as feature_flag_urls
 from datahub.interaction import urls as interaction_urls
 from datahub.investment import urls as investment_urls
 from datahub.leads import urls as leads_urls
@@ -28,6 +29,7 @@
     path('', include((company_urls.company_urls, 'company'), namespace='company')),
     path('', include((company_urls.ch_company_urls, 'ch-company'), namespace='ch-company')),
     path('', include((event_urls, 'event'), namespace='event')),
+    path('', include((feature_flag_urls, 'feature-flag'), namespace='feature-flag')),
     path('', include((interaction_urls, 'interaction'), namespace='interaction')),
     path('', include((investment_urls, 'investment'), namespace='investment')),
     path('', include((leads_urls, 'business-leads'), namespace='business-leads')),

config/celery.py

@@ -2,6 +2,7 @@
 import os
 
 from celery import Celery
+from celery.signals import after_setup_logger
 from raven import Client
 from raven.contrib.celery import register_logger_signal, register_signal
 
@@ -31,3 +32,10 @@
 
 # hook into the Celery error handler
 register_signal(client, ignore_expected=True)
+
+
+@after_setup_logger.connect
+def after_setup_logger_handler(**kwargs):
+    """As the Elasticsearch module is noisy, set its log level to WARNING for Celery workers."""
+    es_logger = logging.getLogger('elasticsearch')
+    es_logger.setLevel(logging.WARNING)

config/settings/common.py

@@ -8,6 +8,7 @@
 """
 
 import ssl
+from datetime import timedelta
 
 import environ
 from celery.schedules import crontab
@@ -59,6 +60,7 @@
     'datahub.company',
     'datahub.documents',
     'datahub.event',
+    'datahub.feature_flag.apps.FeatureFlagConfig',
     'datahub.interaction',
     'datahub.investment',
     'datahub.leads',
@@ -87,6 +89,7 @@
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'admin_ip_restrictor.middleware.AdminIPRestrictorMiddleware',
     'datahub.core.reversion.NonAtomicRevisionMiddleware',
     'whitenoise.middleware.WhiteNoiseMiddleware'
 ]
@@ -148,6 +151,13 @@
     'datahub.core.auth.TeamModelPermissionsBackend'
 ]
 
+
+# django-admin-ip-restrictor
+
+RESTRICT_ADMIN = env.bool('RESTRICT_ADMIN', False)
+ALLOWED_ADMIN_IPS = env.list('ALLOWED_ADMIN_IPS', default=[])
+ALLOWED_ADMIN_IP_RANGES = env.list('ALLOWED_ADMIN_IP_RANGES', default=[])
+
 # django-oauth-toolkit settings
 
 SSO_ENABLED = env.bool('SSO_ENABLED')
@@ -246,6 +256,18 @@
             'ssl_cert_reqs': ssl.CERT_NONE
         }
         CELERY_BROKER_USE_SSL = CELERY_REDIS_BACKEND_USE_SSL
+
+    # Increase timeout from one hour for long-running tasks
+    # (If the timeout is reached before a task, Celery will start it again. This
+    # would affect in particular any long-running tasks using acks_late=True.)
+    CELERY_BROKER_TRANSPORT_OPTIONS = {
+        'visibility_timeout': int(timedelta(hours=9).total_seconds())
+    }
+    CELERY_TASK_ROUTES = {
+        'datahub.search.tasks.sync_model': {
+            'queue': 'long-running'
+        }
+    }
     CELERY_BEAT_SCHEDULE = {
         'refresh_pending_payment_gateway_sessions': {
             'task': 'datahub.omis.payment.tasks.refresh_pending_payment_gateway_sessions',
@@ -254,7 +276,14 @@
                 'age_check': 60  # in minutes
             }
         },
+        'sync_es': {
+            'task': 'datahub.search.tasks.sync_all_models',
+            'schedule': crontab(minute=0, hour=1),
+        },
     }
+    CELERY_WORKER_LOG_FORMAT = (
+        "[%(asctime)s: %(levelname)s/%(processName)s] [%(name)s] %(message)s"
+    )
 
 # FRONTEND
 DATAHUB_FRONTEND_BASE_URL = env('DATAHUB_FRONTEND_BASE_URL', default='http://localhost:3000')

datahub/company/test/factories.py

@@ -35,6 +35,7 @@ class CompanyFactory(factory.django.DjangoModelFactory):
     created_by = factory.SubFactory(AdviserFactory)
     modified_by = factory.SubFactory(AdviserFactory)
     name = factory.Faker('company')
+    alias = factory.Faker('company')
     registered_address_1 = factory.Sequence(lambda n: f'{n} Foo st.')
     registered_address_town = 'London'
     registered_address_country_id = constants.Country.united_kingdom.value.id
@@ -87,6 +88,7 @@ class ContactFactory(factory.django.DjangoModelFactory):
     last_name = factory.Faker('last_name')
     company = factory.SubFactory(CompanyFactory)
     email = '[email protected]'
+    job_title = factory.Faker('job')
     primary = True
     telephone_countrycode = '+44'
     telephone_number = '123456789'

datahub/company/test/test_contact_views.py

@@ -34,7 +34,7 @@ def test_with_manual_address(self):
             },
             'first_name': 'Oratio',
             'last_name': 'Nelson',
-            'job_title': constants.Role.owner.value.name,
+            'job_title': 'Head of Sales',
             'company': {
                 'id': str(company.pk)
             },
@@ -72,7 +72,7 @@ def test_with_manual_address(self):
             'first_name': 'Oratio',
             'last_name': 'Nelson',
             'name': 'Oratio Nelson',
-            'job_title': constants.Role.owner.value.name,
+            'job_title': 'Head of Sales',
             'company': {
                 'id': str(company.pk),
                 'name': company.name
@@ -306,7 +306,7 @@ def test_patch(self):
                 title_id=constants.Title.admiral_of_the_fleet.value.id,
                 first_name='Oratio',
                 last_name='Nelson',
-                job_title=constants.Role.owner.value.name,
+                job_title='Head of Sales',
                 company=company,
                 email='[email protected]',
                 email_alternative='[email protected]',
@@ -347,7 +347,7 @@ def test_patch(self):
             'first_name': 'New Oratio',
             'last_name': 'Nelson',
             'name': 'New Oratio Nelson',
-            'job_title': constants.Role.owner.value.name,
+            'job_title': 'Head of Sales',
             'company': {
                 'id': str(company.pk),
                 'name': company.name
@@ -481,7 +481,7 @@ def test_view(self):
             title_id=constants.Title.admiral_of_the_fleet.value.id,
             first_name='Oratio',
             last_name='Nelson',
-            job_title=constants.Role.owner.value.name,
+            job_title='Head of Sales',
             company=company,
             email='[email protected]',
             email_alternative='[email protected]',
@@ -518,7 +518,7 @@ def test_view(self):
             'first_name': 'Oratio',
             'last_name': 'Nelson',
             'name': 'Oratio Nelson',
-            'job_title': constants.Role.owner.value.name,
+            'job_title': 'Head of Sales',
             'company': {
                 'id': str(company.pk),
                 'name': company.name

datahub/company/views.py

@@ -7,7 +7,7 @@
 
 from datahub.core.audit import AuditViewSet
 from datahub.core.mixins import ArchivableViewSetMixin
-from datahub.core.viewsets import CoreViewSetV3
+from datahub.core.viewsets import CoreViewSet
 from datahub.investment.queryset import get_slim_investment_project_queryset
 from datahub.oauth.scopes import Scope
 from .models import (
@@ -25,7 +25,7 @@
 )
 
 
-class CompanyViewSet(ArchivableViewSetMixin, CoreViewSetV3):
+class CompanyViewSet(ArchivableViewSetMixin, CoreViewSet):
     """Company view set V3."""
 
     required_scopes = (Scope.internal_front_end,)
@@ -72,7 +72,7 @@ class CompaniesHouseCompanyViewSet(
     lookup_field = 'company_number'
 
 
-class ContactViewSet(ArchivableViewSetMixin, CoreViewSetV3):
+class ContactViewSet(ArchivableViewSetMixin, CoreViewSet):
     """Contact ViewSet v3."""
 
     required_scopes = (Scope.internal_front_end,)

datahub/core/constants.py

@@ -288,23 +288,6 @@ class Country(Enum):
     zimbabwe = Constant('Zimbabwe', '39afd8d0-5d95-e211-a939-e4115bead28a')
 
 
-class Role(Enum):
-    """Roles."""
-
-    agent = Constant('Agent', '71756b9a-5d95-e211-a939-e4115bead28a')
-    consultant = Constant('Consultant', '6f756b9a-5d95-e211-a939-e4115bead28a')
-    deceased = Constant('Deceased', '70756b9a-5d95-e211-a939-e4115bead28a')
-    director = Constant('Director', '6af7d52b-63a3-e211-a646-e4115bead28a')
-    employee = Constant('Employee', '6e756b9a-5d95-e211-a939-e4115bead28a')
-    individual = Constant('Individual', '72756b9a-5d95-e211-a939-e4115bead28a')
-    leftorganisation = Constant('LeftOrganisation', '6c756b9a-5d95-e211-a939-e4115bead28a')
-    managing_director = Constant('Managing Director', 'dc02a687-5da3-e211-a646-e4115bead28a')
-    owner = Constant('Owner', '73756b9a-5d95-e211-a939-e4115bead28a')
-    partner = Constant('Partner', '74756b9a-5d95-e211-a939-e4115bead28a')
-    student = Constant('Student', '75756b9a-5d95-e211-a939-e4115bead28a')
-    unknown = Constant('Unknown', '6d756b9a-5d95-e211-a939-e4115bead28a')
-
-
 class Sector(Enum):
     """Sectors (not all of them!)."""
 

datahub/core/management/commands/loadinitialmetadata.py

@@ -13,7 +13,6 @@
 
 SHARED_FIXTURES = (
     SHARED_METADATA_FIXTURE_DIR / 'companies.yaml',
-    SHARED_METADATA_FIXTURE_DIR / 'contacts.yaml',
     SHARED_METADATA_FIXTURE_DIR / 'countries.yaml',
     SHARED_METADATA_FIXTURE_DIR / 'investment.yaml',
     SHARED_METADATA_FIXTURE_DIR / 'referrals.yaml',

datahub/core/test/support/views.py

@@ -2,13 +2,13 @@
 
 from datahub.core.test.support.models import PermissionModel
 from datahub.core.test.support.serializers import PermissionModelSerializer
-from datahub.core.viewsets import CoreViewSetV3
+from datahub.core.viewsets import CoreViewSet
 from datahub.oauth.test.scopes import TestScope
 from .models import MyDisableableModel
 from .serializers import MyDisableableModelSerializer
 
 
-class MyDisableableModelViewset(CoreViewSetV3):
+class MyDisableableModelViewset(CoreViewSet):
     """MyDisableableModel view set."""
 
     permission_classes = (IsAuthenticatedOrTokenHasScope,)
@@ -17,7 +17,7 @@ class MyDisableableModelViewset(CoreViewSetV3):
     queryset = MyDisableableModel.objects.all()
 
 
-class PermissionModelViewset(CoreViewSetV3):
+class PermissionModelViewset(CoreViewSet):
     """PermissionModel view set."""
 
     serializer_class = PermissionModelSerializer