Include robots.txt in UI and API for handling of web crawlers

.gitignore

@@ -125,5 +125,5 @@ dmypy.json
 
 # Editor settings
 .vscode
-
+.idea/
 .DS_Store

dandiapi/api/tests/test_robots.py

@@ -0,0 +1,13 @@
+import pytest
+from django.urls import reverse
+
+@pytest.mark.django_db
+def test_robots_txt(api_client):
+    response = api_client.get(reverse('robots_txt'))
+    assert response.status_code == 200
+    assert response['Content-Type'] == 'text/plain'
+    expected_content = """
+    User-agent: *
+    Disallow: /
+    """
+    assert response.content.decode('utf-8').strip() == expected_content.strip()

dandiapi/api/views/__init__.py

@@ -5,6 +5,7 @@
 from .dandiset import DandisetViewSet
 from .dashboard import DashboardView, mailchimp_csv_view, user_approval_view
 from .info import info_view
+from .robots import robots_txt_view
 from .root import root_content_view
 from .stats import stats_view
 from .upload import (
@@ -36,4 +37,5 @@
     'stats_view',
     'info_view',
     'root_content_view',
+    'robots_txt_view'
 ]

dandiapi/api/views/robots.py

@@ -0,0 +1,10 @@
+from django.views.decorators.cache import cache_page
+from django.http import HttpResponse
+
+@cache_page(60 * 60 * 24)  # 24 hour cache
+def robots_txt_view(request):
+    content = """
+    User-agent: *
+    Disallow: /
+    """
+    return HttpResponse(content, content_type="text/plain")

dandiapi/urls.py

@@ -19,6 +19,7 @@
     blob_read_view,
     info_view,
     mailchimp_csv_view,
+    robots_txt_view,
     root_content_view,
     stats_view,
     upload_complete_view,
@@ -80,6 +81,7 @@ def to_url(self, value):
 register_converter(DandisetIDConverter, 'dandiset_id')
 urlpatterns = [
     path('', root_content_view),
+    path('robots.txt', robots_txt_view),
     path('api/', include(router.urls)),
     path('api/auth/token/', auth_token_view, name='auth-token'),
     path('api/stats/', stats_view),

web/public/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /