diff --git a/scraper/templates/scraper/base.html b/scraper/templates/scraper/base.html index beff4c4..dc3ef98 100644 --- a/scraper/templates/scraper/base.html +++ b/scraper/templates/scraper/base.html @@ -32,6 +32,7 @@
Home Scraping jobs + Scraping tasks
diff --git a/scraper/templates/scraper/scrapingtask_confirm_delete.html b/scraper/templates/scraper/scrapingtask_confirm_delete.html new file mode 100644 index 0000000..4414071 --- /dev/null +++ b/scraper/templates/scraper/scrapingtask_confirm_delete.html @@ -0,0 +1,20 @@ +{% extends 'scraper/base.html' %} +{% load crispy_forms_tags %} + +{% block content %} + +
+
+ {% csrf_token %} + +
+ Delete task + Please confirm deleting task: "{{ object.title }}" +
+
+ + Cancel +
+
+
+{% endblock content %} \ No newline at end of file diff --git a/scraper/templates/scraper/scrapingtask_detail.html b/scraper/templates/scraper/scrapingtask_detail.html new file mode 100644 index 0000000..db9c307 --- /dev/null +++ b/scraper/templates/scraper/scrapingtask_detail.html @@ -0,0 +1,23 @@ +{% extends 'scraper/base.html' %} + +{% block content %} +
+
+
{{ object.title }}
+
+ +
+ 
{{ object.task }}
+
+ + {% if object.user == user %} +
+ Update + Test + Delete +
+ {% endif %} +
+ + +{% endblock content %} \ No newline at end of file diff --git a/scraper/templates/scraper/scrapingtask_form.html b/scraper/templates/scraper/scrapingtask_form.html new file mode 100644 index 0000000..1025224 --- /dev/null +++ b/scraper/templates/scraper/scrapingtask_form.html @@ -0,0 +1,18 @@ +{% extends 'scraper/base.html' %} +{% load crispy_forms_tags %} + +{% block content %} + +
+
+ {% csrf_token %} +
+ Scraping Task + {{ form|crispy }} +
+
+ +
+
+
+{% endblock content %} \ No newline at end of file diff --git a/scraper/templates/scraper/scrapingtask_list.html b/scraper/templates/scraper/scrapingtask_list.html new file mode 100644 index 0000000..ab06900 --- /dev/null +++ b/scraper/templates/scraper/scrapingtask_list.html @@ -0,0 +1,46 @@ +{% extends 'scraper/base.html' %} + +{% block content %} + + {% for task in object_list %} +
+
+
+ + {{ task.title }} + + +
+ +
+ +
+ 
{{ task.task }}
+
+ +
+ {% endfor %} + + {% if is_paginated %} + + {% if page_obj.has_previous %} + First + Previous + {% endif %} + + {% for num in page_obj.paginator.page_range %} + {% if page_obj.number == num %} + {{ num }} + {% elif num > page_obj.number|add:'-3' and num < page_obj.number|add:'3' %} + {{ num }} + {% endif %} + {% endfor %} + + {% if page_obj.has_next %} + Next + Last + {% endif %} + + {% endif %} + +{% endblock content %} \ No newline at end of file diff --git a/scraper/tests/test_models.py b/scraper/tests/test_models.py index c1d2b1d..229d724 100644 --- a/scraper/tests/test_models.py +++ b/scraper/tests/test_models.py @@ -29,4 +29,4 @@ def test_scraping_jobs_for_user(user_factory, scraping_job_factory): scraping_job_factory.create(user=user) scraping_job_factory.create(user=user) - assert ScrapingJob.objects.filter(user=user).count() == 2, ScrapingJob.objects.all() + assert ScrapingJob.objects.filter(user=user).count() == 2, ScrapingJob.objects.count() diff --git a/scraper/urls.py b/scraper/urls.py index d5fe0b5..ab3d229 100644 --- a/scraper/urls.py +++ b/scraper/urls.py @@ -2,15 +2,34 @@ from .views import (home, ScrapingJobListView, ScrapingJobDetailView, ScrapingJobCreateView, ScrapingJobUpdateView, - ScrapingJobDeleteView, scraping_job_test_run, test_site) + ScrapingJobDeleteView, scraping_job_test_run, test_site, + ScrapingTaskListView, ScrapingTaskDetailView, + ScrapingTaskCreateView, ScrapingTaskUpdateView, + ScrapingTaskDeleteView) urlpatterns = [ path('', home, name='scraper-home'), path('test-site/', test_site, name='scraper-testsite'), path('scraping-jobs/', ScrapingJobListView.as_view(), name='scrapingjobs'), - path('scraping-job//', ScrapingJobDetailView.as_view(), name='scrapingjob-detail'), - path('scraping-job//testrun/', scraping_job_test_run, name='scrapingjob-testrun'), - path('scraping-job/new/', ScrapingJobCreateView.as_view(), name='scrapingjob-create'), - path('scraping-job//update/', ScrapingJobUpdateView.as_view(), name='scrapingjob-update'), - path('scraping-job//delete/', ScrapingJobDeleteView.as_view(), name='scrapingjob-delete'), + path('scraping-job//', ScrapingJobDetailView.as_view(), + name='scrapingjob-detail'), + path('scraping-job//testrun/', + scraping_job_test_run, name='scrapingjob-testrun'), + path('scraping-job/new/', ScrapingJobCreateView.as_view(), + name='scrapingjob-create'), + path('scraping-job//update/', + ScrapingJobUpdateView.as_view(), name='scrapingjob-update'), + path('scraping-job//delete/', + ScrapingJobDeleteView.as_view(), name='scrapingjob-delete'), + path('scraping-tasks/', ScrapingTaskListView.as_view(), name='scrapingtasks'), + path('scraping-task//', ScrapingTaskDetailView.as_view(), + name='scrapingtask-detail'), + path('scraping-task//', ScrapingTaskDetailView.as_view(), + name='scrapingtask-testrun'), + path('scraping-task/new/', ScrapingTaskCreateView.as_view(), + name='scrapingtask-create '), + path('scraping-task//update/', + ScrapingTaskUpdateView.as_view(), name='scrapingtask-update'), + path('scraping-task//delete/', + ScrapingTaskDeleteView.as_view(), name='scrapingtask-delete'), ] diff --git a/scraper/views.py b/scraper/views.py index b891bbb..4018b4d 100644 --- a/scraper/views.py +++ b/scraper/views.py @@ -15,8 +15,8 @@ from .item_scraper import item_scraper from .item_scraper.validators import ValidationError as ScraptTaskValidationError -from .forms import ScrapingJobCreateForm, ScrapingJobUpdateForm -from .models import ScrapingJob +from .forms import (ScrapingJobCreateForm, ScrapingJobUpdateForm) +from .models import ScrapingJob, ScrapingTask deals = [ { @@ -68,6 +68,60 @@ def test_request_vs_object_user(view): return False +class ScrapingTaskListView(LoginRequiredMixin, ListView): + model = ScrapingTask + ordering = ['favourite', 'title'] + paginate_by = 10 + + def get_queryset(self, *args, **kwargs): + user = self.request.user + return ScrapingTask.objects.filter(user=user) + + +class ScrapingTaskDetailView(LoginRequiredMixin, UserPassesTestMixin, DetailView): + model = ScrapingTask + + def test_func(self): + return test_request_vs_object_user(self) + + +class ScrapingTaskCreateView(LoginRequiredMixin, CreateView): + model = ScrapingTask + fields = [ + 'title', + 'favourite', + 'task', + ] + + def form_valid(self, form): + form.instance.user = self.request.user + return super().form_valid(form) + + +class ScrapingTaskUpdateView(LoginRequiredMixin, UserPassesTestMixin, UpdateView): + model = ScrapingTask + fields = [ + 'title', + 'favourite', + 'task', + ] + + def form_valid(self, form): + form.instance.user = self.request.user + return super().form_valid(form) + + def test_func(self): + return test_request_vs_object_user(self) + + +class ScrapingTaskDeleteView(LoginRequiredMixin, UserPassesTestMixin, DeleteView): + model = ScrapingTask + success_url = '/scraping-tasks/' + + def test_func(self): + return test_request_vs_object_user(self) + + class ScrapingJobListView(LoginRequiredMixin, ListView): model = ScrapingJob context_object_name = 'tasks' @@ -89,13 +143,6 @@ def test_func(self): class ScrapingJobCreateView(LoginRequiredMixin, CreateView): model = ScrapingJob form_class = ScrapingJobCreateForm - # fields = [ - # 'url', - # 'active', - # 'multiple', - # 'task', - # 'running_time', - # ] def form_valid(self, form): form.instance.user = self.request.user @@ -105,14 +152,6 @@ def form_valid(self, form): class ScrapingJobUpdateView(LoginRequiredMixin, UserPassesTestMixin, UpdateView): model = ScrapingJob form_class = ScrapingJobUpdateForm - # fields = [ - # 'url', - # 'active', - # 'multiple', - # 'running_time', - # 'scraping_task', - # 'description', - # ] def form_valid(self, form): form.instance.user = self.request.user @@ -124,7 +163,7 @@ def test_func(self): class ScrapingJobDeleteView(LoginRequiredMixin, UserPassesTestMixin, DeleteView): model = ScrapingJob - success_url = '/scraping-tasks/' + success_url = '/scraping-jobs/' def test_func(self): return test_request_vs_object_user(self) diff --git a/tests/test_functional.py b/tests/test_functional.py index 1663f58..9b408bb 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -49,15 +49,18 @@ def test_standard_scenario(live_server, browser, user_factory, assert len(jobs_divs) == 2 # but let's double check if detail view of the other one gives an error - browser.get(live_server.url + reverse('scrapingjob-detail', args=[job21.pk])) + browser.get(live_server.url + + reverse('scrapingjob-detail', args=[job21.pk])) assert '403' in browser.page_source # let's assume we just can find both task data within the detail # page - browser.get(live_server.url + reverse('scrapingjob-detail', args=[job11.pk])) + browser.get(live_server.url + + reverse('scrapingjob-detail', args=[job11.pk])) assert task11.task in browser.find_element_by_tag_name('body').text,\ browser.find_element_by_tag_name('body').text - browser.get(live_server.url + reverse('scrapingjob-detail', args=[job12 .pk])) + browser.get(live_server.url + + reverse('scrapingjob-detail', args=[job12 .pk])) assert task12.task in browser.find_element_by_tag_name('body').text,\ browser.find_element_by_tag_name('body').text @@ -78,10 +81,35 @@ def test_standard_scenario(live_server, browser, user_factory, # and after all this change we and with following: # job11 still have task11 - browser.get(live_server.url + reverse('scrapingjob-detail', args=[job11.pk])) + browser.get(live_server.url + + reverse('scrapingjob-detail', args=[job11.pk])) assert task11.task in browser.find_element_by_tag_name('body').text,\ browser.find_element_by_tag_name('body').text # but job12 should have task11 - browser.get(live_server.url + reverse('scrapingjob-detail', args=[job12.pk])) + browser.get(live_server.url + + reverse('scrapingjob-detail', args=[job12.pk])) assert task12.task not in browser.find_element_by_tag_name('body').text,\ browser.find_element_by_tag_name('body').text + + +@pytest.mark.django_db +def test_working_with_tasks(live_server, browser, user_factory, + scraping_job_factory, scraping_task_factory): + user1 = user_factory.create(password=make_password('1234')) + + task1 = scraping_task_factory.create(user=user1) + task2 = scraping_task_factory.create(user=user1) + + # Open the home page and log in as user1 + browser.get(live_server.url) + browser.find_element_by_partial_link_text("jobs").click() + browser.find_element_by_name('username').send_keys(user1.username) + browser.find_element_by_name('password').send_keys('1234') + browser.find_element_by_css_selector("button[type='submit']").click() + + browser.find_element_by_partial_link_text("tasks").click() + found_titles = {el.text for el in browser.find_elements_by_css_selector( + '.entry-title a')} + expected_titles = {t.title for t in (task1, task2)} + + assert found_titles == expected_titles diff --git a/users/tests.py b/users/tests.py index 65da5ce..4b1331b 100644 --- a/users/tests.py +++ b/users/tests.py @@ -8,7 +8,7 @@ @pytest.mark.django_db def test_standard_scenario(user_factory): user1 = user_factory.create() - assert User.objects.all().count() == 1 + assert User.objects.count() == 1 user2 = user_factory.create() - assert User.objects.all().count() == 2 + assert User.objects.count() == 2