From e2f18bbf201ae8acad0f72ecac6e60f3d1816924 Mon Sep 17 00:00:00 2001 From: Luke Date: Sat, 28 Dec 2019 13:24:17 +1300 Subject: [PATCH 1/4] Include scrapy in setup.py --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ee49321..0c8d54b 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ import sys from setuptools import setup -packages = ['raygun4py', 'raygun4py.middleware'] +packages = ['raygun4py', 'raygun4py.middleware', 'raygun4py.extension'] base_dir = 'python2' if sys.version_info[0] == 3: @@ -20,7 +20,8 @@ 'mock >= 2.0.0', 'django == 1.8.8', 'flask >= 0.10', - 'WebTest >= 2.0.32' + 'WebTest >= 2.0.32', + 'scrapy >= 1.8.0' ] setup( From b4e4516fdc56681c36d1634adb9aa36191f6b132 Mon Sep 17 00:00:00 2001 From: Luke Date: Sat, 28 Dec 2019 13:24:34 +1300 Subject: [PATCH 2/4] Add tests for scrapy provider --- python3/tests/extension/__init__.py | 0 python3/tests/extension/test_scrapy.py | 51 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 python3/tests/extension/__init__.py create mode 100644 python3/tests/extension/test_scrapy.py diff --git a/python3/tests/extension/__init__.py b/python3/tests/extension/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python3/tests/extension/test_scrapy.py b/python3/tests/extension/test_scrapy.py new file mode 100644 index 0000000..8bf9a4b --- /dev/null +++ b/python3/tests/extension/test_scrapy.py @@ -0,0 +1,51 @@ +import unittest + +from unittest.mock import MagicMock, call +from twisted.python import failure +from raygun4py.extension.scrapy import Provider +from scrapy.http import Response +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler +from scrapy.item import Item +from scrapy.signals import item_error, spider_error +from scrapy.exceptions import NotConfigured + + +class TestProvider(unittest.TestCase): + def _getDivisionFailure(self): + try: + 1 / 0 + except: + f = failure.Failure() + return f + + def test__handler_exception_called(self): + provider = Provider(api_key="test") + crawler = get_crawler(Spider) + spider = crawler._create_spider("scrapytest.org") + item = Item() + response = Response("scrapytest.org", status=400) + + provider._handle_exception = MagicMock() + failure = self._getDivisionFailure() + + provider.item_error(item, response, spider, failure) + + provider._handle_exception.assert_called_once_with(failure) + + def test_from_crawler_not_configured(self): + crawler = get_crawler(Spider, settings_dict=None) + self.assertRaises(NotConfigured, Provider.from_crawler, crawler) + + def test_from_crawler_configured(self): + crawler = get_crawler(Spider, settings_dict={"RAYGUN_API_KEY": "test"}) + crawler.signals.connect = MagicMock() + provider = Provider.from_crawler(crawler) + + crawler.signals.connect.assert_has_calls( + [ + call(provider.spider_error, spider_error,), + call(provider.item_error, item_error,), + ], + any_order=True, + ) \ No newline at end of file From a23389ab13a2a8c6310130c5617f3764ef4c2f0b Mon Sep 17 00:00:00 2001 From: Luke Date: Sat, 28 Dec 2019 13:25:26 +1300 Subject: [PATCH 3/4] Add extension for scrapy provider --- python3/raygun4py/extension/__init__.py | 0 python3/raygun4py/extension/scrapy.py | 29 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 python3/raygun4py/extension/__init__.py create mode 100644 python3/raygun4py/extension/scrapy.py diff --git a/python3/raygun4py/extension/__init__.py b/python3/raygun4py/extension/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python3/raygun4py/extension/scrapy.py b/python3/raygun4py/extension/scrapy.py new file mode 100644 index 0000000..679d40e --- /dev/null +++ b/python3/raygun4py/extension/scrapy.py @@ -0,0 +1,29 @@ +from raygun4py import raygunprovider +from scrapy import signals +from scrapy.exceptions import NotConfigured + + +class Provider(object): + def __init__(self, api_key): + self.sender = raygunprovider.RaygunSender(api_key) + + @classmethod + def from_crawler(cls, crawler): + api_key = crawler.settings.get("RAYGUN_API_KEY") + if not api_key: + raise NotConfigured + + extension = cls(api_key) + crawler.signals.connect(extension.spider_error, signals.spider_error) + crawler.signals.connect(extension.item_error, signals.item_error) + + return extension + + def _handle_exception(self, failure): + self.sender.send_exception(exc_info=(failure.type, failure.value, failure.tb)) + + def item_error(self, item, response, spider, failure): + self._handle_exception(failure) + + def spider_error(self, failure, response, spider): + self._handle_exception(failure) From 60728e4b9a5b4ca34514a577a34301bdd3a80fb9 Mon Sep 17 00:00:00 2001 From: Luke Date: Sat, 28 Dec 2019 13:59:12 +1300 Subject: [PATCH 4/4] Update README to include scrapy instructions --- README.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.rst b/README.rst index 3b5ec88..91e6154 100644 --- a/README.rst +++ b/README.rst @@ -173,6 +173,21 @@ Attaching raw HTTP request data If you are in a web server environment and have HTTP request details available, you can pass these and the headers through in a dictionary (see :code:`sample.py`). +Scrapy +++++++ + +To configure scrapy to automatically send all exceptions raised in spiders and item pipelines to Raygun: + +settings.py + +.. code:: python + + RAYGUN_API_KEY = 'paste_your_api_key_here' + + EXTENSIONS = { + 'raygun4py.extension.scrapy.Provider': 400 + } + Code running on Google App Engine should now be supported - you can test this locally, and has been reported working once deployed (the latter currently requires a paid account due to needed SSL support). Documentation