Improved captcha handling

README.md

@@ -83,6 +83,8 @@ optional arguments:
   --chromedriver CHROMEDRIVER
                         Path to a specific chromedriver executable instead of
                         the built-in one
+  --no-ublock           Disable the uBlock Chrome extension. Might be needed
+                        to solve captcha
   -v, --verbose         Increases logging verbosity
 ```
 

bin/ublock/ublock-settings.txt

@@ -62,7 +62,7 @@
     "wyciwyg-scheme"
   ],
   "netWhitelist": "about-scheme\nchrome-extension-scheme\nchrome-scheme\nmoz-extension-scheme\nopera-scheme\nvivaldi-scheme\nwyciwyg-scheme",
-  "dynamicFilteringString": "behind-the-scene * * noop\nbehind-the-scene * inline-script noop\nbehind-the-scene * 1p-script noop\nbehind-the-scene * 3p-script noop\nbehind-the-scene * 3p-frame noop\nbehind-the-scene * image noop\nbehind-the-scene * 3p noop\n* * 3p block\nwww.blinkist.com blinkist.io * allow\nwww.blinkist.com d17pjsg7x52x9r.cloudfront.net * allow\nwww.blinkist.com jsdelivr.net * block\nwww.blinkist.com hcaptcha.com * block",
+  "dynamicFilteringString": "behind-the-scene * * noop\nbehind-the-scene * inline-script noop\nbehind-the-scene * 1p-script noop\nbehind-the-scene * 3p-script noop\nbehind-the-scene * 3p-frame noop\nbehind-the-scene * image noop\nbehind-the-scene * 3p noop\n* * 3p block\nwww.blinkist.com blinkist.io * allow\nwww.blinkist.com d17pjsg7x52x9r.cloudfront.net * allow\nwww.blinkist.com jsdelivr.net * block\nwww.blinkist.com hcaptcha.com * allow",
   "urlFilteringString": "",
   "hostnameSwitchesString": "no-large-media: behind-the-scene false\nno-remote-fonts: * true",
   "userFilters": ""

blinkistscraper/__main__.py

@@ -179,6 +179,12 @@ def check_cooldown(value):
         "--chromedriver",
         help="Path to a specific chromedriver executable instead of the built-in one",
     )
+    parser.add_argument(
+        "--no-ublock",
+        action="store_true",
+        default=False,
+        help="Disable the uBlock Chrome extension. Might be needed to solve captcha",
+    )
     parser.add_argument(
         "-v", "--verbose", action="store_true", help="Increases logging verbosity"
     )
@@ -274,7 +280,7 @@ def finish(start_time, processed_books, driver=None):
         match_language = args.language if args.match_language else ""
         start_headless = args.headless
         # add uBlock (except on headless)
-        use_ublock = not args.headless
+        use_ublock = not args.no_ublock and not args.headless
         driver = scraper.initialize_driver(
             headless=start_headless,
             with_ublock=use_ublock,

blinkistscraper/scraper.py

@@ -131,29 +131,53 @@ def initialize_driver(headless=True, with_ublock=False, chromedriver_path=None):
 
 def login(driver, language, email, password):
     # we need to navigate to a page first in order to load eventual cookies
-    driver.get(f"https://www.blinkist.com/{language}")
+    driver.get(f"https://www.blinkist.com/{language}/nc/login")
     is_logged_in = False
 
     # if we have any stored login cookie, load them into the driver
     if has_login_cookies():
         load_login_cookies(driver)
+
+	# assume that a captcha needs to be solved, if no blinkist logo appears within 5sec
+    try:
+        WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located(
+                (By.CLASS_NAME, "header__logo")
+            )
+        )
+    except TimeoutException as ex:
+        log.info("Please solve captcha to proceed!")
+
+	# fail if captcha not solved within 60sec
+    try:
+        WebDriverWait(driver, 60).until(
+            EC.presence_of_element_located(
+                (By.CLASS_NAME, "header__logo")
+            )
+        )
+    except TimeoutException as ex:
+        log.error("Error. Captcha needs to be solved within 1 minute")
+        return False
 
-    # navigate to the login page and check for the login email input
-    # if not found, assume we're logged in
+    # navigate to the login page
     sign_in_url = f"https://www.blinkist.com/{language}/nc/login"
     driver.get(sign_in_url)
+
+	# click on cookie banner, if necessary
+    time.sleep(1.0)
+    try:
+        cookiebanner = driver.find_element_by_class_name("cookie-disclaimer__cta")
+    except:
+        pass
+    else:
+        cookiebanner.click()
+
+	# check for the login email input. if not found, assume we're logged in
     try:
         driver.find_element_by_id("login-form_login_email")
     except NoSuchElementException:
         is_logged_in = True
-    # try:
-    #     WebDriverWait(driver, 360).until(
-    #         EC.presence_of_element_located((By.ID, "login-form_login_email"))
-    #     )
-    # except TimeoutException as ex:
-    #     log.error("Error logging in.")
-    #     return False
-
+
     # if not logged in, autofill the email and password inputs with the
     # provided login credentials
     if not is_logged_in: