From 2f865fb97ff44d2a51709bb1ea504683ea87f44e Mon Sep 17 00:00:00 2001 From: Levi <57452819+l3v11@users.noreply.github.com> Date: Tue, 13 Dec 2022 14:03:58 +0600 Subject: [PATCH] Remove AppDrive support - This removes support for cloning, archiving and extracting data from AppDrive since the website admin has added Google reCAPTCHA on the login page and it can't be bypassed as of now --- bot/__init__.py | 6 --- bot/__main__.py | 6 +-- bot/helper/download_utils/ddl_generator.py | 62 +--------------------- bot/helper/download_utils/gd_downloader.py | 8 +-- bot/helper/drive_utils/gdriveTools.py | 2 +- bot/helper/ext_utils/bot_utils.py | 4 -- bot/modules/archive.py | 19 +++---- bot/modules/clone.py | 26 +++------ config_sample.env | 2 - requirements.txt | 1 - 10 files changed, 21 insertions(+), 115 deletions(-) diff --git a/bot/__init__.py b/bot/__init__.py index 5a1de24c..c4e93aa3 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -151,12 +151,6 @@ except Exception as err: LOGGER.error(f"DRIVE_LIST_URL: {err}") -APPDRIVE_EMAIL = os.environ.get('APPDRIVE_EMAIL', '') -APPDRIVE_PASS = os.environ.get('APPDRIVE_PASS', '') -if len(APPDRIVE_EMAIL) == 0 or len(APPDRIVE_PASS) == 0: - APPDRIVE_EMAIL = None - APPDRIVE_PASS = None - GDTOT_CRYPT = os.environ.get('GDTOT_CRYPT', '') if len(GDTOT_CRYPT) == 0: GDTOT_CRYPT = None diff --git a/bot/__main__.py b/bot/__main__.py index 3c81bf77..9f9ff3c3 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -70,11 +70,11 @@ def restart(update, context):

• /{BotCommands.ListCommand} <query>: Search data in Google Drive

-• /{BotCommands.CloneCommand} <url> <dest_id>: Clone data from Google Drive, AppDrive and GDToT (Destination ID optional) +• /{BotCommands.CloneCommand} <url> <dest_id>: Clone data from Google Drive and GDToT (Destination ID optional)

-• /{BotCommands.CompressCommand} <url>: Compress data from Google Drive, AppDrive and GDToT +• /{BotCommands.CompressCommand} <url>: Compress data from Google Drive and GDToT

-• /{BotCommands.ExtractCommand} <url>: Extract data from Google Drive, AppDrive and GDToT +• /{BotCommands.ExtractCommand} <url>: Extract data from Google Drive and GDToT

• /{BotCommands.CountCommand} <drive_url>: Count data from Google Drive

diff --git a/bot/helper/download_utils/ddl_generator.py b/bot/helper/download_utils/ddl_generator.py index e8cab2f5..d0d5306a 100644 --- a/bot/helper/download_utils/ddl_generator.py +++ b/bot/helper/download_utils/ddl_generator.py @@ -2,71 +2,11 @@ import re import requests -from lxml import etree from urllib.parse import urlparse, parse_qs -from bot import APPDRIVE_EMAIL, APPDRIVE_PASS, GDTOT_CRYPT +from bot import GDTOT_CRYPT from bot.helper.ext_utils.exceptions import DDLExceptionHandler -def account_login(client, url, email, password): - data = { - 'email': email, - 'password': password - } - client.post(f'https://{urlparse(url).netloc}/login', data=data) - -def gen_payload(data, boundary=f'{"-"*6}_'): - data_string = '' - for item in data: - data_string += f'{boundary}\r\n' - data_string += f'Content-Disposition: form-data; name="{item}"\r\n\r\n{data[item]}\r\n' - data_string += f'{boundary}--\r\n' - return data_string - -def appdrive(url: str) -> str: - if (APPDRIVE_EMAIL or APPDRIVE_PASS) is None: - raise DDLExceptionHandler("APPDRIVE_EMAIL and APPDRIVE_PASS env vars not provided") - client = requests.Session() - client.headers.update({ - "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36" - }) - account_login(client, url, APPDRIVE_EMAIL, APPDRIVE_PASS) - res = client.get(url) - try: - key = re.findall(r'"key",\s+"(.*?)"', res.text)[0] - except IndexError: - raise DDLExceptionHandler("Invalid link") - ddl_btn = etree.HTML(res.content).xpath("//button[@id='drc']") - info = {} - info['error'] = False - info['link_type'] = 'login' # direct/login - headers = { - "Content-Type": f"multipart/form-data; boundary={'-'*4}_", - } - data = { - 'type': 1, - 'key': key, - 'action': 'original' - } - if len(ddl_btn): - info['link_type'] = 'direct' - data['action'] = 'direct' - while data['type'] <= 3: - try: - response = client.post(url, data=gen_payload(data), headers=headers).json() - break - except: - data['type'] += 1 - if 'url' in response: - info['gdrive_link'] = response['url'] - elif 'error' in response and response['error']: - info['error'] = True - info['message'] = response['message'] - if not info['error']: - return info - else: - raise DDLExceptionHandler(f"{info['message']}") - def gdtot(url: str) -> str: if GDTOT_CRYPT is None: raise DDLExceptionHandler("GDTOT_CRYPT env var not provided") diff --git a/bot/helper/download_utils/gd_downloader.py b/bot/helper/download_utils/gd_downloader.py index a4eab4d0..2528aab9 100644 --- a/bot/helper/download_utils/gd_downloader.py +++ b/bot/helper/download_utils/gd_downloader.py @@ -7,7 +7,7 @@ from bot.helper.telegram_helper.message_utils import sendMessage, deleteMessage, sendStatusMessage from bot.helper.ext_utils.bot_utils import get_readable_file_size -def add_gd_download(link, path, listener, customname, is_appdrive, appdict, is_gdtot): +def add_gd_download(link, path, listener, customname, is_gdtot): msg = sendMessage(f"Checking: {link}", listener.bot, listener.message) LOGGER.info(f"Checking: {link}") gd = GoogleDriveHelper() @@ -32,10 +32,6 @@ def add_gd_download(link, path, listener, customname, is_appdrive, appdict, is_g download_dict[listener.uid] = download_status sendStatusMessage(listener.message, listener.bot) drive.download(link) - if is_appdrive: - if appdict.get('link_type') == 'login': - LOGGER.info(f"Deleting: {link}") - drive.deleteFile(link) - elif is_gdtot: + if is_gdtot: LOGGER.info(f"Deleting: {link}") drive.deleteFile(link) diff --git a/bot/helper/drive_utils/gdriveTools.py b/bot/helper/drive_utils/gdriveTools.py index 62fbcaf5..da9a56a1 100644 --- a/bot/helper/drive_utils/gdriveTools.py +++ b/bot/helper/drive_utils/gdriveTools.py @@ -594,7 +594,7 @@ def upload(self, file_name: str): if isinstance(err, RetryError): LOGGER.info(f"Total attempts: {err.last_attempt.attempt_number}") err = err.last_attempt.exception() - self.__listener.onUploadError(err) + self.__listener.onUploadError(str(err)) self.__is_errored = True finally: self.__updater.cancel() diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index 04c30e46..5e0c9566 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -123,10 +123,6 @@ def is_url(url: str): def is_gdrive_link(url: str): return "drive.google.com" in url -def is_appdrive_link(url: str): - url = re.match(r'https?://appdrive\.\S+', url) - return bool(url) - def is_gdtot_link(url: str): url = re.match(r'https?://.+\.gdtot\.\S+', url) return bool(url) diff --git a/bot/modules/archive.py b/bot/modules/archive.py index 72db3496..ab5862cc 100644 --- a/bot/modules/archive.py +++ b/bot/modules/archive.py @@ -9,10 +9,10 @@ from telegram.ext import CommandHandler from bot import LOGGER, dispatcher, DOWNLOAD_DIR, Interval, INDEX_URL, download_dict, download_dict_lock, status_reply_dict_lock -from bot.helper.download_utils.ddl_generator import appdrive, gdtot +from bot.helper.download_utils.ddl_generator import gdtot from bot.helper.download_utils.gd_downloader import add_gd_download from bot.helper.drive_utils.gdriveTools import GoogleDriveHelper -from bot.helper.ext_utils.bot_utils import is_gdrive_link, is_appdrive_link, is_gdtot_link +from bot.helper.ext_utils.bot_utils import is_gdrive_link, is_gdtot_link from bot.helper.ext_utils.exceptions import ArchiveExceptionHandler, DDLExceptionHandler from bot.helper.ext_utils.fs_utils import clean_download, clean_target, get_base_name, get_path_size from bot.helper.status_utils.compress_status import CompressStatus @@ -189,9 +189,7 @@ def _archive(bot, message, is_compress=False, is_extract=False): mesg = message.text.split('\n') message_args = mesg[0].split(maxsplit=1) name_arg = mesg[0].split('|', maxsplit=1) - is_appdrive = False is_gdtot = False - appdict = '' if len(message_args) > 1: link = message_args[1].strip() if link.startswith("pswd:"): @@ -214,17 +212,12 @@ def _archive(bot, message, is_compress=False, is_extract=False): reply_to = message.reply_to_message if reply_to is not None: link = reply_to.text.split(maxsplit=1)[0].strip() - is_appdrive = is_appdrive_link(link) is_gdtot = is_gdtot_link(link) - if any([is_appdrive, is_gdtot]): + if is_gdtot: msg = sendMessage(f"Processing: {link}", bot, message) LOGGER.info(f"Processing: {link}") try: - if is_appdrive: - appdict = appdrive(link) - link = appdict.get('gdrive_link') - if is_gdtot: - link = gdtot(link) + link = gdtot(link) deleteMessage(bot, msg) except DDLExceptionHandler as err: deleteMessage(bot, msg) @@ -232,10 +225,10 @@ def _archive(bot, message, is_compress=False, is_extract=False): return sendMessage(str(err), bot, message) listener = ArchiveListener(bot, message, is_compress, is_extract, pswd) if is_gdrive_link(link): - threading.Thread(target=add_gd_download, args=(link, f'{DOWNLOAD_DIR}{listener.uid}', listener, name, is_appdrive, appdict, is_gdtot)).start() + threading.Thread(target=add_gd_download, args=(link, f'{DOWNLOAD_DIR}{listener.uid}', listener, name, is_gdtot)).start() else: help_msg = 'Instructions\nSend a link along with command' - help_msg += '\n\nSupported Sites\n• Google Drive\n• AppDrive\n• GDToT' + help_msg += '\n\nSupported Sites\n• Google Drive\n• GDToT' help_msg += '\n\nSet Custom Name\nAdd "|customname" after the link' help_msg += '\n\nSet Password\nAdd "pswd: xxx" after the link' sendMessage(help_msg, bot, message) diff --git a/bot/modules/clone.py b/bot/modules/clone.py index 6cee1e85..7b7a9210 100644 --- a/bot/modules/clone.py +++ b/bot/modules/clone.py @@ -4,14 +4,13 @@ from telegram.ext import CommandHandler from bot import LOGGER, dispatcher, CLONE_LIMIT, download_dict, download_dict_lock, Interval -from bot.helper.download_utils.ddl_generator import appdrive, gdtot +from bot.helper.download_utils.ddl_generator import gdtot from bot.helper.drive_utils.gdriveTools import GoogleDriveHelper -from bot.helper.ext_utils.bot_utils import new_thread, get_readable_file_size, is_gdrive_link, \ - is_appdrive_link, is_gdtot_link +from bot.helper.ext_utils.bot_utils import new_thread, get_readable_file_size, is_gdrive_link, is_gdtot_link from bot.helper.ext_utils.exceptions import DDLExceptionHandler from bot.helper.status_utils.clone_status import CloneStatus -from bot.helper.telegram_helper.message_utils import sendMessage, editMessage, deleteMessage, \ - delete_all_messages, update_all_messages, sendStatusMessage +from bot.helper.telegram_helper.message_utils import sendMessage, editMessage, deleteMessage, delete_all_messages, \ + update_all_messages, sendStatusMessage from bot.helper.telegram_helper.bot_commands import BotCommands from bot.helper.telegram_helper.filters import CustomFilters @@ -33,17 +32,12 @@ def cloneNode(update, context): dest_id = args[1].strip() except IndexError: pass - is_appdrive = is_appdrive_link(link) is_gdtot = is_gdtot_link(link) - if any([is_appdrive, is_gdtot]): + if is_gdtot: msg = sendMessage(f"Processing: {link}", context.bot, update.message) LOGGER.info(f"Processing: {link}") try: - if is_appdrive: - appdict = appdrive(link) - link = appdict.get('gdrive_link') - if is_gdtot: - link = gdtot(link) + link = gdtot(link) deleteMessage(context.bot, msg) except DDLExceptionHandler as err: deleteMessage(context.bot, msg) @@ -91,16 +85,12 @@ def cloneNode(update, context): except IndexError: pass sendMessage(result, context.bot, update.message) - if is_appdrive: - if appdict.get('link_type') == 'login': - LOGGER.info(f"Deleting: {link}") - gd.deleteFile(link) - elif is_gdtot: + if is_gdtot: LOGGER.info(f"Deleting: {link}") gd.deleteFile(link) else: help_msg = 'Instructions\nSend a link along with command' - help_msg += '\n\nSupported Sites\n• Google Drive\n• AppDrive\n• GDToT' + help_msg += '\n\nSupported Sites\n• Google Drive\n• GDToT' sendMessage(help_msg, context.bot, update.message) clone_handler = CommandHandler(BotCommands.CloneCommand, cloneNode, diff --git a/config_sample.env b/config_sample.env index 6525c425..b16c8d8e 100644 --- a/config_sample.env +++ b/config_sample.env @@ -16,6 +16,4 @@ CLONE_LIMIT= TOKEN_JSON_URL= ACCOUNTS_ZIP_URL= DRIVE_LIST_URL= -APPDRIVE_EMAIL= -APPDRIVE_PASS= GDTOT_CRYPT= diff --git a/requirements.txt b/requirements.txt index af8aa102..96d431f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ dnspython google-api-python-client google-auth-oauthlib -lxml psutil pymongo python-dotenv