Improve destination change support in clone module

- This improves the destination change support for cloning data by providing a Google Drive ID - Remove support for generating the dest_list file by running gen_list.py file - Remove listkeys module - Switch base image to Ubuntu 20.04 - Improve README - Tidy up
l3v11 · Sep 7, 2022 · 378521f · 378521f
1 parent 1846978
commit 378521f
Show file tree

Hide file tree

Showing 11 changed files with 142 additions and 172 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,5 +11,4 @@ log.txt
 accounts/*
 drives.txt
 drive_list
-dest_list
 /temp.py
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM ubuntu:20.04
 
 WORKDIR /usr/src/app
 SHELL ["/bin/bash", "-c"]

diff --git a/README.md b/README.md
@@ -7,33 +7,92 @@
 
 <p align="center">
 
-## SearchX
+# SearchX
 
-> A simple Telegram Bot for searching data on Google Drive. Able to clone data from Google Drive, AppDrive and GDToT. Supports MongoDB for storing authorized users record.
+SearchX is a multipurpose Telegram bot written in Python for Google Drive
 
 </p>
 
 
-### [Features](https://github.com/l3v11/SearchX/wiki/Features)
-
-List of features supported by the bot
-
-### [Manual](https://github.com/l3v11/SearchX/wiki)
-
-Guide for deploying the bot
-
-### [Commands](https://github.com/l3v11/SearchX/wiki/Bot-Commands)
-
-List of commands for the bot
-
-### [Changelog](https://github.com/l3v11/SearchX/wiki/Changelog)
-
-List of changes made to the bot
-
-### [FAQ](https://github.com/l3v11/SearchX/wiki/Frequently-Asked-Questions)
-
-Read this if you have any queries
-
-### [Credits](https://github.com/l3v11/SearchX/wiki/Credits)
-
-List of contributors of the bot
+## Features
+
+- Search data in Google Drive
+- Clone data from Google Drive, AppDrive and GDToT
+- Archive data from Google Drive, AppDrive and GDToT
+- Extract data from Google Drive, AppDrive and GDToT
+- Count data from Google Drive
+- Delete data from Google Drive
+- Set data permission in Google Drive
+- Size Limit support for Clone, Archive and Extraction tasks
+- MongoDB support for storing the list of authorized users
+- Index Link support
+- Multi-token telegraph support
+- Multi-page telegraph page results
+- Service Account (SA) support
+- Execute shell commands
+- Evaluate Python expressions
+- Run Python code
+
+## Configuration
+
+These four files are required to run the bot
+- [credentials.json](https://github.com/l3v11/SearchX/wiki/Getting-the-Configuration-Files#getting-the-credentialsjson-file)
+- [drive_list](https://github.com/l3v11/SearchX/wiki/Getting-the-Configuration-Files#getting-the-drive_list-file)
+- [token.json](https://github.com/l3v11/SearchX/wiki/Getting-the-Configuration-Files#getting-the-tokenjson-file)
+- [config.env](https://github.com/l3v11/SearchX/wiki/Getting-the-Configuration-Files#setting-up-the-configenv-file)
+
+## Deployment
+
+These two guides are available to deploy the bot
+- [Deploying to Heroku](https://github.com/l3v11/SearchX/wiki/Deploying-to-Heroku) (free)
+- [Deploying to VPS](https://github.com/l3v11/SearchX/wiki/Deploying-to-VPS) (paid)
+
+## Commands
+
+This list of commands is supported by the bot
+```
+start - Start the bot
+find - Search data in Google Drive
+clone - Clone data to Google Drive
+archive - Archive data to Google Drive
+extract - Extract data to Google Drive
+count - Count data from Google Drive
+cancel - Cancel a task
+status - Get status of all tasks
+share - Set data permission in Google Drive
+del - Delete data from Google Drive
+authorize - Grant authorization of an user
+unauthorize - Revoke authorization of an user
+users - Get the list of authorized users
+shell - Execute shell commands
+eval - Evaluate Python expressions
+exec - Execute Python code
+clearlocals - Clear the locals
+ping - Ping the bot
+stats - Get the system statistics
+log - Get the log file
+restart - Restart the bot
+help - Get help
+```
+
+## Changelog
+
+> [**Click here**](https://github.com/l3v11/SearchX/wiki/Changelog) to see the list of changes made to the bot
+
+## FAQ
+
+> [**Click here**](https://github.com/l3v11/SearchX/wiki/Frequently-Asked-Questions) to read the answers to 
+some of the most common questions or problems users come across
+
+## Credits
+
+[Levi](https://github.com/l3v11) *(Maintainer)* |
+[Shivam Jha](https://github.com/lzzy12) |
+[Sreeraj V R](https://github.com/SVR666) |
+[Kaizoku](https://github.com/animekaizoku) |
+[Snape](https://github.com/snape541) |
+[Anas](https://github.com/anasty17) |
+[Yuuki](https://github.com/xcscxr) |
+[Hrutvik](https://github.com/hsj51) |
+[SpeedX](https://github.com/SpeedyIndeedy) |
+[Agamya Samuel](https://github.com/agamya-samuel)
diff --git a/bot/__init__.py b/bot/__init__.py
@@ -54,7 +54,6 @@ def get_config(name: str):
 DRIVE_IDS = []
 INDEX_URLS = []
 TELEGRAPH = []
-DEST_DRIVES = {}
 
 AUTHORIZED_CHATS = set()
 
@@ -211,22 +210,6 @@ def get_config(name: str):
 except:
     pass
 
-try:
-    DEST_LIST_URL = get_config('DEST_LIST_URL')
-    if len(DEST_LIST_URL) == 0:
-        raise KeyError
-    try:
-        res = requests.get(DEST_LIST_URL)
-        if res.status_code == 200:
-            with open('dest_list', 'wb+') as f:
-                f.write(res.content)
-        else:
-            LOGGER.error(f"Failed to load dest_list file [{res.status_code}]")
-    except Exception as e:
-        LOGGER.error(f"DEST_LIST_URL: {e}")
-except:
-    pass
-
 try:
     APPDRIVE_EMAIL = get_config('APPDRIVE_EMAIL')
     APPDRIVE_PASS = get_config('APPDRIVE_PASS')
@@ -258,13 +241,6 @@ def get_config(name: str):
             except IndexError:
                 INDEX_URLS.append(None)
 
-if os.path.exists('dest_list'):
-    with open('dest_list', 'r+') as f:
-        lines = f.readlines()
-        for line in lines:
-            line = line.strip().split()
-            DEST_DRIVES[line[0]] = line[1:]
-
 def create_account(sname):
     try:
         telegra_ph = Telegraph()

diff --git a/bot/__main__.py b/bot/__main__.py
@@ -6,7 +6,7 @@
 from sys import executable
 from telegram.ext import CommandHandler
 
-from bot import bot, LOGGER, botStartTime, AUTHORIZED_CHATS, DEST_DRIVES, TELEGRAPH, Interval, dispatcher, updater
+from bot import bot, LOGGER, botStartTime, AUTHORIZED_CHATS, TELEGRAPH, Interval, dispatcher, updater
 from bot.modules import auth, cancel, clone, compress, count, delete, eval, list, permission, shell, status
 from bot.helper.ext_utils.bot_utils import get_readable_file_size, get_readable_time
 from bot.helper.ext_utils.fs_utils import start_cleanup, clean_all, exit_clean_up
@@ -24,12 +24,6 @@ def start(update, context):
     else:
         sendMessage("<b>Access denied</b>", context.bot, update.message)
 
-def listkeys(update, context):
-    keys = ''
-    keys += '\n'.join(f"• <code>{key}</code>" for key in DEST_DRIVES.keys())
-    msg = f"<b><u>Available Keys</u></b>\n{keys}"
-    sendMessage(msg, context.bot, update.message)
-
 def ping(update, context):
     start_time = int(round(time.time() * 1000))
     reply = sendMessage("<b>Pong!</b>", context.bot, update.message)
@@ -74,9 +68,9 @@ def restart(update, context):
 <br><br>
 • <b>/{BotCommands.StartCommand}</b>: Start the bot
 <br><br>
-• <b>/{BotCommands.ListCommand}</b> &lt;query&gt;: Search data on Google Drive
+• <b>/{BotCommands.ListCommand}</b> &lt;query&gt;: Search data in Google Drive
 <br><br>
-• <b>/{BotCommands.CloneCommand}</b> &lt;url&gt; &lt;key&gt;: Copy data from Google Drive, AppDrive and GDToT (Key optional)
+• <b>/{BotCommands.CloneCommand}</b> &lt;url&gt; &lt;dest_id&gt;: Clone data from Google Drive, AppDrive and GDToT (Destination ID optional)
 <br><br>
 • <b>/{BotCommands.ArchiveCommand}</b> &lt;url&gt;: Archive data from Google Drive, AppDrive and GDToT
 <br><br>
@@ -88,8 +82,6 @@ def restart(update, context):
 <br><br>
 • <b>/{BotCommands.StatusCommand}</b>: Get status of all tasks
 <br><br>
-• <b>/{BotCommands.ListKeysCommand}</b>: Get the list of destination drives keys
-<br><br>
 • <b>/{BotCommands.PingCommand}</b>: Ping the bot
 <br><br>
 • <b>/{BotCommands.StatsCommand}</b>: Get the system statistics
@@ -106,23 +98,23 @@ def restart(update, context):
 help_string_admin = f'''
 <b><u>Admin Commands</u></b>
 <br><br>
-• <b>/{BotCommands.PermissionCommand}</b> &lt;drive_url&gt; &lt;email&gt;: Set data permission on Google Drive (Email optional)
+• <b>/{BotCommands.PermissionCommand}</b> &lt;drive_url&gt; &lt;email&gt;: Set data permission in Google Drive (Email optional)
 <br><br>
 • <b>/{BotCommands.DeleteCommand}</b> &lt;drive_url&gt;: Delete data from Google Drive
 <br><br>
-• <b>/{BotCommands.AuthorizeCommand}</b>: Authorize an user or a chat for using the bot
+• <b>/{BotCommands.AuthorizeCommand}</b>: Grant authorization of an user
 <br><br>
-• <b>/{BotCommands.UnauthorizeCommand}</b>: Unauthorize an user or a chat for using the bot
+• <b>/{BotCommands.UnauthorizeCommand}</b>: Revoke authorization of an user
 <br><br>
 • <b>/{BotCommands.UsersCommand}</b>: Get the list of authorized users
 <br><br>
-• <b>/{BotCommands.ShellCommand}</b> &lt;cmd&gt;: Run commands in terminal
+• <b>/{BotCommands.ShellCommand}</b> &lt;cmd&gt;: Execute shell commands
 <br><br>
 • <b>/{BotCommands.EvalCommand}</b>: Evaluate Python expressions using eval() function
 <br><br>
 • <b>/{BotCommands.ExecCommand}</b>: Execute Python code using exec() function
 <br><br>
-• <b>/{BotCommands.ClearLocalsCommand}</b>: Clear locals of eval() and exec() functions
+• <b>/{BotCommands.ClearLocalsCommand}</b>: Clear the locals of eval() and exec() functions
 <br><br>
 • <b>/{BotCommands.LogCommand}</b>: Get the log file
 <br><br>
@@ -150,8 +142,6 @@ def main():
         os.remove(".restartmsg")
 
     start_handler = CommandHandler(BotCommands.StartCommand, start, run_async=True)
-    keys_handler = CommandHandler(BotCommands.ListKeysCommand, listkeys,
-                                  filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True)
     ping_handler = CommandHandler(BotCommands.PingCommand, ping,
                                   filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True)
     stats_handler = CommandHandler(BotCommands.StatsCommand, stats,
@@ -163,7 +153,6 @@ def main():
     help_handler = CommandHandler(BotCommands.HelpCommand, bot_help,
                                   filters=CustomFilters.authorized_chat | CustomFilters.authorized_user, run_async=True)
     dispatcher.add_handler(start_handler)
-    dispatcher.add_handler(keys_handler)
     dispatcher.add_handler(ping_handler)
     dispatcher.add_handler(stats_handler)
     dispatcher.add_handler(log_handler)

diff --git a/bot/helper/drive_utils/gdriveTools.py b/bot/helper/drive_utils/gdriveTools.py
@@ -21,7 +21,7 @@
 from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload
 
 from bot import LOGGER, DRIVE_NAMES, DRIVE_IDS, INDEX_URLS, PARENT_ID, \
-    IS_TEAM_DRIVE, TELEGRAPH, USE_SERVICE_ACCOUNTS, INDEX_URL, DEST_DRIVES
+    IS_TEAM_DRIVE, TELEGRAPH, USE_SERVICE_ACCOUNTS, INDEX_URL
 from bot.helper.ext_utils.bot_utils import SetInterval, get_readable_file_size
 from bot.helper.ext_utils.fs_utils import get_mime_type
 from bot.helper.telegram_helper.button_builder import ButtonMaker
@@ -370,7 +370,7 @@ def __cloneFolder(self, name, local_path, folder_id, parent_id):
             if self.__is_cancelled:
                 break
 
-    def clone(self, link, key):
+    def clone(self, link, dest_id):
         self.__is_cloning = True
         self.__start_time = time.time()
         self.__total_files = 0
@@ -383,12 +383,9 @@ def clone(self, link, key):
             msg = "Drive ID not found"
             LOGGER.error(msg)
             return msg
-        if key in DEST_DRIVES:
-            parent_id = DEST_DRIVES[key][0]
-            try:
-                index_url = DEST_DRIVES[key][1]
-            except IndexError:
-                index_url = None
+        if dest_id != "":
+            parent_id = dest_id
+            index_url = None
         msg = ""
         try:
             meta = self.__getFileMetadata(file_id)

diff --git a/bot/helper/telegram_helper/bot_commands.py b/bot/helper/telegram_helper/bot_commands.py
@@ -13,7 +13,6 @@ def __init__(self):
         self.AuthorizeCommand = 'authorize'
         self.UnauthorizeCommand = 'unauthorize'
         self.UsersCommand = 'users'
-        self.ListKeysCommand = 'listkeys'
         self.ShellCommand = 'shell'
         self.EvalCommand = 'eval'
         self.ExecCommand = 'exec'

diff --git a/bot/modules/clone.py b/bot/modules/clone.py
@@ -20,17 +20,17 @@ def cloneNode(update, context):
     args = update.message.text.split()
     reply_to = update.message.reply_to_message
     link = ''
-    key = ''
+    dest_id = ''
     if len(args) > 1:
         link = args[1].strip()
         try:
-            key = args[2].strip()
+            dest_id = args[2].strip()
         except IndexError:
             pass
     if reply_to:
         link = reply_to.text.split(maxsplit=1)[0].strip()
         try:
-            key = args[1].strip()
+            dest_id = args[1].strip()
         except IndexError:
             pass
     is_appdrive = is_appdrive_link(link)
@@ -47,7 +47,7 @@ def cloneNode(update, context):
             deleteMessage(context.bot, msg)
         except DDLExceptionHandler as e:
             deleteMessage(context.bot, msg)
-            LOGGER.error(e)
+            LOGGER.error(str(e))
             return sendMessage(str(e), context.bot, update.message)
     if is_gdrive_link(link):
         msg = sendMessage(f"<b>Checking:</b> <code>{link}</code>", context.bot, update.message)
@@ -67,7 +67,7 @@ def cloneNode(update, context):
         if files <= 20:
             msg = sendMessage(f"<b>Cloning:</b> <code>{link}</code>", context.bot, update.message)
             LOGGER.info(f"Cloning: {link}")
-            result = gd.clone(link, key)
+            result = gd.clone(link, dest_id)
             deleteMessage(context.bot, msg)
         else:
             drive = GoogleDriveHelper(name)
@@ -77,7 +77,7 @@ def cloneNode(update, context):
                 download_dict[update.message.message_id] = clone_status
             sendStatusMessage(update.message, context.bot)
             LOGGER.info(f"Cloning: {link}")
-            result = drive.clone(link, key)
+            result = drive.clone(link, dest_id)
             with download_dict_lock:
                 del download_dict[update.message.message_id]
                 count = len(download_dict)
@@ -101,7 +101,6 @@ def cloneNode(update, context):
     else:
         help_msg = '<b><u>Instructions</u></b>\nSend a link along with command'
         help_msg += '\n\n<b><u>Supported Sites</u></b>\n• Google Drive\n• AppDrive\n• GDToT'
-        help_msg += '\n\n<b><u>Set Destination Drive</u></b>\nAdd &lt;key&gt; after the link'
         sendMessage(help_msg, context.bot, update.message)
 
 clone_handler = CommandHandler(BotCommands.CloneCommand, cloneNode,

diff --git a/bot/modules/compress.py b/bot/modules/compress.py
@@ -225,7 +225,7 @@ def _compress(bot, message, is_archive=False, is_extract=False, pswd=None):
             deleteMessage(bot, msg)
         except DDLExceptionHandler as e:
             deleteMessage(bot, msg)
-            LOGGER.error(e)
+            LOGGER.error(str(e))
             return sendMessage(str(e), bot, message)
     listener = CompressListener(bot, message, is_archive, is_extract, pswd)
     if is_gdrive_link(link):

diff --git a/config_sample.env b/config_sample.env
@@ -16,7 +16,6 @@ COMPRESS_LIMIT=
 TOKEN_JSON_URL=
 ACCOUNTS_ZIP_URL=
 DRIVE_LIST_URL=
-DEST_LIST_URL=
 APPDRIVE_EMAIL=
 APPDRIVE_PASS=
 GDTOT_CRYPT=