Skip to content

Commit

Permalink
Add Bookmark support for storing Destination Drive IDs into database
Browse files Browse the repository at this point in the history
- This improves the destination change support for cloning data by providing a bookmarked drive key
- Change the authorize command of auth module from /authorize to /auth
- Change the unauthorize command of auth module from /unauthorize to /unauth
- Fix bugs
- Tidy up
  • Loading branch information
l3v11 authored Dec 20, 2022
1 parent 2f865fb commit 7ff4611
Show file tree
Hide file tree
Showing 20 changed files with 193 additions and 133 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ data*
.idea
*.json
*.pickle
*.zip
log.txt
accounts/*
drives.txt
drive_list
fly.toml
/temp.py
4 changes: 3 additions & 1 deletion bot/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
DRIVE_IDS = []
INDEX_URLS = []
TELEGRAPH = []
BOOKMARKS = {}

download_dict_lock = Lock()
status_reply_dict_lock = Lock()
Expand Down Expand Up @@ -184,6 +185,7 @@ def create_account(sname):
create_account(sname)
LOGGER.info(f"Generated {TELEGRAPH_ACCS} telegraph tokens")

updater = tg.Updater(token=BOT_TOKEN, use_context=True)
tgDefaults = tg.Defaults(parse_mode='HTML', allow_sending_without_reply=True, disable_web_page_preview=True, run_async=True)
updater = tg.Updater(token=BOT_TOKEN, defaults=tgDefaults, use_context=True)
bot = updater.bot
dispatcher = updater.dispatcher
28 changes: 17 additions & 11 deletions bot/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@
from telegram.ext import CommandHandler

from bot import bot, LOGGER, botStartTime, TELEGRAPH, Interval, dispatcher, updater
from bot.modules import archive, auth, cancel, clone, count, delete, eval, list, permission, shell, status
from bot.modules import archive, auth, bookmark, cancel, clone, count, delete, eval, list, permission, shell, status
from bot.helper.ext_utils.bot_utils import get_readable_file_size, get_readable_time
from bot.helper.ext_utils.fs_utils import start_cleanup, clean_all, exit_clean_up
from bot.helper.telegram_helper.bot_commands import BotCommands
from bot.helper.telegram_helper.button_builder import ButtonMaker
from bot.helper.telegram_helper.filters import CustomFilters
from bot.helper.telegram_helper.message_utils import sendMessage, sendMarkup, editMessage, sendLogFile
from bot.helper.telegram_helper.message_utils import sendMessage, editMessage, sendLogFile

def start(update, context):
if CustomFilters.authorized_user(update) or CustomFilters.authorized_chat(update):
Expand Down Expand Up @@ -70,7 +70,7 @@ def restart(update, context):
<br><br>
• <b>/{BotCommands.ListCommand}</b> &lt;query&gt;: Search data in Google Drive
<br><br>
• <b>/{BotCommands.CloneCommand}</b> &lt;url&gt; &lt;dest_id&gt;: Clone data from Google Drive and GDToT (Destination ID optional)
• <b>/{BotCommands.CloneCommand}</b> &lt;url&gt; &lt;drive_key&gt;: Clone data from Google Drive and GDToT (Drive Key optional)
<br><br>
• <b>/{BotCommands.CompressCommand}</b> &lt;url&gt;: Compress data from Google Drive and GDToT
<br><br>
Expand All @@ -82,6 +82,8 @@ def restart(update, context):
<br><br>
• <b>/{BotCommands.StatusCommand}</b>: Get status of all tasks
<br><br>
• <b>/{BotCommands.BookmarksCommand}</b>: Get the list of bookmarked destination drives
<br><br>
• <b>/{BotCommands.PingCommand}</b>: Ping the bot
<br><br>
• <b>/{BotCommands.StatsCommand}</b>: Get the system statistics
Expand All @@ -102,6 +104,10 @@ def restart(update, context):
<br><br>
• <b>/{BotCommands.DeleteCommand}</b> &lt;drive_url&gt;: Delete data from Google Drive
<br><br>
• <b>/{BotCommands.AddBookmarkCommand}</b> &lt;drive_key&gt; &lt;drive_id&gt;: Add bookmark of a destination drive
<br><br>
• <b>/{BotCommands.RemBookmarkCommand}</b> &lt;drive_key&gt;: Remove bookmark of a destination drive
<br><br>
• <b>/{BotCommands.AuthorizeCommand}</b>: Grant authorization of an user
<br><br>
• <b>/{BotCommands.UnauthorizeCommand}</b>: Revoke authorization of an user
Expand Down Expand Up @@ -131,30 +137,30 @@ def bot_help(update, context):
button = ButtonMaker()
button.build_button("User", f"https://graph.org/{help_user}")
button.build_button("Admin", f"https://graph.org/{help_admin}")
sendMarkup(help_string, context.bot, update.message, button.build_menu(2))
sendMessage(help_string, context.bot, update.message, button.build_menu(2))

def main():
start_cleanup()
if os.path.isfile(".restartmsg"):
with open(".restartmsg") as f:
chat_id, msg_id = map(int, f)
try:
bot.editMessageText("<b>Restarted successfully</b>", chat_id, msg_id, parse_mode='HTML')
bot.editMessageText("<b>Restarted successfully</b>", chat_id, msg_id)
except:
pass
os.remove(".restartmsg")

start_handler = CommandHandler(BotCommands.StartCommand, start, run_async=True)
start_handler = CommandHandler(BotCommands.StartCommand, start)
ping_handler = CommandHandler(BotCommands.PingCommand, ping,
filters=CustomFilters.authorized_user | CustomFilters.authorized_chat, run_async=True)
filters=CustomFilters.authorized_user | CustomFilters.authorized_chat)
stats_handler = CommandHandler(BotCommands.StatsCommand, stats,
filters=CustomFilters.authorized_user | CustomFilters.authorized_chat, run_async=True)
filters=CustomFilters.authorized_user | CustomFilters.authorized_chat)
log_handler = CommandHandler(BotCommands.LogCommand, log,
filters=CustomFilters.owner_filter, run_async=True)
filters=CustomFilters.owner_filter)
restart_handler = CommandHandler(BotCommands.RestartCommand, restart,
filters=CustomFilters.owner_filter, run_async=True)
filters=CustomFilters.owner_filter)
help_handler = CommandHandler(BotCommands.HelpCommand, bot_help,
filters=CustomFilters.authorized_user | CustomFilters.authorized_chat, run_async=True)
filters=CustomFilters.authorized_user | CustomFilters.authorized_chat)
dispatcher.add_handler(start_handler)
dispatcher.add_handler(ping_handler)
dispatcher.add_handler(stats_handler)
Expand Down
7 changes: 4 additions & 3 deletions bot/helper/drive_utils/gdriveTools.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,13 +343,12 @@ def __copyFile(self, file_id, dest_id):
if reason in ['userRateLimitExceeded', 'dailyLimitExceeded']:
if USE_SERVICE_ACCOUNTS:
if self.__sa_count == SERVICE_ACCOUNTS_NUMBER:
self.__is_cancelled = True
LOGGER.info("SA switch limit exceeded")
raise err
else:
self.__switchServiceAccount()
return self.__copyFile(file_id, dest_id)
else:
self.__is_cancelled = True
LOGGER.error(f"Warning: {reason}")
raise err
else:
Expand Down Expand Up @@ -621,6 +620,8 @@ def __download_file(self, file_id, path, filename, mime_type):
filename = f"{filename[:245]}{ext}"
if self.name.endswith(ext):
self.name = filename
if self.__is_cancelled:
return
fh = FileIO(f"{path}/{filename}", 'wb')
downloader = MediaIoBaseDownload(fh, request, chunksize=50 * 1024 * 1024)
done = False
Expand All @@ -637,7 +638,7 @@ def __download_file(self, file_id, path, filename, mime_type):
raise err
if USE_SERVICE_ACCOUNTS:
if self.__sa_count == SERVICE_ACCOUNTS_NUMBER:
self.__is_cancelled = True
LOGGER.info("SA switch limit exceeded")
raise err
else:
self.__switchServiceAccount()
Expand Down
3 changes: 1 addition & 2 deletions bot/helper/ext_utils/bot_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,7 @@ def get_progress_bar_string(status):
cFull = p // 8
p_str = '⬤' * cFull
p_str += '○' * (12 - cFull)
p_str = f"「{p_str}」"
return p_str
return f"「{p_str}」"

def get_readable_message():
with download_dict_lock:
Expand Down
43 changes: 30 additions & 13 deletions bot/helper/ext_utils/database.py
Original file line number Diff line number Diff line change
@@ -1,47 +1,64 @@
from pymongo import MongoClient
from pymongo.errors import PyMongoError

from bot import LOGGER, AUTHORIZED_USERS, DATABASE_URL
from bot import LOGGER, AUTHORIZED_USERS, BOOKMARKS, DATABASE_URL

class DatabaseHelper:

def __init__(self):
self.__err = False
self.__client = None
self.__db = None
self.__collection = None
self.__connect()

def __connect(self):
try:
self.__client = MongoClient(DATABASE_URL)
self.__db = self.__client['SearchX']
self.__collection = self.__db['users']
except PyMongoError as err:
LOGGER.error(err)
self.__err = True

def auth_user(self, user_id: int):
def load_db(self):
if self.__err:
return
self.__collection.insert_one({"user_id": user_id})
return 'Authorization granted'
if self.__db.users.find_one():
users_dict = self.__db.users.find().sort("user_id")
for user in users_dict:
AUTHORIZED_USERS.add(user["user_id"])
if self.__db.bms.find_one():
bms_dict = self.__db.bms.find().sort("drive_key")
for bm in bms_dict:
BOOKMARKS[bm["drive_key"]] = str(bm["drive_id"])
self.__client.close()

def auth_user(self, user_id):
if self.__err:
return
self.__db.users.insert_one({"user_id": user_id})
self.__client.close()
return 'Authorization granted'

def unauth_user(self, user_id: int):
def unauth_user(self, user_id):
if self.__err:
return
self.__collection.delete_many({"user_id": user_id})
self.__db.users.delete_one({"user_id": user_id})
self.__client.close()
return 'Authorization revoked'

def add_bm(self, drive_key, drive_id):
if self.__err:
return
self.__db.bms.insert_one({"drive_key": drive_key, "drive_id": drive_id})
self.__client.close()
return 'Bookmark added'

def load_users(self):
def remove_bm(self, drive_key):
if self.__err:
return
users = self.__collection.find().sort("user_id")
for user in users:
AUTHORIZED_USERS.add(user["user_id"])
self.__db.bms.delete_one({"drive_key": drive_key})
self.__client.close()
return 'Bookmark removed'

if DATABASE_URL is not None:
DatabaseHelper().load_users()
DatabaseHelper().load_db()
7 changes: 5 additions & 2 deletions bot/helper/telegram_helper/bot_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,11 @@ def __init__(self):
self.StatusCommand = 'status'
self.PermissionCommand = 'share'
self.DeleteCommand = 'del'
self.AuthorizeCommand = 'authorize'
self.UnauthorizeCommand = 'unauthorize'
self.AddBookmarkCommand = 'addbm'
self.RemBookmarkCommand = 'rembm'
self.BookmarksCommand = 'bookmarks'
self.AuthorizeCommand = 'auth'
self.UnauthorizeCommand = 'unauth'
self.UsersCommand = 'users'
self.ShellCommand = 'shell'
self.EvalCommand = 'eval'
Expand Down
35 changes: 9 additions & 26 deletions bot/helper/telegram_helper/message_utils.py
Original file line number Diff line number Diff line change
@@ -1,46 +1,29 @@
import time

from telegram import InlineKeyboardMarkup
from telegram.error import RetryAfter

from bot import bot, LOGGER, Interval, STATUS_UPDATE_INTERVAL, \
status_reply_dict, status_reply_dict_lock
from bot.helper.ext_utils.bot_utils import SetInterval, get_readable_message

def sendMessage(text, bot, message):
def sendMessage(text, bot, message, reply_markup=None):
try:
return bot.sendMessage(message.chat_id,
return bot.sendMessage(chat_id=message.chat_id,
reply_to_message_id=message.message_id,
text=text, parse_mode='HTML',
disable_web_page_preview=True)
text=text, reply_markup=reply_markup)
except RetryAfter as r:
LOGGER.warning(str(r))
time.sleep(r.retry_after * 1.5)
return sendMessage(text, bot, message)
except Exception as err:
LOGGER.error(str(err))
return

def sendMarkup(text, bot, message, reply_markup: InlineKeyboardMarkup):
try:
return bot.sendMessage(message.chat_id,
reply_to_message_id=message.message_id,
text=text, reply_markup=reply_markup,
parse_mode='HTML', disable_web_page_preview=True)
except RetryAfter as r:
LOGGER.warning(str(r))
time.sleep(r.retry_after * 1.5)
return sendMarkup(text, bot, message, reply_markup)
return sendMessage(text, bot, message, reply_markup)
except Exception as err:
LOGGER.error(str(err))
return

def editMessage(text, message, reply_markup=None):
try:
bot.editMessageText(text=text, message_id=message.message_id,
chat_id=message.chat.id,
reply_markup=reply_markup, parse_mode='HTML',
disable_web_page_preview=True)
bot.editMessageText(chat_id=message.chat.id,
message_id=message.message_id,
text=text, reply_markup=reply_markup)
except RetryAfter as r:
LOGGER.warning(str(r))
time.sleep(r.retry_after * 1.5)
Expand All @@ -59,8 +42,8 @@ def deleteMessage(bot, message):
def sendLogFile(bot, message):
with open('log.txt', 'rb') as f:
bot.sendDocument(document=f, filename=f.name,
reply_to_message_id=message.message_id,
chat_id=message.chat_id)
chat_id=message.chat_id,
reply_to_message_id=message.message_id)

def delete_all_messages():
with status_reply_dict_lock:
Expand Down
32 changes: 15 additions & 17 deletions bot/modules/archive.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,27 +188,25 @@ def onUploadError(self, error):
def _archive(bot, message, is_compress=False, is_extract=False):
mesg = message.text.split('\n')
message_args = mesg[0].split(maxsplit=1)
name_arg = mesg[0].split('|', maxsplit=1)
is_gdtot = False
link = ''
if len(message_args) > 1:
link = message_args[1].strip()
if link.startswith("pswd:"):
if link.startswith(('|', 'pswd:')):
link = ''
else:
link = ''
if len(name_arg) > 1:
name = name_arg[1]
name = name.split(' pswd:')[0]
name = name.strip()
name = mesg[0].split('|', maxsplit=1)
if len(name) > 1:
if 'pswd:' in name[0]:
name = ''
else:
name = name[1].split('pswd:')[0].strip()
else:
name = ''
link = re.split(r"pswd:|\|", link)[0]
link = link.strip()
pswd_arg = mesg[0].split(' pswd: ')
if len(pswd_arg) > 1:
pswd = pswd_arg[1]
else:
pswd = None
pswd = mesg[0].split(' pswd: ')
pswd = pswd[1] if len(pswd) > 1 else None
if link != '':
link = re.split(r'pswd:|\|', link)[0]
link = link.strip()
reply_to = message.reply_to_message
if reply_to is not None:
link = reply_to.text.split(maxsplit=1)[0].strip()
Expand Down Expand Up @@ -241,8 +239,8 @@ def extract_data(update, context):
_archive(context.bot, update.message, is_extract=True)

compress_handler = CommandHandler(BotCommands.CompressCommand, compress_data,
filters=CustomFilters.authorized_user | CustomFilters.authorized_chat, run_async=True)
filters=CustomFilters.authorized_user | CustomFilters.authorized_chat)
extract_handler = CommandHandler(BotCommands.ExtractCommand, extract_data,
filters=CustomFilters.authorized_user | CustomFilters.authorized_chat, run_async=True)
filters=CustomFilters.authorized_user | CustomFilters.authorized_chat)
dispatcher.add_handler(compress_handler)
dispatcher.add_handler(extract_handler)
Loading

0 comments on commit 7ff4611

Please sign in to comment.