From c1a0023da923b0dd2a867ff5d8a2dab73600f701 Mon Sep 17 00:00:00 2001 From: Laudanum <86309764+Laudanum-dev@users.noreply.github.com> Date: Thu, 24 Jun 2021 14:35:29 +0500 Subject: [PATCH 1/5] added argparse to compile_packages.py added argparse support to compile_packages.py. also rewrote a bit of a code in directory-searching functions + some code "reformat" by pycharm --- compile_packages.py | 129 +++++++++++++++++++++++++++++--------------- 1 file changed, 85 insertions(+), 44 deletions(-) diff --git a/compile_packages.py b/compile_packages.py index 1c9cc0598f..0930c3ddbd 100644 --- a/compile_packages.py +++ b/compile_packages.py @@ -10,6 +10,7 @@ from zipfile import ZipFile, ZIP_LZMA, ZIP_DEFLATED from srctools import Property, KeyValError, VMF, Entity, conv_bool +import argparse OPTIMISE = False @@ -33,15 +34,17 @@ def clean_vmf(vmf_path): print('Removing hammer_notes...') inst.remove_ent(ent) continue - + # All instances must be in bee2/, so any reference outside there is a map error! # It's ok if it's in p2editor and not in a subfolder though. # There's also an exception needed for the Tag gun instance. if ent['classname'] == 'func_instance': - inst_loc = ent['file'].casefold().replace('\\','/') - if not inst_loc.startswith('instances/bee2/') and not (inst_loc.startswith('instances/p2editor/') and inst_loc.count('/') == 2) and 'alatag' not in inst_loc: + inst_loc = ent['file'].casefold().replace('\\', '/') + if not inst_loc.startswith('instances/bee2/') and not ( + inst_loc.startswith('instances/p2editor/') and inst_loc.count( + '/') == 2) and 'alatag' not in inst_loc: input('Invalid instance path "{}" in\n"{}"! Press Enter to continue..'.format(ent['file'], vmf_path)) - yield from clean_vmf(vmf_path) # Re-run so we check again.. + yield from clean_vmf(vmf_path) # Re-run so we check again.. for solid in ent.solids[:]: if all(face.mat.casefold() == 'tools/toolsskip' for face in solid): @@ -79,29 +82,31 @@ def clean_vmf(vmf_path): DELETE_EXTENSIONS = ['vmx', 'log', 'bsp', 'prt', 'lin'] -def search_folder(zip_path, path): - """Search the given folder for packages. +def search_list_of_dirs(list_of_dirs, zip_path): + """Search the given list of folders for packages zip_path is the folder the zips will be saved in, - and path is the location to search. + and list_of_dirs is the list of locations to search. """ - for package in os.listdir(path): - package_path = os.path.join(path, package) - if not os.path.isdir(package_path): + for dir_path in list_of_dirs: + if not os.path.isdir(dir_path): # it's a file continue - if 'info.txt' not in os.listdir(package_path): - yield from search_folder(zip_path, package_path) + if 'info.txt' not in os.listdir(dir_path): # it's a folder, probably with packages + # go search its contents + yield from search_list_of_dirs([os.path.join(dir_path, i) for i in os.listdir(dir_path)], zip_path) continue - print('| ' + package + '.bee_pack') - pack_zip_path = os.path.join(zip_path, package) + '.bee_pack' + # do not preserve original file structure, dump all found packs in root of zip_path + pack_name = os.path.split(dir_path)[-1] + pack_zip_path = os.path.join(zip_path, pack_name) + ".bee_pack" + print('| ' + pack_name + '.bee_pack') + + yield dir_path, pack_zip_path - yield package_path, pack_zip_path, zip_path - -def build_package(package_path, pack_zip_path, zip_path): +def build_package(package_path, pack_zip_path): """Build the packages in a given folder.""" - + zip_file = ZipFile( pack_zip_path, 'w', @@ -159,41 +164,77 @@ def build_package(package_path, pack_zip_path, zip_path): def main(): + parser = argparse.ArgumentParser(description="This is package compiler, which can compress packages " + "in order for them to be lighter. " + "Also provides an option to optimize them.\n" + "IMPORTANT NOTE: your packages should NOT be zipped before " + "compilation, also they should all have different names, " + "since they all will be dumped in the same directory.") + parser.add_argument("input", nargs="+", + help="Specifies an input path or several input paths. If an input path is a " + "single package, then it will be compiled, otherwise, if an input path is " + "a folder, then it will be recursively searched for packages, and will compile " + "all it finds") + parser.add_argument("-op", "--optimize", action="store_const", const=True, default=False, + help="Will optimize zips (recommended).", dest="optimize") + parser.add_argument("-c", "--confirm", action="store_const", const=True, default=False, + help="Will skip a confirmation prompt.", dest="skip_confirm") + parser.add_argument("-o", "--output", default=None, + help="Will specify an output folder, otherwise \"./zips\" will be used.", dest="output") + parser.add_argument("--zip", nargs="?", default=None, const="", + help="Will put all generated files in one zip. Also skips the prompt at the end. " + "Using this option with a string, followed after it, will create a zip with a " + "specified name. Using this option without a string will just skip prompt " + "without creating zip. Not using this option will generate a prompt at the end", + dest="zip") + + args = parser.parse_args() + inp_list = args.input + output = args.output global OPTIMISE - - OPTIMISE = conv_bool(input('Optimise zips? ')) - - print('Optimising: ', OPTIMISE) - - zip_path = os.path.join( - os.getcwd(), - 'zips', - 'sml' if OPTIMISE else 'lrg', - ) + OPTIMISE = args.optimize + do_zip = args.zip + if not args.skip_confirm: + print("You specified these folders:") + print("\n".join(inp_list)) + print("These will be optimized" if OPTIMISE else "These will NOT be optimized") + if not conv_bool(input("Continue? (y/n) ")): + sys.exit(0) + + if output is None: + zip_path = os.path.join( + os.getcwd(), + 'zips', + 'sml' if OPTIMISE else 'lrg', + ) + else: + zip_path = output if os.path.isdir(zip_path): - for file in os.listdir(zip_path): - print('Deleting', file) - os.remove(os.path.join(zip_path, file)) + if os.listdir(zip_path): + raise ValueError("The output directory is not empty") else: os.makedirs(zip_path, exist_ok=True) - shutil.rmtree('zips/hammer/', ignore_errors=True) - - path = os.path.join(os.getcwd(), 'packages\\', ) - # A list of all the package zips. - for package in search_folder(zip_path, path): + for package in search_list_of_dirs(inp_list, zip_path): build_package(*package) + print('Done!') + + pack_name = "" + if do_zip is None: + if conv_bool(input("Zip it all in one file? (y/n) ")): + pack_name = 'BEE{}_packages.zip'.format(input('Version: ')) + else: + pack_name = do_zip + if pack_name != "": + print('Building main zip...') - print('Building main zip...') + with ZipFile(os.path.join('zips', pack_name), 'w', compression=ZIP_DEFLATED) as zip_file: + for file in os.listdir(zip_path): + zip_file.write(os.path.join(zip_path, file), os.path.join('packages/', file)) + print('.', end='', flush=True) + print('Done!') - pack_name = 'BEE{}_packages.zip'.format(input('Version: ')) - - with ZipFile(os.path.join('zips', pack_name), 'w', compression=ZIP_DEFLATED) as zip_file: - for file in os.listdir(zip_path): - zip_file.write(os.path.join(zip_path, file), os.path.join('packages/', file)) - print('.', end='', flush=True) - print('Done!') if __name__ == '__main__': main() From dc69ffb8f494e96e4231f7dd167319824b4540e8 Mon Sep 17 00:00:00 2001 From: Laudanum <86309764+Laudanum-dev@users.noreply.github.com> Date: Thu, 24 Jun 2021 20:14:18 +0500 Subject: [PATCH 2/5] updated compile_packages.py, fixed not working when optimizing enabled. --- compile_packages.py | 73 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 13 deletions(-) diff --git a/compile_packages.py b/compile_packages.py index 0930c3ddbd..96eb4261cb 100644 --- a/compile_packages.py +++ b/compile_packages.py @@ -87,6 +87,7 @@ def search_list_of_dirs(list_of_dirs, zip_path): zip_path is the folder the zips will be saved in, and list_of_dirs is the list of locations to search. + returns pairs (original_file_path, result_zip_path) """ for dir_path in list_of_dirs: if not os.path.isdir(dir_path): # it's a file @@ -170,15 +171,20 @@ def main(): "IMPORTANT NOTE: your packages should NOT be zipped before " "compilation, also they should all have different names, " "since they all will be dumped in the same directory.") + # argument that accepts one or more filepaths that then will be recursively searched and + # compiled all found stuff parser.add_argument("input", nargs="+", help="Specifies an input path or several input paths. If an input path is a " "single package, then it will be compiled, otherwise, if an input path is " "a folder, then it will be recursively searched for packages, and will compile " "all it finds") + # will give args.optimize True value is specified, False otherwise; parser.add_argument("-op", "--optimize", action="store_const", const=True, default=False, help="Will optimize zips (recommended).", dest="optimize") + # will give args.skip_confirm True value is specified, False otherwise; parser.add_argument("-c", "--confirm", action="store_const", const=True, default=False, help="Will skip a confirmation prompt.", dest="skip_confirm") + # will specify an output path. if not specified, args.output will be set to None, and nothing will be moved then parser.add_argument("-o", "--output", default=None, help="Will specify an output folder, otherwise \"./zips\" will be used.", dest="output") parser.add_argument("--zip", nargs="?", default=None, const="", @@ -197,43 +203,84 @@ def main(): if not args.skip_confirm: print("You specified these folders:") print("\n".join(inp_list)) + # shows list with all files specified (in case using CLI is not + # so intuitive), also tells if you have chosen to optimize output (below) print("These will be optimized" if OPTIMISE else "These will NOT be optimized") if not conv_bool(input("Continue? (y/n) ")): - sys.exit(0) - - if output is None: - zip_path = os.path.join( - os.getcwd(), - 'zips', - 'sml' if OPTIMISE else 'lrg', - ) - else: - zip_path = output + sys.exit(0) # confirmation just in case + + # will dump all temporary files to 'zips' folder. also, all compiled packages will be left in + # zips/sml or zips/lrg depending on whether you chose to optimize it or not + zip_path = os.path.join( + os.getcwd(), + 'zips', + 'sml' if OPTIMISE else 'lrg', + ) + + # if there is no such dir, then create it. if there is, and its empty, + # then use it. if there is and its not empty - throw error (since whole directory will be erased) if os.path.isdir(zip_path): if os.listdir(zip_path): raise ValueError("The output directory is not empty") else: os.makedirs(zip_path, exist_ok=True) - # A list of all the package zips. + # yield will return found packages one by one, so it is easy to iterate them + # without the need to process everything at once for package in search_list_of_dirs(inp_list, zip_path): build_package(*package) print('Done!') + # confusing section, determining whether it need to create main zip and how to name it if not specified + # makes it blank by default pack_name = "" if do_zip is None: if conv_bool(input("Zip it all in one file? (y/n) ")): pack_name = 'BEE{}_packages.zip'.format(input('Version: ')) else: - pack_name = do_zip + pack_name = do_zip # do_zip also can be blank + # if pack_name remained blank to this stage, then no zip will be created if pack_name != "": print('Building main zip...') - with ZipFile(os.path.join('zips', pack_name), 'w', compression=ZIP_DEFLATED) as zip_file: for file in os.listdir(zip_path): zip_file.write(os.path.join(zip_path, file), os.path.join('packages/', file)) print('.', end='', flush=True) print('Done!') + if output is not None: + print("Moving zip to output destination") + shutil.move(os.path.join('zips', pack_name), output) + print('Done!') + if not args.preserve_temp: + print("Deleting temporary files") + shutil.rmtree("zips") + print("Done!") + else: + if not args.preserve_temp: + print("Deleting temporary files") + shutil.rmtree("zips/hammer") + if OPTIMISE: + shutil.rmtree("zips/sml") + else: + shutil.rmtree("zips/lrg") + print("Done!") + else: + # dont make zip + if output is not None: + print("Moving files to output destination") + if OPTIMISE: + shutil.move("zips/sml", output) + else: + shutil.move("zips/lrg", output) + if not args.preserve_temp: + print("Deleting temporary files") + shutil.rmtree("zips") + print("Done!") + else: + if not args.preserve_temp: + print("Deleting temporary files") + shutil.rmtree("zips/hammer") + print("Done!") if __name__ == '__main__': From 650f06c0a8c7b088d336f54cd8fb6e45c34aa989 Mon Sep 17 00:00:00 2001 From: Laudanum <86309764+Laudanum-dev@users.noreply.github.com> Date: Thu, 24 Jun 2021 20:16:56 +0500 Subject: [PATCH 3/5] forgot to push that sry --- compile_packages.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/compile_packages.py b/compile_packages.py index 96eb4261cb..836bf5bf6f 100644 --- a/compile_packages.py +++ b/compile_packages.py @@ -187,13 +187,19 @@ def main(): # will specify an output path. if not specified, args.output will be set to None, and nothing will be moved then parser.add_argument("-o", "--output", default=None, help="Will specify an output folder, otherwise \"./zips\" will be used.", dest="output") + # since after moving output files to output destination (specified above), there remains lot of junk files + # that looks like was for debugging and its safe to delete them. specifying this option will prevent that. + parser.add_argument("-p", "--preserve-temp", action="store_const", const=True, default=False, + help="Will not delete temporary files", dest="preserve_temp") + # if NOT specified, then args.zip will be set to None, if specified BUT NOT set to any string value, then + # args.zip will be set to "" (empty string). Later, if args.zip remains set to "", then no zip will be created parser.add_argument("--zip", nargs="?", default=None, const="", help="Will put all generated files in one zip. Also skips the prompt at the end. " "Using this option with a string, followed after it, will create a zip with a " "specified name. Using this option without a string will just skip prompt " "without creating zip. Not using this option will generate a prompt at the end", dest="zip") - + # if there is something left inside parse_args() brackets, then probably i forgot to remove after debugging args = parser.parse_args() inp_list = args.input output = args.output From b9c0f4b56f8ae1ebcb46f1832b177fa98346f6a8 Mon Sep 17 00:00:00 2001 From: Laudanum <86309764+Laudanum-dev@users.noreply.github.com> Date: Fri, 25 Jun 2021 10:06:07 +0500 Subject: [PATCH 4/5] changed text so it is says that it creates some useful hammer files during compilation, and may not delete them if specified --- compile_packages.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/compile_packages.py b/compile_packages.py index 836bf5bf6f..e6361a5d23 100644 --- a/compile_packages.py +++ b/compile_packages.py @@ -180,7 +180,8 @@ def main(): "all it finds") # will give args.optimize True value is specified, False otherwise; parser.add_argument("-op", "--optimize", action="store_const", const=True, default=False, - help="Will optimize zips (recommended).", dest="optimize") + help="Will optimize zips (not recommended and may make packages " + "unloadable for bee in current version).", dest="optimize") # will give args.skip_confirm True value is specified, False otherwise; parser.add_argument("-c", "--confirm", action="store_const", const=True, default=False, help="Will skip a confirmation prompt.", dest="skip_confirm") @@ -190,7 +191,8 @@ def main(): # since after moving output files to output destination (specified above), there remains lot of junk files # that looks like was for debugging and its safe to delete them. specifying this option will prevent that. parser.add_argument("-p", "--preserve-temp", action="store_const", const=True, default=False, - help="Will not delete temporary files", dest="preserve_temp") + help="Will not delete temporary and hammer files (you probably won't need hammer files " + "if you are not planning to use hammer)", dest="preserve_temp") # if NOT specified, then args.zip will be set to None, if specified BUT NOT set to any string value, then # args.zip will be set to "" (empty string). Later, if args.zip remains set to "", then no zip will be created parser.add_argument("--zip", nargs="?", default=None, const="", @@ -258,12 +260,12 @@ def main(): shutil.move(os.path.join('zips', pack_name), output) print('Done!') if not args.preserve_temp: - print("Deleting temporary files") + print("Deleting temporary and hammer files") shutil.rmtree("zips") print("Done!") else: if not args.preserve_temp: - print("Deleting temporary files") + print("Deleting temporary and hammer files") shutil.rmtree("zips/hammer") if OPTIMISE: shutil.rmtree("zips/sml") @@ -279,12 +281,12 @@ def main(): else: shutil.move("zips/lrg", output) if not args.preserve_temp: - print("Deleting temporary files") + print("Deleting temporary and hammer files") shutil.rmtree("zips") print("Done!") else: if not args.preserve_temp: - print("Deleting temporary files") + print("Deleting hammer files") shutil.rmtree("zips/hammer") print("Done!") From d14aa4717335a5a6a159aa67e46e80369070221f Mon Sep 17 00:00:00 2001 From: Laudanum <86309764+Laudanum-dev@users.noreply.github.com> Date: Sat, 26 Jun 2021 23:35:39 +0500 Subject: [PATCH 5/5] now no file is treated as broad "temporary", now it is hammer file, or (yet) unzipped compiled package. also upgrade to CLI, now at prompt at the beginning, it will state all chosen options. not using temporary folders anymore. fixed some small issues, added comments --- compile_packages.py | 120 +++++++++++++++++++++++--------------------- 1 file changed, 64 insertions(+), 56 deletions(-) diff --git a/compile_packages.py b/compile_packages.py index e6361a5d23..7c6f36d7cf 100644 --- a/compile_packages.py +++ b/compile_packages.py @@ -13,6 +13,8 @@ import argparse OPTIMISE = False +# path to a folder, where all useful hammer files will be created, if it stated that they need to be created +HAMMER_PATH = None def clean_vmf(vmf_path): @@ -107,6 +109,7 @@ def search_list_of_dirs(list_of_dirs, zip_path): def build_package(package_path, pack_zip_path): """Build the packages in a given folder.""" + global HAMMER_PATH zip_file = ZipFile( pack_zip_path, @@ -141,8 +144,16 @@ def build_package(package_path, pack_zip_path): # Skip music files.. hammer_path = None else: - hammer_path = os.path.join('zips/hammer/', hammer_path) - os.makedirs(os.path.dirname(hammer_path), exist_ok=True) + # if this files can be added to HAMMER_PATH folder, + # check if this folder was specified (because if its not, + # then you should not create hammer files) + if HAMMER_PATH is not None: + hammer_path = os.path.join(HAMMER_PATH, hammer_path) + os.makedirs(os.path.dirname(hammer_path), exist_ok=True) + else: + hammer_path = None + # yea, if at this point hammer_path is None, then file we currently looking at do not + # need to be put into a HAMMER_PATH folder (including every case) print('.', end='', flush=True) @@ -186,13 +197,13 @@ def main(): parser.add_argument("-c", "--confirm", action="store_const", const=True, default=False, help="Will skip a confirmation prompt.", dest="skip_confirm") # will specify an output path. if not specified, args.output will be set to None, and nothing will be moved then - parser.add_argument("-o", "--output", default=None, + parser.add_argument("-o", "--output", default="zips", help="Will specify an output folder, otherwise \"./zips\" will be used.", dest="output") - # since after moving output files to output destination (specified above), there remains lot of junk files - # that looks like was for debugging and its safe to delete them. specifying this option will prevent that. - parser.add_argument("-p", "--preserve-temp", action="store_const", const=True, default=False, - help="Will not delete temporary and hammer files (you probably won't need hammer files " - "if you are not planning to use hammer)", dest="preserve_temp") + # during packing files, it can create several useful hammer files, which will be placed in specified directory, + # or won't be created if not specified + parser.add_argument("-hp", "--hammer_path", default=None, + help="Will create some useful files, which you can use in hammer later. " + "All files will be stored in specified directory.", dest="hammer") # if NOT specified, then args.zip will be set to None, if specified BUT NOT set to any string value, then # args.zip will be set to "" (empty string). Later, if args.zip remains set to "", then no zip will be created parser.add_argument("--zip", nargs="?", default=None, const="", @@ -205,25 +216,28 @@ def main(): args = parser.parse_args() inp_list = args.input output = args.output - global OPTIMISE + global OPTIMISE, HAMMER_PATH OPTIMISE = args.optimize do_zip = args.zip + HAMMER_PATH = args.hammer if not args.skip_confirm: - print("You specified these folders:") - print("\n".join(inp_list)) - # shows list with all files specified (in case using CLI is not - # so intuitive), also tells if you have chosen to optimize output (below) - print("These will be optimized" if OPTIMISE else "These will NOT be optimized") - if not conv_bool(input("Continue? (y/n) ")): + # shows list with all files specified (in case using CLI is not so intuitive) + print("You specified these folders:\n-------------------------------") + for inp in inp_list: + print("|", inp) + print("-------------------------------") + print("Selected options:") + print("* Optimization: " + ("ON" if OPTIMISE else "OFF")) + print("* Output folder: \"" + output + "\"") + print("* Hammer folder: " + ("OFF" if (HAMMER_PATH is None) else ("\"" + HAMMER_PATH + "\""))) + print("* Final zip: " + ("skip prompt" if do_zip == "" else + ("not skip prompt" if do_zip is None else + do_zip))) + print() + if not conv_bool(input("---> Continue? (y/n) ")): sys.exit(0) # confirmation just in case - # will dump all temporary files to 'zips' folder. also, all compiled packages will be left in - # zips/sml or zips/lrg depending on whether you chose to optimize it or not - zip_path = os.path.join( - os.getcwd(), - 'zips', - 'sml' if OPTIMISE else 'lrg', - ) + zip_path = output # if there is no such dir, then create it. if there is, and its empty, # then use it. if there is and its not empty - throw error (since whole directory will be erased) @@ -233,9 +247,22 @@ def main(): else: os.makedirs(zip_path, exist_ok=True) + # if hammer path was specified then + if HAMMER_PATH: + # if there is no such dir, then create it. if there is, and its empty, + # then use it. if there is and its not empty - throw error (why not) + if os.path.isdir(HAMMER_PATH): + if os.listdir(HAMMER_PATH): + raise ValueError("The hammer files directory is not empty") + else: + os.makedirs(HAMMER_PATH, exist_ok=True) + + packages_list = [] # list with filenames of all packages # yield will return found packages one by one, so it is easy to iterate them # without the need to process everything at once for package in search_list_of_dirs(inp_list, zip_path): + # package is a tuple (raw_package_directory, path_to_compiled_version) + packages_list.append(os.path.relpath(package[1], zip_path)) build_package(*package) print('Done!') @@ -247,48 +274,29 @@ def main(): pack_name = 'BEE{}_packages.zip'.format(input('Version: ')) else: pack_name = do_zip # do_zip also can be blank + # if its not blank, and not ending with .zip, then add .zip to filename + if pack_name and pack_name[-4:] != ".zip": + pack_name += ".zip" # if pack_name remained blank to this stage, then no zip will be created if pack_name != "": print('Building main zip...') - with ZipFile(os.path.join('zips', pack_name), 'w', compression=ZIP_DEFLATED) as zip_file: - for file in os.listdir(zip_path): + with ZipFile(os.path.join(zip_path, pack_name), 'w', compression=ZIP_DEFLATED) as zip_file: + for file in packages_list: + # previously it put in zip all contains of a folder, including itself (no clue why it + # haven't raised any error), also, since hammer files do not need to appear in zip, + # packages_list variable was created zip_file.write(os.path.join(zip_path, file), os.path.join('packages/', file)) print('.', end='', flush=True) print('Done!') - if output is not None: - print("Moving zip to output destination") - shutil.move(os.path.join('zips', pack_name), output) - print('Done!') - if not args.preserve_temp: - print("Deleting temporary and hammer files") - shutil.rmtree("zips") - print("Done!") - else: - if not args.preserve_temp: - print("Deleting temporary and hammer files") - shutil.rmtree("zips/hammer") - if OPTIMISE: - shutil.rmtree("zips/sml") - else: - shutil.rmtree("zips/lrg") - print("Done!") + print("Deleting unzipped files...") + # removes all files that are in packages_list var + for file in packages_list: + os.remove(os.path.join(zip_path, file)) + print("Done!") else: # dont make zip - if output is not None: - print("Moving files to output destination") - if OPTIMISE: - shutil.move("zips/sml", output) - else: - shutil.move("zips/lrg", output) - if not args.preserve_temp: - print("Deleting temporary and hammer files") - shutil.rmtree("zips") - print("Done!") - else: - if not args.preserve_temp: - print("Deleting hammer files") - shutil.rmtree("zips/hammer") - print("Done!") + # we dont actually need to do anything else + pass if __name__ == '__main__':