From ebfea5c1216cea4dce55f0be243f6aba968606ed Mon Sep 17 00:00:00 2001 From: Steve Messick Date: Fri, 23 Aug 2024 19:31:23 +0000 Subject: [PATCH] cleanup --- kaggle/api/kaggle_api_extended.py | 64 ++++++++-------- kaggle/cli.py | 4 +- src/kaggle/api/kaggle_api_extended.py | 104 +++++++++++++------------- 3 files changed, 87 insertions(+), 85 deletions(-) diff --git a/kaggle/api/kaggle_api_extended.py b/kaggle/api/kaggle_api_extended.py index 6ec55ae..e24e0aa 100644 --- a/kaggle/api/kaggle_api_extended.py +++ b/kaggle/api/kaggle_api_extended.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2024 Kaggle Inc +# Copyright 2019 Kaggle Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -242,16 +242,16 @@ def cleanup(self): def to_dict(self): return { 'path': - self.path, + self.path, 'start_blob_upload_request': - self.start_blob_upload_request.to_dict(), + self.start_blob_upload_request.to_dict(), 'timestamp': - self.timestamp, + self.timestamp, 'start_blob_upload_response': - self.start_blob_upload_response.to_dict() - if self.start_blob_upload_response is not None else None, + self.start_blob_upload_response.to_dict() + if self.start_blob_upload_response is not None else None, 'upload_complete': - self.upload_complete, + self.upload_complete, } def from_dict(other, context): @@ -420,7 +420,7 @@ def authenticate(self): config_data = self.read_config_file(config_data) elif self._is_help_or_version_command(api_command) or (len( sys.argv) > 2 and api_command.startswith( - self.command_prefixes_allowing_anonymous_access)): + self.command_prefixes_allowing_anonymous_access)): # Some API commands should be allowed without authentication. return else: @@ -428,7 +428,7 @@ def authenticate(self): ' {}. Or use the environment method. See setup' ' instructions at' ' https://github.com/Kaggle/kaggle-api/'.format( - self.config_file, self.config_dir)) + self.config_file, self.config_dir)) # Step 3: load into configuration! self._load_config(config_data) @@ -1759,10 +1759,10 @@ def dataset_create_version(self, self.process_response( self.with_retry( self.datasets_create_version_by_id_with_http_info)( - id_no, request))) + id_no, request))) else: if ref == self.config_values[ - self.CONFIG_NAME_USER] + '/INSERT_SLUG_HERE': + self.CONFIG_NAME_USER] + '/INSERT_SLUG_HERE': raise ValueError( 'Default slug detected, please change values before ' 'uploading') @@ -1880,7 +1880,7 @@ def dataset_create_new(self, # validations if ref == self.config_values[ - self.CONFIG_NAME_USER] + '/INSERT_SLUG_HERE': + self.CONFIG_NAME_USER] + '/INSERT_SLUG_HERE': raise ValueError( 'Default slug detected, please change values before uploading') if title == 'INSERT_TITLE_HERE': @@ -2249,25 +2249,25 @@ def kernels_initialize(self, folder): username = self.get_config_value(self.CONFIG_NAME_USER) meta_data = { 'id': - username + '/INSERT_KERNEL_SLUG_HERE', + username + '/INSERT_KERNEL_SLUG_HERE', 'title': - 'INSERT_TITLE_HERE', + 'INSERT_TITLE_HERE', 'code_file': - 'INSERT_CODE_FILE_PATH_HERE', + 'INSERT_CODE_FILE_PATH_HERE', 'language': - 'Pick one of: {' + - ','.join(x for x in self.valid_push_language_types) + '}', + 'Pick one of: {' + + ','.join(x for x in self.valid_push_language_types) + '}', 'kernel_type': - 'Pick one of: {' + - ','.join(x for x in self.valid_push_kernel_types) + '}', + 'Pick one of: {' + + ','.join(x for x in self.valid_push_kernel_types) + '}', 'is_private': - 'true', + 'true', 'enable_gpu': - 'false', + 'false', 'enable_tpu': - 'false', + 'false', 'enable_internet': - 'true', + 'true', 'dataset_sources': [], 'competition_sources': [], 'kernel_sources': [], @@ -3509,8 +3509,8 @@ def model_instance_version_create(self, self.process_response( self.with_retry( self.models_create_instance_version_with_http_info)( - owner_slug, model_slug, framework, instance_slug, - request))) + owner_slug, model_slug, framework, instance_slug, + request))) return result @@ -3980,9 +3980,9 @@ def upload_files(self, """ for file_name in os.listdir(folder): if (file_name in [ - self.DATASET_METADATA_FILE, self.OLD_DATASET_METADATA_FILE, - self.KERNEL_METADATA_FILE, self.MODEL_METADATA_FILE, - self.MODEL_INSTANCE_METADATA_FILE + self.DATASET_METADATA_FILE, self.OLD_DATASET_METADATA_FILE, + self.KERNEL_METADATA_FILE, self.MODEL_METADATA_FILE, + self.MODEL_INSTANCE_METADATA_FILE ]): continue upload_file = self._upload_file_or_folder(folder, file_name, @@ -4127,10 +4127,10 @@ def upload_complete(self, path, url, quiet, resume=False): fp.seek(start_at) session.headers.update({ 'Content-Length': - '%d' % upload_size, + '%d' % upload_size, 'Content-Range': - 'bytes %d-%d/%d' % - (start_at, file_size - 1, file_size) + 'bytes %d-%d/%d' % + (start_at, file_size - 1, file_size) }) reader = TqdmBufferedReader(fp, progress_bar) retries = Retry(total=10, backoff_factor=0.5) @@ -4312,7 +4312,7 @@ def validate_model_instance_version_string(self, model_instance_version): split = model_instance_version.split('/') if not split[0] or not split[1] or not split[2] or not split[ - 3] or not split[4]: + 3] or not split[4]: raise ValueError( 'Invalid model instance version specification ' + model_instance_version) diff --git a/kaggle/cli.py b/kaggle/cli.py index 2f32625..d248326 100644 --- a/kaggle/cli.py +++ b/kaggle/cli.py @@ -1674,11 +1674,11 @@ class Help(object): command_model_instances_new = 'Create a new model instance' param_model_instance_downfile = ( 'Folder for downloading the special model-instance-metadata.json file ' - '(https://github.com/Kaggle/kaggle-api/wiki/ModelInstance-Metadata). ') + '(https://github.com/Kaggle/kaggle-api/wiki/Model-Metadata#model-instance). ') param_model_instance_upfile = ( 'Folder for upload, containing data files and a ' 'special model-instance-metadata.json file ' - '(https://github.com/Kaggle/kaggle-api/wiki/ModelInstance-Metadata). ' + '(https://github.com/Kaggle/kaggle-api/wiki/Model-Metadata#model-instance). ' 'Defaults to current working directory') command_model_instances_delete = 'Delete a model instance' command_model_instances_update = 'Update a model instance' diff --git a/src/kaggle/api/kaggle_api_extended.py b/src/kaggle/api/kaggle_api_extended.py index c393cd4..fddaa7c 100644 --- a/src/kaggle/api/kaggle_api_extended.py +++ b/src/kaggle/api/kaggle_api_extended.py @@ -226,16 +226,16 @@ def cleanup(self): def to_dict(self): return { 'path': - self.path, + self.path, 'start_blob_upload_request': - self.start_blob_upload_request.to_dict(), + self.start_blob_upload_request.to_dict(), 'timestamp': - self.timestamp, + self.timestamp, 'start_blob_upload_response': - self.start_blob_upload_response.to_dict() - if self.start_blob_upload_response is not None else None, + self.start_blob_upload_response.to_dict() + if self.start_blob_upload_response is not None else None, 'upload_complete': - self.upload_complete, + self.upload_complete, } def from_dict(other, context): @@ -404,7 +404,7 @@ def authenticate(self): config_data = self.read_config_file(config_data) elif self._is_help_or_version_command(api_command) or (len( sys.argv) > 2 and api_command.startswith( - self.command_prefixes_allowing_anonymous_access)): + self.command_prefixes_allowing_anonymous_access)): # Some API commands should be allowed without authentication. return else: @@ -412,7 +412,7 @@ def authenticate(self): ' {}. Or use the environment method. See setup' ' instructions at' ' https://github.com/Kaggle/kaggle-api/'.format( - self.config_file, self.config_dir)) + self.config_file, self.config_dir)) # Step 3: load into configuration! self._load_config(config_data) @@ -680,7 +680,7 @@ def build_kaggle_client(self): return KaggleClient(env=env, verbose=verbose) def camel_to_snake(self, name): - """ TODO Remove this and rewrite field lists using snake case. + """ :param name: field in camel case :return: field in snake case """ @@ -759,11 +759,9 @@ def competitions_list_cli(self, ] if competitions: if csv_display: - self.print_csv(competitions, - [self.camel_to_snake(f) for f in fields]) + self.print_csv(competitions, fields) else: - self.print_table(competitions, - [self.camel_to_snake(f) for f in fields]) + self.print_table(competitions, fields) else: print('No competitions found') @@ -844,7 +842,7 @@ def competition_submit_cli(self, def competition_submissions(self, competition, group=None, - sort=None, + sort=None, # SubmissionSortBy.SUBMISSION_SORT_BY_NAME, page_token=None, page_size=20): """ Get the list of Submission for a particular competition. @@ -898,8 +896,8 @@ def competition_submissions_cli(self, submissions = self.competition_submissions(competition, page_token=page_token, page_size=page_size) fields = [ - 'file_name', 'date', 'description', 'status', 'public_score', # Breaking change: column headers - 'private_score' + 'fileName', 'date', 'description', 'status', 'publicScore', + 'privateScore' ] if submissions: if csv_display: @@ -960,7 +958,7 @@ def competition_list_files_cli(self, next_page_token = result.next_page_token if next_page_token: print('Next Page Token = {}'.format(next_page_token)) - fields = ['name', 'total_bytes', 'creation_date'] # Breaking change: column header + fields = ['name', 'totalBytes', 'creationDate'] # Breaking change: column header if result: if csv_display: self.print_csv(result.files, fields) @@ -1028,10 +1026,6 @@ def competition_download_files(self, request.competition_name = competition response = kaggle.competitions.competition_api_client.download_data_files(request) url = response.url.split('?')[0] - # response = self.process_response( - # self.competitions_data_download_files_with_http_info( - # id=competition, _preload_content=False)) - # url = response.retries.history[0].redirect_location.split('?')[0] outfile = os.path.join(effective_path, competition + '.' + url.split('.')[-1]) @@ -1148,7 +1142,7 @@ def competition_leaderboard_cli(self, if view: results = self.competition_leaderboard_view(competition) - fields = ['team_id', 'team_name', 'submission_date', 'score'] # Breaking change: column headers + fields = ['teamId', 'teamName', 'submissionDate', 'score'] if results: if csv_display: self.print_csv(results, fields) @@ -1556,8 +1550,16 @@ def dataset_download_files(self, z.extractall(effective_path) except zipfile.BadZipFile as e: raise ValueError( - 'Bad zip file, please report on ' - 'www.github.com/kaggle/kaggle-api', e) + f"The file {outfile} is corrupted or not a valid zip file. " + "Please report this issue at https://www.github.com/kaggle/kaggle-api") + except FileNotFoundError: + raise FileNotFoundError( + f"The file {outfile} was not found. " + "Please report this issue at https://www.github.com/kaggle/kaggle-api") + except Exception as e: + raise RuntimeError( + f"An unexpected error occurred: {e}. " + "Please report this issue at https://www.github.com/kaggle/kaggle-api") try: os.remove(outfile) @@ -1741,10 +1743,10 @@ def dataset_create_version(self, self.process_response( self.with_retry( self.datasets_create_version_by_id_with_http_info)( - id_no, request))) + id_no, request))) else: if ref == self.config_values[ - self.CONFIG_NAME_USER] + '/INSERT_SLUG_HERE': + self.CONFIG_NAME_USER] + '/INSERT_SLUG_HERE': raise ValueError( 'Default slug detected, please change values before ' 'uploading') @@ -1862,7 +1864,7 @@ def dataset_create_new(self, # validations if ref == self.config_values[ - self.CONFIG_NAME_USER] + '/INSERT_SLUG_HERE': + self.CONFIG_NAME_USER] + '/INSERT_SLUG_HERE': raise ValueError( 'Default slug detected, please change values before uploading') if title == 'INSERT_TITLE_HERE': @@ -2231,25 +2233,25 @@ def kernels_initialize(self, folder): username = self.get_config_value(self.CONFIG_NAME_USER) meta_data = { 'id': - username + '/INSERT_KERNEL_SLUG_HERE', + username + '/INSERT_KERNEL_SLUG_HERE', 'title': - 'INSERT_TITLE_HERE', + 'INSERT_TITLE_HERE', 'code_file': - 'INSERT_CODE_FILE_PATH_HERE', + 'INSERT_CODE_FILE_PATH_HERE', 'language': - 'Pick one of: {' + - ','.join(x for x in self.valid_push_language_types) + '}', + 'Pick one of: {' + + ','.join(x for x in self.valid_push_language_types) + '}', 'kernel_type': - 'Pick one of: {' + - ','.join(x for x in self.valid_push_kernel_types) + '}', + 'Pick one of: {' + + ','.join(x for x in self.valid_push_kernel_types) + '}', 'is_private': - 'true', + 'true', 'enable_gpu': - 'false', + 'false', 'enable_tpu': - 'false', + 'false', 'enable_internet': - 'true', + 'true', 'dataset_sources': [], 'competition_sources': [], 'kernel_sources': [], @@ -3491,8 +3493,8 @@ def model_instance_version_create(self, self.process_response( self.with_retry( self.models_create_instance_version_with_http_info)( - owner_slug, model_slug, framework, instance_slug, - request))) + owner_slug, model_slug, framework, instance_slug, + request))) return result @@ -3815,9 +3817,9 @@ def print_table(self, items, fields): return for f in fields: length = max(len(f), - max([len(self.string(getattr(i, f))) for i in items])) + max([len(self.string(getattr(i, self.camel_to_snake(f)))) for i in items])) justify = '>' if isinstance(getattr( - items[0], f), int) or f == 'size' or f == 'reward' else '<' + items[0], self.camel_to_snake(f)), int) or f == 'size' or f == 'reward' else '<' formats.append('{:' + justify + self.string(length + 2) + '}') borders.append('-' * length + ' ') row_format = u''.join(formats) @@ -3825,7 +3827,7 @@ def print_table(self, items, fields): print(row_format.format(*headers)) print(row_format.format(*borders)) for i in items: - i_fields = [self.string(getattr(i, f)) + ' ' for f in fields] + i_fields = [self.string(getattr(i, self.camel_to_snake(f))) + ' ' for f in fields] try: print(row_format.format(*i_fields)) except UnicodeEncodeError: @@ -3842,7 +3844,7 @@ def print_csv(self, items, fields): writer = csv.writer(sys.stdout) writer.writerow(fields) for i in items: - i_fields = [self.string(getattr(i, f)) for f in fields] + i_fields = [self.string(getattr(i, self.camel_to_snake(f))) for f in fields] writer.writerow(i_fields) def string(self, item): @@ -3962,9 +3964,9 @@ def upload_files(self, """ for file_name in os.listdir(folder): if (file_name in [ - self.DATASET_METADATA_FILE, self.OLD_DATASET_METADATA_FILE, - self.KERNEL_METADATA_FILE, self.MODEL_METADATA_FILE, - self.MODEL_INSTANCE_METADATA_FILE + self.DATASET_METADATA_FILE, self.OLD_DATASET_METADATA_FILE, + self.KERNEL_METADATA_FILE, self.MODEL_METADATA_FILE, + self.MODEL_INSTANCE_METADATA_FILE ]): continue upload_file = self._upload_file_or_folder(folder, file_name, @@ -4109,10 +4111,10 @@ def upload_complete(self, path, url, quiet, resume=False): fp.seek(start_at) session.headers.update({ 'Content-Length': - '%d' % upload_size, + '%d' % upload_size, 'Content-Range': - 'bytes %d-%d/%d' % - (start_at, file_size - 1, file_size) + 'bytes %d-%d/%d' % + (start_at, file_size - 1, file_size) }) reader = TqdmBufferedReader(fp, progress_bar) retries = Retry(total=10, backoff_factor=0.5) @@ -4294,7 +4296,7 @@ def validate_model_instance_version_string(self, model_instance_version): split = model_instance_version.split('/') if not split[0] or not split[1] or not split[2] or not split[ - 3] or not split[4]: + 3] or not split[4]: raise ValueError( 'Invalid model instance version specification ' + model_instance_version)