From 3a37738d3ccdc6d1077513d201482a546173b206 Mon Sep 17 00:00:00 2001 From: AlexanderMann Date: Thu, 29 Aug 2019 10:38:26 -0500 Subject: [PATCH 1/2] Performance: REMOVEing from an internal table stage before a DROP TABLE command is redundant --- target_snowflake/snowflake.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/target_snowflake/snowflake.py b/target_snowflake/snowflake.py index cad96f9..808a0bd 100644 --- a/target_snowflake/snowflake.py +++ b/target_snowflake/snowflake.py @@ -413,15 +413,6 @@ def perform_update(self, cur, target_table_name, temp_table_name, key_properties insert_columns=insert_columns, dedupped_columns=dedupped_columns)) - # Clear out the associated stage for the table - cur.execute(''' - REMOVE @{db}.{schema}.%{temp_table} - '''.format( - db=sql.identifier(self.connection.configured_database), - schema=sql.identifier(self.connection.configured_schema), - temp_table=sql.identifier(temp_table_name) - )) - # Drop the tmp table cur.execute(''' DROP TABLE {temp_table}; From 46266b3189b6ea816df89f181fa06cfbe1886c2d Mon Sep 17 00:00:00 2001 From: AlexanderMann Date: Thu, 29 Aug 2019 10:59:28 -0500 Subject: [PATCH 2/2] Feature: Only REMOVE when using internal stage --- target_snowflake/snowflake.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/target_snowflake/snowflake.py b/target_snowflake/snowflake.py index 808a0bd..ade0e9d 100644 --- a/target_snowflake/snowflake.py +++ b/target_snowflake/snowflake.py @@ -413,6 +413,15 @@ def perform_update(self, cur, target_table_name, temp_table_name, key_properties insert_columns=insert_columns, dedupped_columns=dedupped_columns)) + if not self.s3: + # Clear out the associated stage for the table + cur.execute(''' + REMOVE @{db}.{schema}.%{temp_table} + '''.format( + db=sql.identifier(self.connection.configured_database), + schema=sql.identifier(self.connection.configured_schema), + temp_table=sql.identifier(temp_table_name))) + # Drop the tmp table cur.execute(''' DROP TABLE {temp_table};