From 337f5f52b7f3311ba54cef124c5996daa222437f Mon Sep 17 00:00:00 2001 From: Mourad El Hadj Mimoune Date: Thu, 7 Apr 2016 11:45:31 +0200 Subject: [PATCH 1/8] [WIP] Resolve performence issue when migrate partner_address to partner --- .../base/migrations/7.0.1.3/post-migration.py | 229 ++++++++++++++---- 1 file changed, 186 insertions(+), 43 deletions(-) diff --git a/openerp/addons/base/migrations/7.0.1.3/post-migration.py b/openerp/addons/base/migrations/7.0.1.3/post-migration.py index fe9c07006e3d..c9a38fde23cc 100644 --- a/openerp/addons/base/migrations/7.0.1.3/post-migration.py +++ b/openerp/addons/base/migrations/7.0.1.3/post-migration.py @@ -20,7 +20,8 @@ ############################################################################## from openupgrade import openupgrade -from openerp import pooler, SUPERUSER_ID +from openerp import pooler, SUPERUSER_ID, osv +import psycopg2 force_defaults = { 'ir.mail_server': [('active', True)], @@ -188,10 +189,18 @@ def migrate_partner_address(cr, pool): "ALTER TABLE res_partner_address " "ADD column openupgrade_7_migrated_to_partner_id " " INTEGER") + cr.execute( + "ALTER TABLE res_partner " + "ADD column openupgrade_7_migrated_from_address_id " + " INTEGER") cr.execute( "ALTER TABLE res_partner_address ADD FOREIGN KEY " "(openupgrade_7_migrated_to_partner_id) " "REFERENCES res_partner ON DELETE SET NULL") + cr.execute( + "ALTER TABLE res_partner_address " + "ADD column openupgrade_7_address_processed " + " BOOLEAN") fields = [ 'id', 'birthdate', 'city', 'country_id', 'email', 'fax', 'function', 'mobile', 'phone', 'state_id', 'street', 'street2', 'type', 'zip', @@ -226,6 +235,19 @@ def set_address_partner(address_id, partner_id): "WHERE id = %s", (partner_id, address_id)) + def set_address_processed(processed_ids): + ## ---> Set BreakPoint + import pdb; + pdb.set_trace() + while processed_ids: + ids = processed_ids[:2000] + del processed_ids[:2000] + cr.execute( + "UPDATE res_partner_address " + "SET openupgrade_7_address_processed = True " + "WHERE id in %s", + (ids)) + def create_partner(address_id, vals, defaults): """ Create a partner from an address. Update the vals @@ -240,58 +262,116 @@ def create_partner(address_id, vals, defaults): partner_id = partner_obj.create(cr, SUPERUSER_ID, vals) set_address_partner(address_id, partner_id) - def process_address_type(cr, whereclause, args=None): + def format_mass_update_val(rows_values, fields, type='insert'): + """ + Format vals to create a partners from list of address. + Add a co the vals + with the defaults only if the keys do not occur + already in vals. Register the created partner_id + on the obsolete address table + """ + + partner_store = [] + for row in rows_values: + row_cleaned = [val or False for val in row] + dict_values = dict(zip(fields, row_cleaned)) + + dict_values['openupgrade_7_migrated_from_address_id'] = \ + dict_values['id'] + + if type == 'insert_with_parent': + dict_values.update({ + 'is_company': False, + 'parent_id': dict_values['partner_id']}) + + if type == 'insert' or type == 'insert_parent': + partner_defaults = { + # list of values that we should not overwrite + # in existing partners + 'customer': False, + 'is_company': dict_values['type'] != 'contact', + 'type': dict_values['type'], + 'name': dict_values['name'] or '/', + } + for f in ['name', 'id', 'type', 'partner_id']: + del dict_values[f] + for key in partner_defaults: + if key not in dict_values: + dict_values[key] = partner_defaults[key] + values = partner_obj._add_missing_default_values( + cr, SUPERUSER_ID, dict_values) + if type == 'update': + for f in ['name', 'id', 'type', 'partner_id']: + del dict_values[f] + values = dict_values + partner_store.append(values) + processed_ids.append( + values['openupgrade_7_migrated_from_address_id']) + return partner_store + + def process_address_type(cr, pool, whereclause, args=None): """ Migrate addresses to partners, based on sql WHERE clause """ + ## ---> Set BreakPoint + import pdb; + pdb.set_trace() + # Mass process Dangling addresses, create with not is_company + # not supplier and not customer openupgrade.logged_query( cr, "\n" "SELECT " + ', '.join(fields) + "\n" "FROM res_partner_address\n" + "WHERE " + whereclause + " AND partner_id IS NULL", args or ()) + rows_values = cr.fetchall() + partner_store = format_mass_update_val(rows_values, fields, 'insert') + _insert_partners(cr, pool, partner_store) + + select_field = ', '.join(fields) + # Mass update partner with first address (Main partner address) + select_distinct_field = select_field.replace( + "partner_id", "DISTINCT ON (partner_id) partner_id") + openupgrade.logged_query( + cr, "\n" + "SELECT " + select_distinct_field + "\n" + "FROM res_partner_address add1 \n" "WHERE " + whereclause, args or ()) - for row in cr.fetchall(): - row_cleaned = [val or False for val in row] - address = dict(zip(fields, row_cleaned)) - partner_vals = address.copy() - partner_defaults = { - # list of values that we should not overwrite - # in existing partners - 'customer': False, - 'is_company': address['type'] != 'contact', - 'type': address['type'], - 'name': address['name'] or '/', - } - for f in ['name', 'id', 'type', 'partner_id']: - del partner_vals[f] - if not address['partner_id']: - # Dangling addresses, create with not is_company, - # not supplier and not customer - create_partner(address['id'], partner_vals, partner_defaults) - else: - if address['partner_id'] not in partner_found: - # Main partner address - partner_obj.write( - cr, SUPERUSER_ID, address['partner_id'], partner_vals) - partner_found.append(address['partner_id']) - set_address_partner(address['id'], address['partner_id']) - else: - # any following address for an existing partner - partner_vals.update({ - 'is_company': False, - 'parent_id': address['partner_id']}) - propagated_values = partner_obj.read( - cr, SUPERUSER_ID, address['partner_id'], - propagate_fields, load="_classic_write") - propagated_values.pop('id') - partner_vals.update(propagated_values) - create_partner( - address['id'], partner_vals, partner_defaults) - processed_ids.append(address['id']) + + rows_values = cr.fetchall() + partner_store = format_mass_update_val(rows_values, fields, 'update') + _update_partners(cr, pool, partner_store) + + # any following address must be create a new partner wich will be + # attached for an existing partner + + openupgrade.logged_query( + cr, "\n" + "SELECT " + select_field + "\n" + "FROM res_partner_address add1 \n" + "LEFT JOIN (DISTINCT ON (partner_id) partner_id_2" + ", id as id2 FROM res_partner_address ) add2 \n" + "ON add1.id = add2.id " + "WHERE add2.id IS NULL AND " + whereclause, args or ()) + + rows_values = cr.fetchall() + partner_store = format_mass_update_val( + rows_values, fields, 'insert_with_parent') + _insert_partners(cr, pool, partner_store) + + # set_address_partner by mass update + openupgrade.logged_query( + cr, "\n" + "UPDATE res_partner_address addr " + "SET openupgrade_7_migrated_to_partner_id = part.id " + "FROM res_partner part " + "WHERE addr.id = part.openupgrade_7_migrated_from_address_id") # Process all addresses, default type first - process_address_type(cr, "type = 'default'") - process_address_type(cr, "type IS NULL OR type = ''") - process_address_type(cr, "id NOT IN %s", (tuple(processed_ids),)) + process_address_type(cr, pool, "type = 'default'") + process_address_type(cr, pool, "(type IS NULL OR type = '')") + # Not in clause is very slow. we replace them by an ubptade on a new column + set_address_processed(processed_ids) + process_address_type(cr, pool, "openupgrade_7_address_processed = True ") # Check that all addresses have been migrated cr.execute( @@ -354,6 +434,69 @@ def migrate_res_company_logo(cr, pool): partner_obj.write(cr, SUPERUSER_ID, row[0], vals) +def _prepare_insert(pool, partner_val, cols): + """ Apply column formating to prepare data for SQL inserting + Return a copy of move + """ + partner_obj = pool.get('res.partner') + st_copy = partner_val + for k, col in st_copy.iteritems(): + if k in cols: + st_copy[k] = partner_obj._columns[k]._symbol_set[1](col) + return st_copy + + +def _prepare_manyinsert(pool, partner_store, cols): + """ Apply column formating to prepare multiple SQL inserts + Return a copy of move_store + """ + values = [] + for partner_val in partner_store: + values.append(_prepare_insert(pool, partner_val, cols)) + return values + + +def _insert_partners(cr, pool, partner_store): + """ Do raw insert into database because ORM is awfully slow + when doing batch write. It is a shame that batch function + does not exist""" + fields = partner_store and partner_store[0].keys() or [] + partner_store = _prepare_manyinsert(partner_store, fields) + tmp_vals = (', '.join(fields), ', '.join(['%%(%s)s' % i for i in fields])) + sql = "INSERT INTO res_partner (%s) " \ + "VALUES (%s);" % tmp_vals + try: + cr.executemany(sql, tuple(partner_store)) + # TODO handle serialized fields + # sql, tuple(self._serialize_sparse_fields(cols, partner_store))) + except psycopg2.Error as sql_err: + cr.rollback() + raise osv.except_osv("ORM bypass error", sql_err.pgerror) + + +def _update_partners(cr, pool, vals): + """ Do raw update into database because ORM is awfully slow + when cheking security. + TODO / WARM: sparse fields are skipped by the method. IOW, if your + completion rule update an sparse field, the updated value will never + be stored in the database. It would be safer to call the update method + from the ORM for records updating this kind of fields. + """ + + partner_obj = pool.get('res.partner') + fields = vals and vals[0].keys() or [] + vals = partner_obj._prepare_insert(vals, fields) + tmp_vals = (', '.join(['%s = %%(%s)s' % (i, i) for i in fields])) + sql = "UPDATE res_partner " \ + "SET %s where id = %%(id)s;" % tmp_vals + try: + cr.execute(sql, vals) + except psycopg2.Error as sql_err: + cr.rollback() + raise osv.except_osv("ORM bypass error", sql_err.pgerror) + + + @openupgrade.migrate() def migrate(cr, version): pool = pooler.get_pool(cr.dbname) From 50e88e349b1d229534b4e6d0533aa105af5ae6e3 Mon Sep 17 00:00:00 2001 From: Mourad El Hadj Mimoune Date: Wed, 27 Apr 2016 17:07:56 +0200 Subject: [PATCH 2/8] [FIX] complete the code to improve partners's migration performance --- .../base/migrations/7.0.1.3/post-migration.py | 126 +++++++++++------- 1 file changed, 80 insertions(+), 46 deletions(-) diff --git a/openerp/addons/base/migrations/7.0.1.3/post-migration.py b/openerp/addons/base/migrations/7.0.1.3/post-migration.py index c9a38fde23cc..245e46f02b59 100644 --- a/openerp/addons/base/migrations/7.0.1.3/post-migration.py +++ b/openerp/addons/base/migrations/7.0.1.3/post-migration.py @@ -20,7 +20,8 @@ ############################################################################## from openupgrade import openupgrade -from openerp import pooler, SUPERUSER_ID, osv +from openerp import pooler, SUPERUSER_ID +from openerp.osv import osv import psycopg2 force_defaults = { @@ -201,11 +202,16 @@ def migrate_partner_address(cr, pool): "ALTER TABLE res_partner_address " "ADD column openupgrade_7_address_processed " " BOOLEAN") + #To fix bug where category_id is not yet created after module update + cr.execute( + "ALTER TABLE res_partner " + "ADD column category_id " + " INTEGER") fields = [ 'id', 'birthdate', 'city', 'country_id', 'email', 'fax', 'function', 'mobile', 'phone', 'state_id', 'street', 'street2', 'type', 'zip', 'partner_id', 'name', 'company_id' - ] + ] propagate_fields = [ 'lang', 'tz', 'customer', 'supplier', ] @@ -236,17 +242,13 @@ def set_address_partner(address_id, partner_id): (partner_id, address_id)) def set_address_processed(processed_ids): - ## ---> Set BreakPoint - import pdb; - pdb.set_trace() while processed_ids: ids = processed_ids[:2000] del processed_ids[:2000] cr.execute( "UPDATE res_partner_address " "SET openupgrade_7_address_processed = True " - "WHERE id in %s", - (ids)) + "WHERE id in %s", (tuple(ids),)) def create_partner(address_id, vals, defaults): """ @@ -278,21 +280,23 @@ def format_mass_update_val(rows_values, fields, type='insert'): dict_values['openupgrade_7_migrated_from_address_id'] = \ dict_values['id'] - + values = {} if type == 'insert_with_parent': dict_values.update({ - 'is_company': False, - 'parent_id': dict_values['partner_id']}) - - if type == 'insert' or type == 'insert_parent': + 'is_company': False, + 'parent_id': dict_values['partner_id']}) + # for f in ['id', 'partner_id']: + # del dict_values[f] + # values = dict_values + if type == 'insert' or type == 'insert_with_parent': partner_defaults = { - # list of values that we should not overwrite - # in existing partners - 'customer': False, - 'is_company': dict_values['type'] != 'contact', - 'type': dict_values['type'], - 'name': dict_values['name'] or '/', - } + # list of values that we should not overwrite + # in existing partners + 'customer': False, + 'is_company': dict_values['type'] != 'contact', + 'type': dict_values['type'], + 'name': dict_values['name'] or '/', + } for f in ['name', 'id', 'type', 'partner_id']: del dict_values[f] for key in partner_defaults: @@ -301,23 +305,28 @@ def format_mass_update_val(rows_values, fields, type='insert'): values = partner_obj._add_missing_default_values( cr, SUPERUSER_ID, dict_values) if type == 'update': - for f in ['name', 'id', 'type', 'partner_id']: + for f in ['name', 'id', 'type']: del dict_values[f] + dict_values['id'] = dict_values['partner_id'] + del dict_values['partner_id'] values = dict_values partner_store.append(values) processed_ids.append( - values['openupgrade_7_migrated_from_address_id']) + values['openupgrade_7_migrated_from_address_id']) return partner_store - def process_address_type(cr, pool, whereclause, args=None): + def process_address_type(cr, pool, fields, whereclause, args=None): """ Migrate addresses to partners, based on sql WHERE clause """ - ## ---> Set BreakPoint - import pdb; - pdb.set_trace() # Mass process Dangling addresses, create with not is_company # not supplier and not customer + + # Select distinct must be a first column + fields.remove('partner_id') + fields = list(fields) + fields[:0] = ["partner_id"] + openupgrade.logged_query( cr, "\n" "SELECT " + ', '.join(fields) + "\n" @@ -328,30 +337,38 @@ def process_address_type(cr, pool, whereclause, args=None): _insert_partners(cr, pool, partner_store) select_field = ', '.join(fields) + # fields = set(fields) + # Mass update partner with first address (Main partner address) select_distinct_field = select_field.replace( - "partner_id", "DISTINCT ON (partner_id) partner_id") + "partner_id", + "DISTINCT ON (partner_id) FIRST_VALUE(partner_id) OVER (PARTITION " + "BY partner_id ORDER BY id) partner_id") + update_sql = "\n"\ + "SELECT " + select_distinct_field + "\n"\ + "FROM res_partner_address add1 \n"\ + "WHERE " + whereclause openupgrade.logged_query( - cr, "\n" - "SELECT " + select_distinct_field + "\n" - "FROM res_partner_address add1 \n" - "WHERE " + whereclause, args or ()) + cr, update_sql, args or ()) rows_values = cr.fetchall() partner_store = format_mass_update_val(rows_values, fields, 'update') _update_partners(cr, pool, partner_store) - # any following address must be create a new partner wich will be # attached for an existing partner - - openupgrade.logged_query( - cr, "\n" + new_part_sql = ( + "\n" "SELECT " + select_field + "\n" "FROM res_partner_address add1 \n" - "LEFT JOIN (DISTINCT ON (partner_id) partner_id_2" + "LEFT JOIN (" + "SELECT DISTINCT ON (partner_id) FIRST_VALUE(partner_id) " + " OVER (PARTITION " + "BY partner_id ORDER BY id) partner_id2" ", id as id2 FROM res_partner_address ) add2 \n" - "ON add1.id = add2.id " - "WHERE add2.id IS NULL AND " + whereclause, args or ()) + "ON add1.id = add2.id2 " + "WHERE add2.id2 IS NULL AND add1.partner_id IS NOT NULL AND " + whereclause) + openupgrade.logged_query( + cr, new_part_sql, args or ()) rows_values = cr.fetchall() partner_store = format_mass_update_val( @@ -367,11 +384,12 @@ def process_address_type(cr, pool, whereclause, args=None): "WHERE addr.id = part.openupgrade_7_migrated_from_address_id") # Process all addresses, default type first - process_address_type(cr, pool, "type = 'default'") - process_address_type(cr, pool, "(type IS NULL OR type = '')") + process_address_type(cr, pool, fields.copy(), "type = 'default'") + process_address_type(cr, pool, fields.copy(), "(type IS NULL OR type = '')") # Not in clause is very slow. we replace them by an ubptade on a new column set_address_processed(processed_ids) - process_address_type(cr, pool, "openupgrade_7_address_processed = True ") + process_address_type(cr, pool, fields.copy(), + "openupgrade_7_address_processed = True ") # Check that all addresses have been migrated cr.execute( @@ -436,19 +454,20 @@ def migrate_res_company_logo(cr, pool): def _prepare_insert(pool, partner_val, cols): """ Apply column formating to prepare data for SQL inserting - Return a copy of move + Return a copy of partner """ partner_obj = pool.get('res.partner') st_copy = partner_val for k, col in st_copy.iteritems(): if k in cols: - st_copy[k] = partner_obj._columns[k]._symbol_set[1](col) + if k not in ['id', 'openupgrade_7_migrated_from_address_id']: + st_copy[k] = partner_obj._columns[k]._symbol_set[1](col) return st_copy def _prepare_manyinsert(pool, partner_store, cols): """ Apply column formating to prepare multiple SQL inserts - Return a copy of move_store + Return a copy of partner_store """ values = [] for partner_val in partner_store: @@ -461,15 +480,27 @@ def _insert_partners(cr, pool, partner_store): when doing batch write. It is a shame that batch function does not exist""" fields = partner_store and partner_store[0].keys() or [] - partner_store = _prepare_manyinsert(partner_store, fields) + partner_store = _prepare_manyinsert(pool, partner_store, fields) tmp_vals = (', '.join(fields), ', '.join(['%%(%s)s' % i for i in fields])) sql = "INSERT INTO res_partner (%s) " \ "VALUES (%s);" % tmp_vals try: + # cr.execute("select * from res_partner limit 1") + + # db_colnames = [desc[0] for desc in cr.description] + # missing_cols = [col for col in fields if col not in db_colnames] + + # partner_obj = pool.get('res.partner') + # obj_colnames = partner_obj._columns.keys() + # for col in missing_cols: + # if col in obj_colnames: + cr.execute('SAVEPOINT insert_partner') + cr.executemany(sql, tuple(partner_store)) # TODO handle serialized fields # sql, tuple(self._serialize_sparse_fields(cols, partner_store))) except psycopg2.Error as sql_err: + cr.execute('ROLLBACK TO SAVEPOINT insert_partner') cr.rollback() raise osv.except_osv("ORM bypass error", sql_err.pgerror) @@ -485,13 +516,16 @@ def _update_partners(cr, pool, vals): partner_obj = pool.get('res.partner') fields = vals and vals[0].keys() or [] - vals = partner_obj._prepare_insert(vals, fields) + vals = _prepare_manyinsert(pool, vals, fields) tmp_vals = (', '.join(['%s = %%(%s)s' % (i, i) for i in fields])) sql = "UPDATE res_partner " \ "SET %s where id = %%(id)s;" % tmp_vals try: - cr.execute(sql, vals) + cr.execute('SAVEPOINT update_partner') + not_null_id_vals = [v for v in vals if v['id'] != False] + cr.executemany(sql, tuple(not_null_id_vals)) except psycopg2.Error as sql_err: + cr.execute('ROLLBACK TO SAVEPOINT update_partner') cr.rollback() raise osv.except_osv("ORM bypass error", sql_err.pgerror) From 9ff18c4ae108418f9e51a281016776572cefd30c Mon Sep 17 00:00:00 2001 From: Mourad El Hadj Mimoune Date: Wed, 11 May 2016 16:56:07 +0200 Subject: [PATCH 3/8] [FIX] Fix bug of address missed in migration (not test yet) --- .../base/migrations/7.0.1.3/post-migration.py | 125 ++++++++++++++---- 1 file changed, 96 insertions(+), 29 deletions(-) diff --git a/openerp/addons/base/migrations/7.0.1.3/post-migration.py b/openerp/addons/base/migrations/7.0.1.3/post-migration.py index 245e46f02b59..bafbd29763c2 100644 --- a/openerp/addons/base/migrations/7.0.1.3/post-migration.py +++ b/openerp/addons/base/migrations/7.0.1.3/post-migration.py @@ -273,7 +273,8 @@ def format_mass_update_val(rows_values, fields, type='insert'): on the obsolete address table """ - partner_store = [] + partner_store_update = [] + partner_store_insert = [] for row in rows_values: row_cleaned = [val or False for val in row] dict_values = dict(zip(fields, row_cleaned)) @@ -281,6 +282,19 @@ def format_mass_update_val(rows_values, fields, type='insert'): dict_values['openupgrade_7_migrated_from_address_id'] = \ dict_values['id'] values = {} + processed_part = False + if type == 'update': + if dict_values['partner_id'] in partner_found: + # Do not update partner twice + processed_part = True + else: + for f in ['name', 'id', 'type']: + del dict_values[f] + dict_values['id'] = dict_values['partner_id'] + del dict_values['partner_id'] + values = dict_values + partner_found.append( values['id']) + partner_store_update.append(values) if type == 'insert_with_parent': dict_values.update({ 'is_company': False, @@ -288,7 +302,7 @@ def format_mass_update_val(rows_values, fields, type='insert'): # for f in ['id', 'partner_id']: # del dict_values[f] # values = dict_values - if type == 'insert' or type == 'insert_with_parent': + if type == 'insert' or type == 'insert_with_parent' or processed_part: partner_defaults = { # list of values that we should not overwrite # in existing partners @@ -304,16 +318,10 @@ def format_mass_update_val(rows_values, fields, type='insert'): dict_values[key] = partner_defaults[key] values = partner_obj._add_missing_default_values( cr, SUPERUSER_ID, dict_values) - if type == 'update': - for f in ['name', 'id', 'type']: - del dict_values[f] - dict_values['id'] = dict_values['partner_id'] - del dict_values['partner_id'] - values = dict_values - partner_store.append(values) + partner_store_insert.append(values) processed_ids.append( values['openupgrade_7_migrated_from_address_id']) - return partner_store + return partner_store_insert, partner_store_update def process_address_type(cr, pool, fields, whereclause, args=None): """ @@ -326,19 +334,71 @@ def process_address_type(cr, pool, fields, whereclause, args=None): fields.remove('partner_id') fields = list(fields) fields[:0] = ["partner_id"] - openupgrade.logged_query( cr, "\n" "SELECT " + ', '.join(fields) + "\n" "FROM res_partner_address\n" - "WHERE " + whereclause + " AND partner_id IS NULL", args or ()) + "WHERE " + whereclause, args or ()) rows_values = cr.fetchall() + partner_store_update = [] + partner_store_insert = [] + for row in rows_values: + row_cleaned = [val or False for val in row] + dict_values = dict(zip(fields, row_cleaned)) + + dict_values['openupgrade_7_migrated_from_address_id'] = \ + dict_values['id'] + values = {} + partner_defaults = { + # list of values that we should not overwrite + # in existing partners + 'customer': False, + 'is_company': dict_values['type'] != 'contact', + 'type': dict_values['type'], + 'name': dict_values['name'] or '/', + 'parent_id': dict_values['partner_id'], + } + + if not dict_values['partner_id']: + for f in ['name', 'id', 'type', 'partner_id']: + del dict_values[f] + for key in partner_defaults: + if key not in dict_values: + dict_values[key] = partner_defaults[key] + values = partner_obj._add_missing_default_values( + cr, SUPERUSER_ID, dict_values) + partner_store_insert.append(values) + else: + if dict_values['partner_id'] not in partner_found: + for f in ['name', 'id', 'type']: + del dict_values[f] + dict_values['id'] = dict_values['partner_id'] + del dict_values['partner_id'] + values = dict_values + partner_found.append(values['id']) + partner_store_update.append(values) + else: + dict_values.update({'is_company': False}) + for f in ['name', 'id', 'type', 'partner_id']: + del dict_values[f] + for key in partner_defaults: + if key not in dict_values: + dict_values[key] = partner_defaults[key] + values = partner_obj._add_missing_default_values( + cr, SUPERUSER_ID, dict_values) + partner_store_insert.append(values) + processed_ids.append( + values['openupgrade_7_migrated_from_address_id']) + + _insert_partners(cr, pool, partner_store_insert) + _update_partners(cr, pool, partner_store_update) + """ partner_store = format_mass_update_val(rows_values, fields, 'insert') - _insert_partners(cr, pool, partner_store) + _insert_partners(cr, pool, partner_store[0]) select_field = ', '.join(fields) # fields = set(fields) - + # Main partner address # Mass update partner with first address (Main partner address) select_distinct_field = select_field.replace( "partner_id", @@ -347,41 +407,43 @@ def process_address_type(cr, pool, fields, whereclause, args=None): update_sql = "\n"\ "SELECT " + select_distinct_field + "\n"\ "FROM res_partner_address add1 \n"\ - "WHERE " + whereclause - openupgrade.logged_query( - cr, update_sql, args or ()) + "WHERE " + whereclause + " AND partner_id IS NOT NULL" + openupgrade.logged_query(cr, update_sql, args or ()) rows_values = cr.fetchall() partner_store = format_mass_update_val(rows_values, fields, 'update') - _update_partners(cr, pool, partner_store) + _update_partners(cr, pool, partner_store[1]) + _insert_partners(cr, pool, partner_store[0]) # any following address must be create a new partner wich will be # attached for an existing partner new_part_sql = ( "\n" - "SELECT " + select_field + "\n" - "FROM res_partner_address add1 \n" + "SELECT " + select_field + " \n" + " FROM res_partner_address add1 \n" "LEFT JOIN (" "SELECT DISTINCT ON (partner_id) FIRST_VALUE(partner_id) " " OVER (PARTITION " "BY partner_id ORDER BY id) partner_id2" - ", id as id2 FROM res_partner_address ) add2 \n" + ", id as id2 " + "FROM res_partner_address \n" + "WHERE " + whereclause + " AND partner_id IS NOT NULL) add2 \n" "ON add1.id = add2.id2 " - "WHERE add2.id2 IS NULL AND add1.partner_id IS NOT NULL AND " + whereclause) - openupgrade.logged_query( - cr, new_part_sql, args or ()) + "WHERE add2.id2 IS NULL AND add1.partner_id IS NOT NULL AND " + whereclause ) + openupgrade.logged_query(cr, new_part_sql, args or ()) rows_values = cr.fetchall() partner_store = format_mass_update_val( rows_values, fields, 'insert_with_parent') - _insert_partners(cr, pool, partner_store) - + _insert_partners(cr, pool, partner_store[0]) + """ # set_address_partner by mass update openupgrade.logged_query( cr, "\n" "UPDATE res_partner_address addr " "SET openupgrade_7_migrated_to_partner_id = part.id " "FROM res_partner part " - "WHERE addr.id = part.openupgrade_7_migrated_from_address_id") + "WHERE addr.id = part.openupgrade_7_migrated_from_address_id" + " AND openupgrade_7_migrated_to_partner_id IS NULL") # Process all addresses, default type first process_address_type(cr, pool, fields.copy(), "type = 'default'") @@ -389,7 +451,7 @@ def process_address_type(cr, pool, fields, whereclause, args=None): # Not in clause is very slow. we replace them by an ubptade on a new column set_address_processed(processed_ids) process_address_type(cr, pool, fields.copy(), - "openupgrade_7_address_processed = True ") + "openupgrade_7_address_processed IS NULL ") # Check that all addresses have been migrated cr.execute( @@ -497,6 +559,9 @@ def _insert_partners(cr, pool, partner_store): cr.execute('SAVEPOINT insert_partner') cr.executemany(sql, tuple(partner_store)) + # openupgrade.logger.debug('Running %s', sql % tuple(partner_store)) + openupgrade.logger.debug( + '%s rows inserted into res_partner', cr.rowcount) # TODO handle serialized fields # sql, tuple(self._serialize_sparse_fields(cols, partner_store))) except psycopg2.Error as sql_err: @@ -524,13 +589,15 @@ def _update_partners(cr, pool, vals): cr.execute('SAVEPOINT update_partner') not_null_id_vals = [v for v in vals if v['id'] != False] cr.executemany(sql, tuple(not_null_id_vals)) + # openupgrade.logger.debug('Running %s', sql % tuple(partner_store)) + openupgrade.logger.debug( + '%s rows updated into res_partner', cr.rowcount) except psycopg2.Error as sql_err: cr.execute('ROLLBACK TO SAVEPOINT update_partner') cr.rollback() raise osv.except_osv("ORM bypass error", sql_err.pgerror) - @openupgrade.migrate() def migrate(cr, version): pool = pooler.get_pool(cr.dbname) From ebdaf164dd25eaa1950fd1f658069594f8568f94 Mon Sep 17 00:00:00 2001 From: Mourad El Hadj Mimoune Date: Tue, 17 May 2016 17:03:03 +0200 Subject: [PATCH 4/8] [FIX] Clean Code --- .../base/migrations/7.0.1.3/post-migration.py | 122 +----------------- 1 file changed, 6 insertions(+), 116 deletions(-) diff --git a/openerp/addons/base/migrations/7.0.1.3/post-migration.py b/openerp/addons/base/migrations/7.0.1.3/post-migration.py index bafbd29763c2..10b2ac169180 100644 --- a/openerp/addons/base/migrations/7.0.1.3/post-migration.py +++ b/openerp/addons/base/migrations/7.0.1.3/post-migration.py @@ -264,65 +264,6 @@ def create_partner(address_id, vals, defaults): partner_id = partner_obj.create(cr, SUPERUSER_ID, vals) set_address_partner(address_id, partner_id) - def format_mass_update_val(rows_values, fields, type='insert'): - """ - Format vals to create a partners from list of address. - Add a co the vals - with the defaults only if the keys do not occur - already in vals. Register the created partner_id - on the obsolete address table - """ - - partner_store_update = [] - partner_store_insert = [] - for row in rows_values: - row_cleaned = [val or False for val in row] - dict_values = dict(zip(fields, row_cleaned)) - - dict_values['openupgrade_7_migrated_from_address_id'] = \ - dict_values['id'] - values = {} - processed_part = False - if type == 'update': - if dict_values['partner_id'] in partner_found: - # Do not update partner twice - processed_part = True - else: - for f in ['name', 'id', 'type']: - del dict_values[f] - dict_values['id'] = dict_values['partner_id'] - del dict_values['partner_id'] - values = dict_values - partner_found.append( values['id']) - partner_store_update.append(values) - if type == 'insert_with_parent': - dict_values.update({ - 'is_company': False, - 'parent_id': dict_values['partner_id']}) - # for f in ['id', 'partner_id']: - # del dict_values[f] - # values = dict_values - if type == 'insert' or type == 'insert_with_parent' or processed_part: - partner_defaults = { - # list of values that we should not overwrite - # in existing partners - 'customer': False, - 'is_company': dict_values['type'] != 'contact', - 'type': dict_values['type'], - 'name': dict_values['name'] or '/', - } - for f in ['name', 'id', 'type', 'partner_id']: - del dict_values[f] - for key in partner_defaults: - if key not in dict_values: - dict_values[key] = partner_defaults[key] - values = partner_obj._add_missing_default_values( - cr, SUPERUSER_ID, dict_values) - partner_store_insert.append(values) - processed_ids.append( - values['openupgrade_7_migrated_from_address_id']) - return partner_store_insert, partner_store_update - def process_address_type(cr, pool, fields, whereclause, args=None): """ Migrate addresses to partners, based on sql WHERE clause @@ -392,50 +333,7 @@ def process_address_type(cr, pool, fields, whereclause, args=None): _insert_partners(cr, pool, partner_store_insert) _update_partners(cr, pool, partner_store_update) - """ - partner_store = format_mass_update_val(rows_values, fields, 'insert') - _insert_partners(cr, pool, partner_store[0]) - - select_field = ', '.join(fields) - # fields = set(fields) - # Main partner address - # Mass update partner with first address (Main partner address) - select_distinct_field = select_field.replace( - "partner_id", - "DISTINCT ON (partner_id) FIRST_VALUE(partner_id) OVER (PARTITION " - "BY partner_id ORDER BY id) partner_id") - update_sql = "\n"\ - "SELECT " + select_distinct_field + "\n"\ - "FROM res_partner_address add1 \n"\ - "WHERE " + whereclause + " AND partner_id IS NOT NULL" - openupgrade.logged_query(cr, update_sql, args or ()) - rows_values = cr.fetchall() - partner_store = format_mass_update_val(rows_values, fields, 'update') - _update_partners(cr, pool, partner_store[1]) - _insert_partners(cr, pool, partner_store[0]) - # any following address must be create a new partner wich will be - # attached for an existing partner - new_part_sql = ( - "\n" - "SELECT " + select_field + " \n" - " FROM res_partner_address add1 \n" - "LEFT JOIN (" - "SELECT DISTINCT ON (partner_id) FIRST_VALUE(partner_id) " - " OVER (PARTITION " - "BY partner_id ORDER BY id) partner_id2" - ", id as id2 " - "FROM res_partner_address \n" - "WHERE " + whereclause + " AND partner_id IS NOT NULL) add2 \n" - "ON add1.id = add2.id2 " - "WHERE add2.id2 IS NULL AND add1.partner_id IS NOT NULL AND " + whereclause ) - openupgrade.logged_query(cr, new_part_sql, args or ()) - - rows_values = cr.fetchall() - partner_store = format_mass_update_val( - rows_values, fields, 'insert_with_parent') - _insert_partners(cr, pool, partner_store[0]) - """ # set_address_partner by mass update openupgrade.logged_query( cr, "\n" @@ -447,10 +345,12 @@ def process_address_type(cr, pool, fields, whereclause, args=None): # Process all addresses, default type first process_address_type(cr, pool, fields.copy(), "type = 'default'") - process_address_type(cr, pool, fields.copy(), "(type IS NULL OR type = '')") + process_address_type( + cr, pool, fields.copy(), "(type IS NULL OR type = '')") # Not in clause is very slow. we replace them by an ubptade on a new column set_address_processed(processed_ids) - process_address_type(cr, pool, fields.copy(), + process_address_type( + cr, pool, fields.copy(), "openupgrade_7_address_processed IS NULL ") # Check that all addresses have been migrated @@ -479,7 +379,7 @@ def update_users_partner(cr, pool): 'lang': row[2] or False, 'tz': row[3] or False, 'email': row[4] or False, - } + } partner_obj.write(cr, SUPERUSER_ID, row[1], partner_vals) @@ -547,15 +447,6 @@ def _insert_partners(cr, pool, partner_store): sql = "INSERT INTO res_partner (%s) " \ "VALUES (%s);" % tmp_vals try: - # cr.execute("select * from res_partner limit 1") - - # db_colnames = [desc[0] for desc in cr.description] - # missing_cols = [col for col in fields if col not in db_colnames] - - # partner_obj = pool.get('res.partner') - # obj_colnames = partner_obj._columns.keys() - # for col in missing_cols: - # if col in obj_colnames: cr.execute('SAVEPOINT insert_partner') cr.executemany(sql, tuple(partner_store)) @@ -579,7 +470,6 @@ def _update_partners(cr, pool, vals): from the ORM for records updating this kind of fields. """ - partner_obj = pool.get('res.partner') fields = vals and vals[0].keys() or [] vals = _prepare_manyinsert(pool, vals, fields) tmp_vals = (', '.join(['%s = %%(%s)s' % (i, i) for i in fields])) @@ -587,7 +477,7 @@ def _update_partners(cr, pool, vals): "SET %s where id = %%(id)s;" % tmp_vals try: cr.execute('SAVEPOINT update_partner') - not_null_id_vals = [v for v in vals if v['id'] != False] + not_null_id_vals = [v for v in vals if v['id'] is not False] cr.executemany(sql, tuple(not_null_id_vals)) # openupgrade.logger.debug('Running %s', sql % tuple(partner_store)) openupgrade.logger.debug( From a94d78f56f3a16f6aeeea756fca680b48501faed Mon Sep 17 00:00:00 2001 From: Mourad El Hadj Mimoune Date: Tue, 17 May 2016 17:03:47 +0200 Subject: [PATCH 5/8] [FIX] Update propagate_fields from partner parent --- .../base/migrations/7.0.1.3/post-migration.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/openerp/addons/base/migrations/7.0.1.3/post-migration.py b/openerp/addons/base/migrations/7.0.1.3/post-migration.py index 10b2ac169180..ea4106046646 100644 --- a/openerp/addons/base/migrations/7.0.1.3/post-migration.py +++ b/openerp/addons/base/migrations/7.0.1.3/post-migration.py @@ -334,6 +334,19 @@ def process_address_type(cr, pool, fields, whereclause, args=None): _insert_partners(cr, pool, partner_store_insert) _update_partners(cr, pool, partner_store_update) + # Update propagate_fields from partner parent + # TODO Use propagate_fields = [ 'lang', 'tz', 'customer', 'supplier',] + # to generate this query + openupgrade.logged_query( + cr, "\n" + "UPDATE res_partner part " + "SET lang = parent.lang, " + "tz = parent.tz," + "customer = parent.customer," + "supplier = parent.supplier" + "FROM res_partner parent " + "WHERE part.parent_id = parent.id" + " AND part.parent_id IS NOT NULL") # set_address_partner by mass update openupgrade.logged_query( cr, "\n" From d3e7b90a9e0459371bb539f64a87cf0b356d470e Mon Sep 17 00:00:00 2001 From: Mourad El Hadj Mimoune Date: Wed, 18 May 2016 09:16:19 +0200 Subject: [PATCH 6/8] [FIX] Pep8 - pretty up code --- openerp/addons/base/__init__.py | 7 ++----- openerp/addons/base/migrations/7.0.1.3/post-migration.py | 8 ++++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/openerp/addons/base/__init__.py b/openerp/addons/base/__init__.py index d666a6abbf75..b1799596cb4f 100644 --- a/openerp/addons/base/__init__.py +++ b/openerp/addons/base/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ############################################################################## -# +# # OpenERP, Open Source Management Solution # Copyright (C) 2004-2009 Tiny SPRL (). # @@ -15,7 +15,7 @@ # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . +# along with this program. If not, see # ############################################################################## @@ -24,6 +24,3 @@ import res import report import test - -# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: - diff --git a/openerp/addons/base/migrations/7.0.1.3/post-migration.py b/openerp/addons/base/migrations/7.0.1.3/post-migration.py index ea4106046646..b274ab0aa28b 100644 --- a/openerp/addons/base/migrations/7.0.1.3/post-migration.py +++ b/openerp/addons/base/migrations/7.0.1.3/post-migration.py @@ -202,7 +202,7 @@ def migrate_partner_address(cr, pool): "ALTER TABLE res_partner_address " "ADD column openupgrade_7_address_processed " " BOOLEAN") - #To fix bug where category_id is not yet created after module update + # To fix bug where category_id is not yet created after module update cr.execute( "ALTER TABLE res_partner " "ADD column category_id " @@ -212,9 +212,9 @@ def migrate_partner_address(cr, pool): 'mobile', 'phone', 'state_id', 'street', 'street2', 'type', 'zip', 'partner_id', 'name', 'company_id' ] - propagate_fields = [ - 'lang', 'tz', 'customer', 'supplier', - ] + # propagate_fields = [ + # 'lang', 'tz', 'customer', 'supplier', + # ] partner_found = [] processed_ids = [] From 92cba3ead6372abb871d00965086dbc8fc2dcd7b Mon Sep 17 00:00:00 2001 From: Mourad El Hadj Mimoune Date: Thu, 19 May 2016 23:46:50 +0200 Subject: [PATCH 7/8] [FIX] fix update propagate_fields query --- openerp/addons/base/migrations/7.0.1.3/post-migration.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openerp/addons/base/migrations/7.0.1.3/post-migration.py b/openerp/addons/base/migrations/7.0.1.3/post-migration.py index b274ab0aa28b..1439994cf13d 100644 --- a/openerp/addons/base/migrations/7.0.1.3/post-migration.py +++ b/openerp/addons/base/migrations/7.0.1.3/post-migration.py @@ -341,11 +341,11 @@ def process_address_type(cr, pool, fields, whereclause, args=None): cr, "\n" "UPDATE res_partner part " "SET lang = parent.lang, " - "tz = parent.tz," - "customer = parent.customer," - "supplier = parent.supplier" + "tz = parent.tz, " + "customer = parent.customer, " + "supplier = parent.supplier " "FROM res_partner parent " - "WHERE part.parent_id = parent.id" + "WHERE part.parent_id = parent.id " " AND part.parent_id IS NOT NULL") # set_address_partner by mass update openupgrade.logged_query( From b367c53d4649d7874049cdcb096d670111f5dd35 Mon Sep 17 00:00:00 2001 From: Mourad Elhadj Mimoune Date: Tue, 14 Jun 2016 10:23:56 +0200 Subject: [PATCH 8/8] [FIX] propagate_fields only for inserted partner --- openerp/addons/base/migrations/7.0.1.3/post-migration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openerp/addons/base/migrations/7.0.1.3/post-migration.py b/openerp/addons/base/migrations/7.0.1.3/post-migration.py index 1439994cf13d..c7128d7a6df5 100644 --- a/openerp/addons/base/migrations/7.0.1.3/post-migration.py +++ b/openerp/addons/base/migrations/7.0.1.3/post-migration.py @@ -346,7 +346,8 @@ def process_address_type(cr, pool, fields, whereclause, args=None): "supplier = parent.supplier " "FROM res_partner parent " "WHERE part.parent_id = parent.id " - " AND part.parent_id IS NOT NULL") + " AND part.parent_id IS NOT NULL" + " AND part.openupgrade_7_migrated_from_address_id IS NOT NULL") # set_address_partner by mass update openupgrade.logged_query( cr, "\n"