From 58c5d064e12445eb67ab986a735fc010bb9af9a9 Mon Sep 17 00:00:00 2001 From: Bruno Zanotti Date: Tue, 7 Nov 2023 09:09:37 -0300 Subject: [PATCH] [ADD] read_group refactor script --- odoo_module_migrate/base_migration_script.py | 5 +- .../migration_scripts/migrate_160_170.py | 250 +++++++++++++++++- .../module_160_170/models/res_partner.py | 15 ++ .../module_160/models/res_partner.py | 15 ++ 4 files changed, 283 insertions(+), 2 deletions(-) diff --git a/odoo_module_migrate/base_migration_script.py b/odoo_module_migrate/base_migration_script.py index 36cd9061..f42e792c 100644 --- a/odoo_module_migrate/base_migration_script.py +++ b/odoo_module_migrate/base_migration_script.py @@ -399,7 +399,10 @@ def handle_removed_models(self, removed_models): table_name = model_name.replace(".", "_") model_name_esc = re.escape(model_name) - msg = "The model %s has been .%s" % (model_name, (" %s" % more_info) or "") + msg = "The model %s has been deprecated.%s" % ( + model_name, + (" %s" % more_info) or "", + ) res["errors"].update( { diff --git a/odoo_module_migrate/migration_scripts/migrate_160_170.py b/odoo_module_migrate/migration_scripts/migrate_160_170.py index 4796e7e5..02252988 100644 --- a/odoo_module_migrate/migration_scripts/migrate_160_170.py +++ b/odoo_module_migrate/migration_scripts/migrate_160_170.py @@ -3,6 +3,237 @@ from odoo_module_migrate.base_migration_script import BaseMigrationScript import lxml.etree as et from pathlib import Path +import sys +import os +import ast +from typing import Any + +empty_list = ast.parse("[]").body[0].value + + +class AbstractVisitor(ast.NodeVisitor): + def __init__(self) -> None: + # ((line, line_end, col_offset, end_col_offset), replace_by) NO OVERLAPS + self.change_todo = [] + + def post_process(self, all_code: str, file: str) -> str: + all_lines = all_code.split("\n") + for (lineno, line_end, col_offset, end_col_offset), new_substring in sorted( + self.change_todo, reverse=True + ): + if lineno == line_end: + line = all_lines[lineno - 1] + all_lines[lineno - 1] = ( + line[:col_offset] + new_substring + line[end_col_offset:] + ) + else: + print( + f"Ignore replacement {file}: {(lineno, line_end, col_offset, end_col_offset), new_substring}" + ) + return "\n".join(all_lines) + + def add_change(self, old_node, new_node): + position = ( + old_node.lineno, + old_node.end_lineno, + old_node.col_offset, + old_node.end_col_offset, + ) + if isinstance(new_node, str): + self.change_todo.append((position, new_node)) + else: + self.change_todo.append((position, ast.unparse(new_node))) + + +class VisitorToPrivateReadGroup(AbstractVisitor): + def post_process(self, all_code: str, file: str) -> str: + all_lines = all_code.split("\n") + for i, line in enumerate(all_lines): + if "super(" not in line: + all_lines[i] = line.replace(".read_group(", "._read_group(") + return "\n".join(all_lines) + + +class VisitorInverseGroupbyFields(AbstractVisitor): + def visit_Call(self, node: ast.Call) -> Any: + if isinstance(node.func, ast.Attribute) and node.func.attr == "_read_group": + # Should have the same number of args/keywords + # Inverse fields/groupby order + keywords_by_key = {keyword.arg: keyword.value for keyword in node.keywords} + key_i_by_key = {keyword.arg: i for i, keyword in enumerate(node.keywords)} + if len(node.args) >= 3: + self.add_change(node.args[2], node.args[1]) + self.add_change(node.args[1], node.args[2]) + elif len(node.args) == 2: + new_args_value = keywords_by_key.get("groupby", empty_list) + if "groupby" in keywords_by_key: + fields_args = ast.keyword("fields", node.args[1]) + self.add_change(node.args[1], new_args_value) + self.add_change(node.keywords[key_i_by_key["groupby"]], fields_args) + else: + self.add_change( + node.args[1], + f"{ast.unparse(new_args_value)}, {ast.unparse(node.args[1])}", + ) + else: # len(node.args) <= 2 + if ( + "groupby" in key_i_by_key + and "fields" in key_i_by_key + and key_i_by_key["groupby"] > key_i_by_key["fields"] + ): + self.add_change( + node.keywords[key_i_by_key["groupby"]], + node.keywords[key_i_by_key["fields"]], + ) + self.add_change( + node.keywords[key_i_by_key["fields"]], + node.keywords[key_i_by_key["groupby"]], + ) + else: + raise ValueError(f"{key_i_by_key}, {keywords_by_key}, {node.args}") + self.generic_visit(node) + + +class VisitorRenameKeywords(AbstractVisitor): + def visit_Call(self, node: ast.Call) -> Any: + if isinstance(node.func, ast.Attribute) and node.func.attr == "_read_group": + # Replace fields by aggregate and orderby by order + for keyword in node.keywords: + if keyword.arg == "fields": + new_keyword = ast.keyword("aggregates", keyword.value) + self.add_change(keyword, new_keyword) + if keyword.arg == "orderby": + new_keyword = ast.keyword("order", keyword.value) + self.add_change(keyword, new_keyword) + self.generic_visit(node) + + +class VisitorRemoveLazy(AbstractVisitor): + def post_process(self, all_code: str, file: str) -> str: + # remove extra comma ',' and extra line if possible + all_code = super().post_process(all_code, file) + all_lines = all_code.split("\n") + for (lineno, __, col_offset, __), __ in sorted(self.change_todo, reverse=True): + comma_find = False + line = all_lines[lineno - 1] + remaining = line[col_offset:] + line = line[:col_offset] + while not comma_find: + if "," not in line: + all_lines.pop(lineno - 1) + lineno -= 1 + line = all_lines[lineno - 1] + else: + comma_find = True + last_index_comma = -(line[::-1].index(",") + 1) + all_lines[lineno - 1] = line[:last_index_comma] + remaining + + return "\n".join(all_lines) + + def visit_Call(self, node: ast.Call) -> Any: + if isinstance(node.func, ast.Attribute) and node.func.attr == "_read_group": + # Replace fields by aggregate and orderby by order + if len(node.args) == 7: + self.add_change(node.args[6], "") + else: + for keyword in node.keywords: + if keyword.arg == "lazy": + self.add_change(keyword, "") + self.generic_visit(node) + + +class VisitorAggregatesSpec(AbstractVisitor): + def visit_Call(self, node: ast.Call) -> Any: + if isinstance(node.func, ast.Attribute) and node.func.attr == "_read_group": + + keywords_by_key = {keyword.arg: keyword.value for keyword in node.keywords} + aggregate_values = None + if len(node.args) >= 3: + aggregate_values = node.args[2] + elif "aggregates" in keywords_by_key: + aggregate_values = keywords_by_key["aggregates"] + + groupby_values = empty_list + if len(node.args) >= 2: + groupby_values = node.args[1] + elif "groupby" in keywords_by_key: + groupby_values = keywords_by_key["groupby"] + + if aggregate_values: + aggregates = None + try: + aggregates = ast.literal_eval(ast.unparse(aggregate_values)) + if not isinstance(aggregates, (list, tuple)): + raise ValueError( + f"{aggregate_values} is not a list but literal ?" + ) + + aggregates = [ + f"{field_spec.split('(')[1][:-1]}:{field_spec.split(':')[1].split('(')[0]}" + if "(" in field_spec + else field_spec + for field_spec in aggregates + ] + aggregates = [ + "__count" + if field_spec in ("id:count", "id:count_distinct") + else field_spec + for field_spec in aggregates + ] + + groupby = ast.literal_eval(ast.unparse(groupby_values)) + if isinstance(groupby, str): + groupby = [groupby] + + aggregates = [ + f"{field}:sum" + if (":" not in field and field != "__count") + else field + for field in aggregates + if field not in groupby + ] + if not aggregates: + aggregates = ["__count"] + except SyntaxError: + pass + except ValueError: + pass + + if aggregates is not None: + self.add_change(aggregate_values, repr(aggregates)) + self.generic_visit(node) + + +Steps_visitor: list[AbstractVisitor] = [ + VisitorToPrivateReadGroup, + VisitorInverseGroupbyFields, + VisitorRenameKeywords, + VisitorAggregatesSpec, + VisitorRemoveLazy, +] + + +def replace_read_group_signature(logger, filename): + with open(filename, mode="rt") as file: + new_all = all_code = file.read() + if ".read_group(" in all_code or "._read_group(" in all_code: + for Step in Steps_visitor: + visitor = Step() + try: + visitor.visit(ast.parse(new_all)) + except Exception: + logger.info( + f"ERROR in {filename} at step {visitor.__class__}: \n{new_all}" + ) + raise + new_all = visitor.post_process(new_all, filename) + if new_all == all_code: + logger.info("read_group detected but not changed in file %s" % filename) + + if new_all != all_code: + logger.info("Script read_group replace applied in file %s" % filename) + with open(filename, mode="wt") as file: + file.write(new_all) def _get_files(module_path, reformat_file_ext): @@ -41,6 +272,23 @@ def _check_open_form( _check_open_form_view(logger, file_path) +def _reformat_read_group( + logger, module_path, module_name, manifest_path, migration_steps, tools +): + """Reformat read_group method in py files.""" + + reformat_file_ext = ".py" + file_paths = _get_files(module_path, reformat_file_ext) + logger.debug(f"{reformat_file_ext} files found:\n" f"{list(map(str, file_paths))}") + + reformatted_files = list() + for file_path in file_paths: + reformatted_file = replace_read_group_signature(logger, file_path) + if reformatted_file: + reformatted_files.append(reformatted_file) + logger.debug("Reformatted files:\n" f"{list(reformatted_files)}") + + class MigrationScript(BaseMigrationScript): - _GLOBAL_FUNCTIONS = [_check_open_form] + _GLOBAL_FUNCTIONS = [_check_open_form, _reformat_read_group] diff --git a/tests/data_result/module_160_170/models/res_partner.py b/tests/data_result/module_160_170/models/res_partner.py index aba5c617..38863a15 100644 --- a/tests/data_result/module_160_170/models/res_partner.py +++ b/tests/data_result/module_160_170/models/res_partner.py @@ -6,3 +6,18 @@ class ResPartner(models.Model): test_field_1 = fields.Boolean() task_ids = fields.One2many('project.task') + task_count = fields.Integer(compute='_compute_task_count', string='# Tasks') + + def _compute_task_count(self): + # retrieve all children partners and prefetch 'parent_id' on them + all_partners = self.with_context(active_test=False).search_fetch( + [('id', 'child_of', self.ids)], + ['parent_id'], + ) + task_data = self.env['project.task']._read_group( + domain=[('partner_id', 'in', all_partners.ids)], + groupby=['partner_id'], aggregates=['__count'] + ) + group_dependent = self.env['project.task']._read_group([ + ('depend_on_ids', 'in', task_data.ids), + ], ['depend_on_ids'], ['__count']) diff --git a/tests/data_template/module_160/models/res_partner.py b/tests/data_template/module_160/models/res_partner.py index aba5c617..487a2e3f 100644 --- a/tests/data_template/module_160/models/res_partner.py +++ b/tests/data_template/module_160/models/res_partner.py @@ -6,3 +6,18 @@ class ResPartner(models.Model): test_field_1 = fields.Boolean() task_ids = fields.One2many('project.task') + task_count = fields.Integer(compute='_compute_task_count', string='# Tasks') + + def _compute_task_count(self): + # retrieve all children partners and prefetch 'parent_id' on them + all_partners = self.with_context(active_test=False).search_fetch( + [('id', 'child_of', self.ids)], + ['parent_id'], + ) + task_data = self.env['project.task']._read_group( + domain=[('partner_id', 'in', all_partners.ids)], + fields=['partner_id'], groupby=['partner_id'] + ) + group_dependent = self.env['project.task']._read_group([ + ('depend_on_ids', 'in', task_data.ids), + ], ['depend_on_ids'], ['depend_on_ids'])