diff --git a/README.rst b/README.rst index d4fd885..767bcbc 100644 --- a/README.rst +++ b/README.rst @@ -192,7 +192,12 @@ the aggregated directory. - echo 'a first command' - echo 'a second command' -A real life example: applying a patch +Patches +------- + +Patches can be applied with ``git am`` after the merges (and before the +``shell_command_after`` commands). Each entry is a local patch file, a +directory of patch files (applied in sorted order) or a URL. .. code-block:: yaml @@ -203,8 +208,13 @@ A real life example: applying a patch merges: - oca 9.0 target: acsone aggregated_branch_name - shell_command_after: - - git am "$(git format-patch -1 XXXXXX -o ../patches)" + patches: + - ../patches/0001-fix.patch + - ../patches/odoo + - https://github.com/OCA/OCB/pull/1234.patch + +Relative paths are resolved from the directory where ``gitaggregate`` is run. +A missing, unreachable or empty patch stops the aggregation with an error. Command line Usage ================== @@ -413,11 +423,13 @@ Contributors * Simone Orsi (camptocamp_) * Artem Kostyuk * Jan Verbeek +* Michael Tietz (MT_Software_) .. _ACSONE: https://www.acsone.eu .. _Tecnativa: https://www.tecnativa.com .. _camptocamp: https://www.camptocamp.com .. _LasLabs: https://laslabs.com +.. _MT_Software: https://github.com/mt-software-de Maintainer ---------- diff --git a/git_aggregator/command.py b/git_aggregator/command.py new file mode 100644 index 0000000..cb2e522 --- /dev/null +++ b/git_aggregator/command.py @@ -0,0 +1,31 @@ +# © 2015 ACSONE SA/NV +# Copyright 2023 Michael Tietz (MT Software) +# License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) +# Parts of the code comes from ANYBOX +# https://github.com/anybox/anybox.recipe.odoo +import logging +import subprocess + +from ._compat import console_to_str + +logger = logging.getLogger(__name__) + + +class CommandExecutor: + def __init__(self, cwd): + self.cwd = cwd + + def log_call(self, cmd, callwith=subprocess.check_call, + log_level=logging.DEBUG, **kw): + """Wrap a subprocess call with logging + :param meth: the calling method to use. + """ + logger.log(log_level, "%s> call %r", self.cwd, cmd) + try: + ret = callwith(cmd, **kw) + except Exception: + logger.error("%s> error calling %r", self.cwd, cmd) + raise + if callwith == subprocess.check_output: + ret = console_to_str(ret) + return ret diff --git a/git_aggregator/config.py b/git_aggregator/config.py index 1f95b66..7f52596 100644 --- a/git_aggregator/config.py +++ b/git_aggregator/config.py @@ -1,4 +1,5 @@ # © 2015 ACSONE SA/NV +# Copyright 2023 Michael Tietz (MT Software) # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) import logging @@ -9,10 +10,21 @@ from ._compat import string_types from .exception import ConfigException +from .patch import Patches log = logging.getLogger(__name__) +def update_patches(repo_dict, repo_data): + """Check and update repo_dict with patch files""" + patches_data = repo_data.get("patches") + patches = repo_dict.setdefault("patches", Patches()) + if not patches_data: + return + for patch in patches_data: + patches += Patches.prepare_patches(patch, repo_dict.get("cwd")) + + def get_repos(config, force=False): """Return a :py:obj:`list` list of repos from config file. :param config: the repos config in :py:class:`dict` format. @@ -126,6 +138,7 @@ def get_repos(config, force=False): cmds = [cmds] commands = cmds repo_dict['shell_command_after'] = commands + update_patches(repo_dict, repo_data) repo_list.append(repo_dict) return repo_list diff --git a/git_aggregator/patch.py b/git_aggregator/patch.py new file mode 100644 index 0000000..dbdc4a3 --- /dev/null +++ b/git_aggregator/patch.py @@ -0,0 +1,75 @@ +# Copyright 2023 Michael Tietz (MT Software) +# License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) +import logging +import subprocess +from pathlib import Path + +from .command import CommandExecutor +from .exception import ConfigException, GitAggregatorException + +logger = logging.getLogger(__name__) + + +class Patch(CommandExecutor): + is_local = False + + def __init__(self, path, cwd): + super().__init__(cwd) + self.path = path + if "://" not in path: + if not Path(path).is_file(): + raise ConfigException(f"Patch file not found: {path}") + self.is_local = True + + def retrive_data(self): + if self.is_local: + data = Path(self.path).read_bytes() + else: + cmd = [ + "curl", + "--fail", + "--location", + self.path, + ] + if logger.getEffectiveLevel() != logging.DEBUG: + cmd.append('--silent') + cmd.append('--show-error') + data = self.log_call( + cmd, + callwith=subprocess.run, + check=True, + stdout=subprocess.PIPE, + ).stdout + if not data.strip(): + raise GitAggregatorException(f"Patch is empty: {self.path}") + return data + + def apply(self): + data = self.retrive_data() + cmd = [ + "git", + "am", + ] + if logger.getEffectiveLevel() != logging.DEBUG: + cmd.append('--quiet') + self.log_call(cmd, callwith=subprocess.run, check=True, + cwd=self.cwd, input=data) + + +class Patches(list): + """List of patches""" + @staticmethod + def prepare_patches(path, cwd): + _path = Path(path) + patches = Patches() + if not _path.exists() or _path.is_file(): + patches.append(Patch(path, cwd)) + elif _path.is_dir(): + for fpath in sorted(_path.iterdir()): + if fpath.is_file(): + patches.append(Patch(str(fpath), cwd)) + return patches + + def apply(self): + for patch in self: + patch.apply() diff --git a/git_aggregator/repo.py b/git_aggregator/repo.py index 3801434..f4032e7 100644 --- a/git_aggregator/repo.py +++ b/git_aggregator/repo.py @@ -1,4 +1,5 @@ # © 2015 ACSONE SA/NV +# Copyright 2023 Michael Tietz (MT Software) # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) # Parts of the code comes from ANYBOX # https://github.com/anybox/anybox.recipe.odoo @@ -10,7 +11,9 @@ import requests from ._compat import console_to_str +from .command import CommandExecutor from .exception import DirtyException, GitAggregatorException +from .patch import Patches FETCH_DEFAULTS = ("depth", "shallow-since", "shallow-exclude") logger = logging.getLogger(__name__) @@ -30,13 +33,13 @@ def ishex(s): return True -class Repo: +class Repo(CommandExecutor): _git_version = None def __init__(self, cwd, remotes, merges, target, shell_command_after=None, fetch_all=False, defaults=None, - force=False): + force=False, patches=None): """Initialize a git repository aggregator :param cwd: path to the directory where to initialize the repository @@ -56,7 +59,7 @@ def __init__(self, cwd, remotes, merges, target, :param bool force: When ``False``, it will stop if repo is dirty. """ - self.cwd = cwd + super().__init__(cwd) self.remotes = remotes if fetch_all is True: self.fetch_all = frozenset(r["name"] for r in remotes) @@ -67,6 +70,7 @@ def __init__(self, cwd, remotes, merges, target, self.shell_command_after = shell_command_after or [] self.defaults = defaults or dict() self.force = force + self.patches = patches or Patches() @property def git_version(self): @@ -148,21 +152,6 @@ def query_remote_ref(self, remote, ref): return 'HEAD', sha return None, ref - def log_call(self, cmd, callwith=subprocess.check_call, - log_level=logging.DEBUG, **kw): - """Wrap a subprocess call with logging - :param meth: the calling method to use. - """ - logger.log(log_level, "%s> call %r", self.cwd, cmd) - try: - ret = callwith(cmd, **kw) - except Exception: - logger.error("%s> error calling %r", self.cwd, cmd) - raise - if callwith == subprocess.check_output: - ret = console_to_str(ret) - return ret - def aggregate(self): """ Aggregate all merges into the target branch If the target_dir doesn't exist, create an empty git repo otherwise @@ -187,6 +176,7 @@ def aggregate(self): self._reset_to(origin["remote"], origin["ref"]) for merge in merges: self._merge(merge) + self.patches.apply() self._execute_shell_command_after() logger.info('End aggregation of %s', self.cwd) diff --git a/tests/test_config.py b/tests/test_config.py index ab2eba2..5190396 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,4 +1,5 @@ # © 2015 ACSONE SA/NV +# Copyright 2025 Michael Tietz (MT Software) # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) import os import tempfile @@ -10,6 +11,7 @@ from git_aggregator import config from git_aggregator._compat import PY2 from git_aggregator.exception import ConfigException +from git_aggregator.patch import Patches class TestConfig(unittest.TestCase): @@ -44,6 +46,7 @@ def test_load(self): 'merges': [{'ref': '8.0', 'remote': 'oca'}, {'ref': 'refs/pull/105/head', 'remote': 'oca'}, {'ref': 'refs/pull/106/head', 'remote': 'oca'}], + 'patches': [], 'remotes': [], 'shell_command_after': [], 'target': {'branch': 'aggregated_branch_name', @@ -57,6 +60,40 @@ def test_load(self): 'url': 'git+ssh://git@github.com/acsone/product-attribute.git'}]) + def test_load_patches(self): + with tempfile.TemporaryDirectory() as tmpdir: + patch_file = os.path.join(tmpdir, 'single.patch') + patch_dir = os.path.join(tmpdir, 'patches') + os.mkdir(patch_dir) + os.mkdir(os.path.join(patch_dir, 'subdir')) + for path in (patch_file, + os.path.join(patch_dir, '0002-b.patch'), + os.path.join(patch_dir, '0001-a.patch')): + with open(path, 'w') as f: + f.write('') + url = 'https://github.com/OCA/web/pull/1.patch' + config_yaml = dedent(f""" + /web: + remotes: + oca: https://github.com/OCA/web.git + merges: + - oca 8.0 + target: oca aggregated_branch_name + patches: + - {patch_file} + - {patch_dir} + - {url} + """) + repos = config.get_repos(self._parse_config(config_yaml)) + patches = repos[0]['patches'] + self.assertIsInstance(patches, Patches) + self.assertEqual( + [(p.path, p.is_local, p.cwd) for p in patches], + [(patch_file, True, '/web'), + (os.path.join(patch_dir, '0001-a.patch'), True, '/web'), + (os.path.join(patch_dir, '0002-b.patch'), True, '/web'), + (url, False, '/web')]) + def test_load_defaults(self): config_yaml = dedent(""" /web: @@ -91,6 +128,7 @@ def test_load_defaults(self): 'merges': [{'ref': '8.0', 'remote': 'oca', 'depth': 1000}, {'ref': 'refs/pull/105/head', 'remote': 'oca'}, {'ref': 'refs/pull/106/head', 'remote': 'oca'}], + 'patches': [], 'remotes': [], 'shell_command_after': [], 'target': {'branch': 'aggregated_branch_name', diff --git a/tests/test_repo.py b/tests/test_repo.py index 91c303e..7c4fc26 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,4 +1,5 @@ # © 2015 ACSONE SA/NV +# Copyright 2026 Michael Tietz (MT Software) # License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html) # Parts of the code comes from ANYBOX # https://github.com/anybox/anybox.recipe.odoo @@ -22,6 +23,7 @@ from textwrap import dedent from git_aggregator import exception, main +from git_aggregator.patch import Patches from git_aggregator.repo import Repo from git_aggregator.utils import ( WorkingDirectoryKeeper, @@ -126,6 +128,84 @@ def test_minimal(self): last_rev = git_get_last_rev(self.cwd) self.assertEqual(last_rev, self.commit_1_sha) + def _make_patches(self, count): + """Commit ``count`` changes on top of remote1 master in a scratch + clone and export them with git format-patch. + Return the directory holding the patch files.""" + scratch = os.path.join(self.sandbox, 'scratch') + patch_dir = os.path.join(self.sandbox, 'patches') + subprocess.check_call(['git', 'clone', self.url_remote1, scratch]) + for i in range(1, count + 1): + git_write_commit( + scratch, 'patched', f"patch {i}", msg=f"patch commit {i}") + subprocess.check_call( + ['git', 'format-patch', f'-{count}', '-o', patch_dir], + cwd=scratch) + return patch_dir + + def _aggregate_with_patches(self, patch_path): + remotes = [{ + 'name': 'r1', + 'url': self.url_remote1 + }] + merges = [{ + 'remote': 'r1', + 'ref': 'master' + }] + target = { + 'remote': 'r1', + 'branch': 'agg' + } + patches = Patches.prepare_patches(patch_path, self.cwd) + repo = Repo(self.cwd, remotes, merges, target, patches=patches) + repo.aggregate() + + def _git_log_subjects(self, count): + return subprocess.check_output( + ['git', 'log', f'-{count}', '--format=%s'], + cwd=self.cwd, universal_newlines=True).splitlines() + + def test_patch_file(self): + patch_dir = self._make_patches(1) + patch_file = os.path.join(patch_dir, os.listdir(patch_dir)[0]) + self._aggregate_with_patches(patch_file) + self.assertEqual( + self._git_log_subjects(2), ["patch commit 1", "last commit"]) + with open(os.path.join(self.cwd, 'patched')) as f: + self.assertEqual(f.read(), "patch 1") + + def test_patch_dir(self): + patch_dir = self._make_patches(2) + self._aggregate_with_patches(patch_dir) + self.assertEqual( + self._git_log_subjects(3), + ["patch commit 2", "patch commit 1", "last commit"]) + with open(os.path.join(self.cwd, 'patched')) as f: + self.assertEqual(f.read(), "patch 2") + + def test_patch_url(self): + patch_dir = self._make_patches(1) + patch_file = os.path.join(patch_dir, os.listdir(patch_dir)[0]) + self._aggregate_with_patches(path2url(patch_file)) + self.assertEqual( + self._git_log_subjects(2), ["patch commit 1", "last commit"]) + + def test_patch_missing_file(self): + with self.assertRaises(exception.ConfigException): + Patches.prepare_patches( + os.path.join(self.sandbox, 'missing.patch'), self.cwd) + + def test_patch_unreachable_url(self): + url = path2url(os.path.join(self.sandbox, 'missing.patch')) + with self.assertRaises(subprocess.CalledProcessError): + self._aggregate_with_patches(url) + + def test_patch_empty(self): + patch_file = os.path.join(self.sandbox, 'empty.patch') + open(patch_file, 'w').close() + with self.assertRaises(exception.GitAggregatorException): + self._aggregate_with_patches(patch_file) + def test_empty_dir(self): # ensure git clone in empty directory works remotes = [{