From 37e27800669ef0c0acd4555e36984f12d503e661 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Aug 08 2016 11:42:45 +0000 Subject: Add a command to copy part of compose This command will call rsync to copy selected variants or arches to another location. Depending on how the filters are set, there may be more than one call to rsync. There is basic logic to try to minimize the number of rsync invocations, but there are cases where the job can not be done with a single call. There is a dry-run option to check what will happen. All unrecognized options will be passed directly to rsync. Signed-off-by: Lubomír Sedlář --- diff --git a/bin/compose-partial-copy b/bin/compose-partial-copy new file mode 100755 index 0000000..763cf5f --- /dev/null +++ b/bin/compose-partial-copy @@ -0,0 +1,47 @@ +#!/usr/bin/env python2 +# -*- encoding: utf-8 -*- + +import argparse +import os +import sys + +import productmd.compose + +here = sys.path[0] +if here != '/usr/bin': + sys.path[0] = os.path.dirname(here) + +import compose_utils + + +def run(opts, rsync_opts=[]): + compose = productmd.compose.Compose(opts.compose) + + compose_utils.copy_compose(compose, opts.dest, + variants=opts.variant, arches=opts.arch, + dry_run=opts.dry_run, + rsync_opts=rsync_opts) + + +DESCRIPTION = ('Copy parts of compose to another location via rsync. ' + 'You can specify which variants and/or architectures you ' + 'want to copy. Multiple invocations of rsync may be used ' + 'to achieve the desired result. ' + 'Unrecognized options will be passed directly to rsync.') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=DESCRIPTION) + parser.add_argument('compose', metavar='COMPOSE', + help='path to compose that should be copied') + parser.add_argument('dest', metavar='DEST', + help='where to copy the data (variants will form subdirectories here)') + parser.add_argument('--arch', action='append', default=[], + help='copy only this architecture; can be used multiple times') + parser.add_argument('--variant', action='append', default=[], + help='copy only this variant; can be used multiple times') + parser.add_argument('-n', '--dry-run', default=False, action='store_true', + help='only print rsync commands, do not copy anything') + + opts, remaining = parser.parse_known_args() + run(opts, remaining) diff --git a/compose-utils.spec b/compose-utils.spec index d6722b4..3fcd40c 100644 --- a/compose-utils.spec +++ b/compose-utils.spec @@ -18,6 +18,7 @@ BuildRequires: kobo-rpmlib Requires: python-productmd Requires: kobo Requires: kobo-rpmlib +Requires: rsync BuildArch: noarch diff --git a/compose_utils/__init__.py b/compose_utils/__init__.py index e69de29..a2ad8fd 100644 --- a/compose_utils/__init__.py +++ b/compose_utils/__init__.py @@ -0,0 +1,4 @@ +# -*- encoding: utf-8 -*- + + +from .copy_compose import copy_compose # noqa diff --git a/compose_utils/copy_compose.py b/compose_utils/copy_compose.py new file mode 100644 index 0000000..5baf86f --- /dev/null +++ b/compose_utils/copy_compose.py @@ -0,0 +1,103 @@ +# -*- encoding: utf-8 -*- + +from __future__ import print_function + +import os +from kobo import shortcuts + + +def copy_compose(compose, dest, arches=None, variants=None, dry_run=False, rsync_opts=[]): + """Copy (possibly part of) a compose to another location. + + :param compose: a compose instance to work with + :param dest: a file path where the to copy + :param arches: a list of arches that should be copied; if not specified, + all arches are considered + :param variants: a list of variant UIDs to copy; if not specified, all of + them are copied + :param dry_run: When `True`, the commands are printed, but not actually ran + """ + arches = set(arches or []) + + paths = [] + + for variant in compose.info.variants.get_variants(recursive=True): + if variants and variant.uid not in variants: + continue + + if not arches or (set(variant.arches) & arches) == variant.arches: + paths.extend(_collect_variant(compose, variant, dest, arch=None)) + else: + for arch in variant.arches: + if arch not in arches: + continue + paths.extend(_collect_variant(compose, variant, dest, arch=arch)) + + paths = _reduce(compose, paths) + for parent, children in paths.iteritems(): + sources = sorted([os.path.join(parent, child) for child in children]) + destination = os.path.join(dest, parent.replace(compose.compose_path, '').lstrip('/')) + _run_rsync(sources, destination, dry_run=dry_run, opts=rsync_opts) + + +def _run_rsync(sources, destination, dry_run=False, opts=[]): + cmd = ['rsync', '-avHh'] + opts + sources + [destination] + print(' '.join(cmd)) + if not dry_run: + shortcuts.run(cmd, stdout=True) + + +def _collect_variant(compose, variant, dest, arch=None): + """Find all paths in a variant that should be copied. + If arch is not specified, that means all paths, otherwise only paths + defined for the arch will be copied. + """ + sources = set() + for path_type in variant.paths._fields: + paths = getattr(variant.paths, path_type) + if not arch: + for path in paths.itervalues(): + sources.add(path) + elif arch in paths: + sources.add(paths[arch]) + + return sources + + +def _reduce(compose, paths): + """Given a list of paths, try to find a minimal set that can be rsync-ed to + achive the same result. + + The main idea is that if we are supposed to copy everything in a particular + directory, we can just simply copy the parent. + """ + # Create a mapping from parent dirs to a set of children that should be + # copied. + sources = [os.path.split(os.path.join(compose.compose_path, x)) + for x in sorted(paths)] + mapping = {} + for dir, content in sources: + mapping.setdefault(dir, set()).add(content) + + while True: + has_change = False + for dir in mapping: + if set(os.listdir(dir)) <= mapping[dir]: + # The directory has all children marked, so we can remove it. + # Instead, we will add it as a child of its own parent. + mapping.pop(dir) + parent, name = os.path.split(dir) + mapping.setdefault(parent, set()).add(name) + has_change = True + + # If any other path starts with the removed directory, we can + # remove it too. + for other in mapping.keys(): + if (other.rstrip('/') + '/').startswith(dir + '/'): + mapping.pop(other) + + break + if not has_change: + break + + return mapping diff --git a/tests/test_copy_compose.py b/tests/test_copy_compose.py new file mode 100644 index 0000000..b88e641 --- /dev/null +++ b/tests/test_copy_compose.py @@ -0,0 +1,158 @@ +# -*- encoding: utf-8 -*- + +import unittest +import mock +from StringIO import StringIO + +from compose_utils import copy_compose + +from .helpers import get_compose + + +DEST = '/mnt/public/composes/' + + +class ChangelogTest(unittest.TestCase): + + def setUp(self): + self.compose = get_compose('DP-1.0-20160315.t.0') + + @mock.patch('kobo.shortcuts.run') + def test_copy_full(self, mock_run): + copy_compose(self.compose, DEST, variants=[], arches=[]) + self.assertItemsEqual( + mock_run.mock_calls, + [mock.call(['rsync', '-avHh', + self.compose.compose_path + '/Client', + self.compose.compose_path + '/Server', + DEST], + stdout=True)]) + + @mock.patch('kobo.shortcuts.run') + def test_copy_full_with_extra_opts(self, mock_run): + copy_compose(self.compose, DEST, variants=[], arches=[], rsync_opts=['--exclude=repodata']) + self.assertItemsEqual( + mock_run.mock_calls, + [mock.call(['rsync', '-avHh', '--exclude=repodata', + self.compose.compose_path + '/Client', + self.compose.compose_path + '/Server', + DEST], + stdout=True)]) + + @mock.patch('kobo.shortcuts.run') + def test_copy_one_variant(self, mock_run): + copy_compose(self.compose, DEST, variants=['Client'], arches=[]) + self.assertItemsEqual( + mock_run.mock_calls, + [mock.call(['rsync', '-avHh', + self.compose.compose_path + '/Client', DEST], + stdout=True)]) + + @mock.patch('kobo.shortcuts.run') + def test_copy_one_arch_in_single_variant(self, mock_run): + copy_compose(self.compose, DEST, variants=[], arches=['i386']) + self.assertItemsEqual( + mock_run.mock_calls, + [mock.call(['rsync', '-avHh', + self.compose.compose_path + '/Client/i386', + self.compose.compose_path + '/Client/source', + DEST + 'Client'], + stdout=True)]) + + @mock.patch('kobo.shortcuts.run') + def test_copy_one_arch_in_both_variants(self, mock_run): + copy_compose(self.compose, DEST, variants=[], arches=['x86_64']) + self.assertItemsEqual( + mock_run.mock_calls, + [mock.call(['rsync', '-avHh', + self.compose.compose_path + '/Client/source', + self.compose.compose_path + '/Client/x86_64', + DEST + 'Client'], + stdout=True), + mock.call(['rsync', '-avHh', + self.compose.compose_path + '/Server/source', + self.compose.compose_path + '/Server/x86_64', + DEST + 'Server'], + stdout=True)]) + + @mock.patch('kobo.shortcuts.run') + def test_copy_one_arch_in_one_variant_with_filter(self, mock_run): + copy_compose(self.compose, DEST, variants=['Client'], arches=['x86_64']) + self.assertItemsEqual( + mock_run.mock_calls, + [mock.call(['rsync', '-avHh', + self.compose.compose_path + '/Client/source', + self.compose.compose_path + '/Client/x86_64', + DEST + 'Client'], + stdout=True)]) + + @mock.patch('kobo.shortcuts.run') + def test_copy_full_dry_run(self, mock_run): + with mock.patch('sys.stdout', new=StringIO()) as out: + copy_compose(self.compose, DEST, variants=[], arches=[], dry_run=True) + + self.assertEqual(mock_run.mock_calls, []) + self.assertItemsEqual( + out.getvalue().strip().split('\n'), + [' '.join(['rsync', '-avHh', + self.compose.compose_path + '/Client', + self.compose.compose_path + '/Server', + DEST])]) + + @mock.patch('kobo.shortcuts.run') + def test_copy_one_variant_dry_run(self, mock_run): + with mock.patch('sys.stdout', new=StringIO()) as out: + copy_compose(self.compose, DEST, variants=['Client'], arches=[], dry_run=True) + + self.assertEqual(mock_run.mock_calls, []) + self.assertItemsEqual( + out.getvalue().strip().split('\n'), + [' '.join(['rsync', '-avHh', + self.compose.compose_path + '/Client', DEST])]) + + @mock.patch('kobo.shortcuts.run') + def test_copy_one_arch_in_single_variant_dry_run(self, mock_run): + with mock.patch('sys.stdout', new=StringIO()) as out: + copy_compose(self.compose, DEST, variants=[], arches=['i386'], dry_run=True) + + self.assertEqual(mock_run.mock_calls, []) + self.assertItemsEqual( + out.getvalue().strip().split('\n'), + [' '.join(['rsync', '-avHh', + self.compose.compose_path + '/Client/i386', + self.compose.compose_path + '/Client/source', + DEST + 'Client'])]) + + @mock.patch('kobo.shortcuts.run') + def test_copy_one_arch_in_both_variants_dry_run(self, mock_run): + with mock.patch('sys.stdout', new=StringIO()) as out: + copy_compose(self.compose, DEST, variants=[], arches=['x86_64'], dry_run=True) + + self.assertEqual(mock_run.mock_calls, []) + self.assertItemsEqual( + out.getvalue().strip().split('\n'), + [' '.join(['rsync', '-avHh', + self.compose.compose_path + '/Client/source', + self.compose.compose_path + '/Client/x86_64', + DEST + 'Client']), + ' '.join(['rsync', '-avHh', + self.compose.compose_path + '/Server/source', + self.compose.compose_path + '/Server/x86_64', + DEST + 'Server'])]) + + @mock.patch('kobo.shortcuts.run') + def test_copy_one_arch_in_one_variant_with_filter_dry_run(self, mock_run): + with mock.patch('sys.stdout', new=StringIO()) as out: + copy_compose(self.compose, DEST, variants=['Client'], arches=['x86_64'], dry_run=True) + + self.assertEqual(mock_run.mock_calls, []) + self.assertEqual( + out.getvalue().strip(), + ' '.join(['rsync', '-avHh', + self.compose.compose_path + '/Client/source', + self.compose.compose_path + '/Client/x86_64', + DEST + 'Client'])) + + +if __name__ == '__main__': + unittest.main()