From 897bab4d4f784e55a8396333f530e606e71d3c53 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 30 Jul 2015 09:23:00 -0600 Subject: [PATCH] Fixes for pkgbuild, added support for dependencies building package (cherry picked from commit 1c86e0110f06f5bee5e53d796de2b1dab8ad4fe6) --- salt/modules/debbuild.py | 32 ++++++++++++----- salt/modules/rpmbuild.py | 75 +++++++++++++++++++++++++++++++++++----- salt/states/pkgbuild.py | 25 +++++++++----- 3 files changed, 106 insertions(+), 26 deletions(-) diff --git a/salt/modules/debbuild.py b/salt/modules/debbuild.py index 19fefaa094..90bcd27b45 100644 --- a/salt/modules/debbuild.py +++ b/salt/modules/debbuild.py @@ -22,7 +22,6 @@ import salt.utils # pylint: disable=import-error - __virtualname__ = 'pkgbuild' @@ -129,7 +128,7 @@ def _get_src(tree_base, source, saltenv='base'): sbase = os.path.basename(source) dest = os.path.join(tree_base, sbase) if parsed.scheme: - lsrc = __salt__['cp.get_url'](source, dest, saltenv=saltenv) + __salt__['cp.get_url'](source, dest, saltenv=saltenv) else: shutil.copy(source, dest) @@ -148,18 +147,34 @@ def make_src_pkg(dest_dir, spec, sources, template=None, saltenv='base'): ''' _create_pbuilders() tree_base = _mk_tree() + ret = [] + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + spec_pathfile = _get_spec(tree_base, spec, template, saltenv) + + # build salt equivalents from scratch if isinstance(sources, str): sources = sources.split(',') for src in sources: _get_src(tree_base, src, saltenv) + #.dsc then assumes sources already build + if spec_pathfile.endswith('.dsc'): + for efile in os.listdir(tree_base): + full = os.path.join(tree_base, efile) + trgt = os.path.join(dest_dir, efile) + shutil.copy(full, trgt) + ret.append(trgt) + + trgt = os.path.join(dest_dir, os.path.basename(spec_pathfile)) + shutil.copy(spec_pathfile, trgt) + ret.append(trgt) + + return ret + # obtain name of 'python setup.py sdist' generated tarball, extract the version # and manipulate the name for debian use (convert minix and add '+ds') - ret = [] - if not os.path.isdir(dest_dir): - os.makedirs(dest_dir) - salttarball = None for afile in os.listdir(tree_base): if afile.startswith('salt-') and afile.endswith('.tar.gz'): @@ -206,7 +221,7 @@ def make_src_pkg(dest_dir, spec, sources, template=None, saltenv='base'): return ret -def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'): +def build(runas, tgt, dest_dir, spec, sources, deps, template, saltenv='base'): ''' Given the package destination directory, the tarball containing debian files (e.g. control) and package sources, use pbuilder to safely build the platform package @@ -232,8 +247,7 @@ def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'): for dsc in dscs: afile = os.path.basename(dsc) adist = os.path.join(dest_dir, afile) - if afile != os.path.basename(spec): - shutil.copy(dsc, adist) + shutil.copy(dsc, adist) if dsc.endswith('.dsc'): dbase = os.path.dirname(dsc) diff --git a/salt/modules/rpmbuild.py b/salt/modules/rpmbuild.py index 209f40def4..15b8344c1e 100644 --- a/salt/modules/rpmbuild.py +++ b/salt/modules/rpmbuild.py @@ -19,6 +19,7 @@ from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: dis # Import salt libs import salt.utils +from salt.exceptions import SaltInvocationError __virtualname__ = 'pkgbuild' @@ -71,9 +72,9 @@ def _get_spec(tree_base, spec, template, saltenv='base'): spec_tgt = os.path.basename(spec) dest = os.path.join(tree_base, 'SPECS', spec_tgt) return __salt__['cp.get_url']( - spec, - dest, - saltenv=saltenv) + spec, + dest, + saltenv=saltenv) def _get_src(tree_base, source, saltenv='base'): @@ -89,6 +90,46 @@ def _get_src(tree_base, source, saltenv='base'): shutil.copy(source, dest) +def _get_distset(tgt): + ''' + Get the distribution string for use with rpmbuild and mock + ''' + # Centos adds that string to rpm names, removing that to have + # consistent naming on Centos and Redhat + tgtattrs = tgt.split('-') + if tgtattrs[1] in ['5', '6', '7']: + distset = '--define "dist .el{0}"'.format(tgtattrs[1]) + else: + distset = "" + + return distset + + +def _get_deps(deps, tree_base, saltenv='base'): + ''' + Get include string for list of dependent rpms to build package + ''' + deps_list = "" + if deps is None: + return deps_list + if not isinstance(deps, list): + raise SaltInvocationError( + '\'deps\' must be a Python list or comma-separated string' + ) + for deprpm in deps: + parsed = _urlparse(deprpm) + depbase = os.path.basename(deprpm) + dest = os.path.join(tree_base, depbase) + if parsed.scheme: + __salt__['cp.get_url'](deprpm, dest, saltenv=saltenv) + else: + shutil.copy(deprpm, dest) + + deps_list += ' --install {0}'.format(dest) + + return deps_list + + def make_src_pkg(dest_dir, spec, sources, template=None, saltenv='base'): ''' Create a source rpm from the given spec file and sources @@ -107,7 +148,9 @@ def make_src_pkg(dest_dir, spec, sources, template=None, saltenv='base'): sources = sources.split(',') for src in sources: _get_src(tree_base, src, saltenv) - cmd = 'rpmbuild --define "_topdir {0}" -bs {1}'.format(tree_base, spec_path) + + # make source rpms for dist el5, usable with mock on other dists + cmd = 'rpmbuild --define "_topdir {0}" -bs --define "_source_filedigest_algorithm md5" --define "_binary_filedigest_algorithm md5" --define "dist .el5" {1}'.format(tree_base, spec_path) __salt__['cmd.run'](cmd) srpms = os.path.join(tree_base, 'SRPMS') ret = [] @@ -121,7 +164,7 @@ def make_src_pkg(dest_dir, spec, sources, template=None, saltenv='base'): return ret -def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'): +def build(runas, tgt, dest_dir, spec, sources, deps, template, saltenv='base'): ''' Given the package destination directory, the spec file source and package sources, use mock to safely build the rpm defined in the spec file @@ -141,6 +184,17 @@ def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'): pass srpm_dir = tempfile.mkdtemp() srpms = make_src_pkg(srpm_dir, spec, sources, template, saltenv) + + distset = _get_distset(tgt) + + noclean = "" + deps_dir = tempfile.mkdtemp() + deps_list = _get_deps(deps, deps_dir, saltenv) + if deps_list and not deps_list.isspace(): + cmd = 'mock --root={0} {1}'.format(tgt, deps_list) + __salt__['cmd.run'](cmd, runas=runas) + noclean += " --no-clean" + for srpm in srpms: dbase = os.path.dirname(srpm) cmd = 'chown {0} -R {1}'.format(runas, dbase) @@ -148,10 +202,12 @@ def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'): results_dir = tempfile.mkdtemp() cmd = 'chown {0} -R {1}'.format(runas, results_dir) __salt__['cmd.run'](cmd) - cmd = 'mock -r {0} --rebuild {1} --resultdir {2}'.format( - tgt, - srpm, - results_dir) + cmd = 'mock --root={0} --resultdir={1} {2} {3} {4}'.format( + tgt, + results_dir, + distset, + noclean, + srpm) __salt__['cmd.run'](cmd, runas=runas) for rpm in os.listdir(results_dir): full = os.path.join(results_dir, rpm) @@ -170,6 +226,7 @@ def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'): with salt.utils.fopen(full, 'r') as fp_: ret[rpm] = fp_.read() shutil.rmtree(results_dir) + shutil.rmtree(deps_dir) shutil.rmtree(srpm_dir) return ret diff --git a/salt/states/pkgbuild.py b/salt/states/pkgbuild.py index 630af60182..5dd717f51d 100644 --- a/salt/states/pkgbuild.py +++ b/salt/states/pkgbuild.py @@ -53,6 +53,7 @@ def built( sources, template, tgt, + deps=None, results=None, always=False, saltenv='base'): @@ -80,6 +81,13 @@ def built( tgt The target platform to run the build on + deps + Packages required to ensure that the named package is built + can be hosted on either the salt master server or on an HTTP + or FTP server. Both HTTPS and HTTP are supported as well as + downloading directly from Amazon S3 compatible URLs with both + pre-configured and automatic IAM credentials + results The names of the expected rpms that will be built @@ -101,7 +109,7 @@ def built( present = set() if os.path.isdir(dest_dir): for fn_ in os.listdir(dest_dir): - present.appd(fn_) + present.add(fn_) need = results.difference(present) if not need: ret['comment'] = 'All needed packages exist' @@ -111,13 +119,14 @@ def built( ret['result'] = None return ret ret['changes'] = __salt__['pkgbuild.build']( - runas, - tgt, - dest_dir, - spec, - sources, - template, - saltenv) + runas, + tgt, + dest_dir, + spec, + sources, + deps, + template, + saltenv) ret['comment'] = 'Packages Built' return ret