mirror of
https://github.com/valitydev/salt.git
synced 2024-11-07 08:58:59 +00:00
Merge pull request #25906 from saltstack/dgm_pkgbuild
Cherry-pick of pkgbuild changes from develop branch
This commit is contained in:
Justin Findlay
commit
76d08d93f4
311
salt/modules/debbuild.py
Normal file
311
salt/modules/debbuild.py
Normal file
@ -0,0 +1,311 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
Debian Package builder system
|
||||
|
||||
.. versionadded:: Beryllium
|
||||
|
||||
This system allows for all of the components to build debs safely in chrooted
|
||||
environments. This also provides a function to generate debian repositories
|
||||
|
||||
This module impliments the pkgbuild interface
|
||||
'''
|
||||
|
||||
# import python libs
|
||||
from __future__ import absolute_import, print_function
|
||||
import os
|
||||
import tempfile
|
||||
import shutil
|
||||
from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: disable=no-name-in-module,import-error
|
||||
|
||||
# Import salt libs
|
||||
import salt.utils
|
||||
|
||||
# pylint: disable=import-error
|
||||
|
||||
__virtualname__ = 'pkgbuild'
|
||||
|
||||
|
||||
def __virtual__():
|
||||
'''
|
||||
Confirm this module is on a Debian based system
|
||||
'''
|
||||
if __grains__.get('os_family', False) in ('Kali', 'Debian'):
|
||||
return __virtualname__
|
||||
return False
|
||||
|
||||
|
||||
def _create_pbuilders():
|
||||
'''
|
||||
Create the .pbuilder family of files in user's home directory
|
||||
'''
|
||||
hook_text = '''#!/bin/sh
|
||||
set -e
|
||||
cat > "/etc/apt/preferences" << EOF
|
||||
|
||||
Package: python-abalaster
|
||||
Pin: release a=testing
|
||||
Pin-Priority: 950
|
||||
|
||||
Package: python-sphinx
|
||||
Pin: release a=experimental
|
||||
Pin-Priority: 900
|
||||
|
||||
Package: sphinx-common
|
||||
Pin: release a=experimental
|
||||
Pin-Priority: 900
|
||||
|
||||
Package: *
|
||||
Pin: release a=jessie-backports
|
||||
Pin-Priority: 750
|
||||
|
||||
Package: *
|
||||
Pin: release a=stable
|
||||
Pin-Priority: 700
|
||||
|
||||
Package: *
|
||||
Pin: release a=testing
|
||||
Pin-Priority: 650
|
||||
|
||||
Package: *
|
||||
Pin: release a=unstable
|
||||
Pin-Priority: 600
|
||||
|
||||
Package: *
|
||||
Pin: release a=experimental
|
||||
Pin-Priority: 550
|
||||
|
||||
EOF
|
||||
'''
|
||||
|
||||
pbldrc_text = '''DIST="jessie"
|
||||
if [ -n "${DIST}" ]; then
|
||||
TMPDIR=/tmp
|
||||
BASETGZ="`dirname $BASETGZ`/$DIST-base.tgz"
|
||||
DISTRIBUTION=$DIST
|
||||
APTCACHE="/var/cache/pbuilder/$DIST/aptcache"
|
||||
fi
|
||||
HOOKDIR="${HOME}/.pbuilder-hooks"
|
||||
OTHERMIRROR="deb http://ftp.us.debian.org/debian/ testing main contrib non-free | deb http://ftp.us.debian.org/debian/ experimental main contrib non-free"
|
||||
'''
|
||||
home = os.path.expanduser('~')
|
||||
pbuilder_hooksdir = os.path.join(home, '.pbuilder-hooks')
|
||||
if not os.path.isdir(pbuilder_hooksdir):
|
||||
os.makedirs(pbuilder_hooksdir)
|
||||
|
||||
d05hook = os.path.join(pbuilder_hooksdir, 'D05apt-preferences')
|
||||
with open(d05hook, "w") as fow:
|
||||
fow.write('{0}'.format(hook_text))
|
||||
|
||||
pbuilderrc = os.path.join(home, '.pbuilderrc')
|
||||
with open(pbuilderrc, "w") as fow:
|
||||
fow.write('{0}'.format(pbldrc_text))
|
||||
|
||||
|
||||
def _mk_tree():
|
||||
'''
|
||||
Create the debian build area
|
||||
'''
|
||||
basedir = tempfile.mkdtemp()
|
||||
return basedir
|
||||
|
||||
|
||||
def _get_spec(tree_base, spec, template, saltenv='base'):
|
||||
'''
|
||||
Get the spec file (tarball of the debian sub-dir to use)
|
||||
and place it in build area
|
||||
|
||||
'''
|
||||
spec_tgt = os.path.basename(spec)
|
||||
dest = os.path.join(tree_base, spec_tgt)
|
||||
return __salt__['cp.get_url'](spec, dest, saltenv=saltenv)
|
||||
|
||||
|
||||
def _get_src(tree_base, source, saltenv='base'):
|
||||
'''
|
||||
Get the named sources and place them into the tree_base
|
||||
'''
|
||||
parsed = _urlparse(source)
|
||||
sbase = os.path.basename(source)
|
||||
dest = os.path.join(tree_base, sbase)
|
||||
if parsed.scheme:
|
||||
__salt__['cp.get_url'](source, dest, saltenv=saltenv)
|
||||
else:
|
||||
shutil.copy(source, dest)
|
||||
|
||||
|
||||
def make_src_pkg(dest_dir, spec, sources, template=None, saltenv='base'):
|
||||
'''
|
||||
Create a platform specific source package from the given platform spec/control file and sources
|
||||
|
||||
CLI Example:
|
||||
|
||||
Debian
|
||||
salt '*' pkgbuild.make_src_pkg /var/www/html/ https://raw.githubusercontent.com/saltstack/libnacl/master/pkg/deb/python-libnacl.control.tar.xz https://pypi.python.org/packages/source/l/libnacl/libnacl-1.3.5.tar.gz
|
||||
|
||||
This example command should build the libnacl SOURCE package and place it in
|
||||
/var/www/html/ on the minion
|
||||
'''
|
||||
_create_pbuilders()
|
||||
tree_base = _mk_tree()
|
||||
ret = []
|
||||
if not os.path.isdir(dest_dir):
|
||||
os.makedirs(dest_dir)
|
||||
|
||||
spec_pathfile = _get_spec(tree_base, spec, template, saltenv)
|
||||
|
||||
# build salt equivalents from scratch
|
||||
if isinstance(sources, str):
|
||||
sources = sources.split(',')
|
||||
for src in sources:
|
||||
_get_src(tree_base, src, saltenv)
|
||||
|
||||
#.dsc then assumes sources already build
|
||||
if spec_pathfile.endswith('.dsc'):
|
||||
for efile in os.listdir(tree_base):
|
||||
full = os.path.join(tree_base, efile)
|
||||
trgt = os.path.join(dest_dir, efile)
|
||||
shutil.copy(full, trgt)
|
||||
ret.append(trgt)
|
||||
|
||||
trgt = os.path.join(dest_dir, os.path.basename(spec_pathfile))
|
||||
shutil.copy(spec_pathfile, trgt)
|
||||
ret.append(trgt)
|
||||
|
||||
return ret
|
||||
|
||||
# obtain name of 'python setup.py sdist' generated tarball, extract the version
|
||||
# and manipulate the name for debian use (convert minix and add '+ds')
|
||||
salttarball = None
|
||||
for afile in os.listdir(tree_base):
|
||||
if afile.startswith('salt-') and afile.endswith('.tar.gz'):
|
||||
salttarball = afile
|
||||
break
|
||||
else:
|
||||
return ret
|
||||
|
||||
frontname = salttarball.split('.tar.gz')
|
||||
salttar_name = frontname[0]
|
||||
debname = salttar_name.replace('-', '_')
|
||||
debname += '+ds'
|
||||
debname_orig = debname + '.orig.tar.gz'
|
||||
abspath_debname = os.path.join(tree_base, debname)
|
||||
|
||||
cmd = 'tar -xvzf {0}'.format(salttarball)
|
||||
__salt__['cmd.run'](cmd, cwd=tree_base)
|
||||
cmd = 'mv {0} {1}'.format(salttar_name, debname)
|
||||
__salt__['cmd.run'](cmd, cwd=tree_base)
|
||||
cmd = 'tar -cvzf {0} {1}'.format(os.path.join(tree_base, debname_orig), debname)
|
||||
__salt__['cmd.run'](cmd, cwd=tree_base)
|
||||
cmd = 'rm -f {0}'.format(salttarball)
|
||||
__salt__['cmd.run'](cmd, cwd=tree_base)
|
||||
cmd = 'cp {0} {1}'.format(spec_pathfile, abspath_debname)
|
||||
__salt__['cmd.run'](cmd, cwd=abspath_debname)
|
||||
cmd = 'tar -xvJf {0}'.format(spec_pathfile)
|
||||
__salt__['cmd.run'](cmd, cwd=abspath_debname)
|
||||
cmd = 'rm -f {0}'.format(os.path.basename(spec_pathfile))
|
||||
__salt__['cmd.run'](cmd, cwd=abspath_debname)
|
||||
cmd = 'debuild -S -uc -us'
|
||||
__salt__['cmd.run'](cmd, cwd=abspath_debname, python_shell=True)
|
||||
|
||||
cmd = 'rm -fR {0}'.format(abspath_debname)
|
||||
__salt__['cmd.run'](cmd)
|
||||
|
||||
for dfile in os.listdir(tree_base):
|
||||
if dfile.startswith('salt_'):
|
||||
if not dfile.endswith('.build'):
|
||||
full = os.path.join(tree_base, dfile)
|
||||
trgt = os.path.join(dest_dir, dfile)
|
||||
shutil.copy(full, trgt)
|
||||
ret.append(trgt)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def build(runas, tgt, dest_dir, spec, sources, deps, template, saltenv='base'):
|
||||
'''
|
||||
Given the package destination directory, the tarball containing debian files (e.g. control)
|
||||
and package sources, use pbuilder to safely build the platform package
|
||||
|
||||
CLI Example:
|
||||
|
||||
Debian
|
||||
salt '*' pkgbuild.make_src_pkg deb-8-x86_64 /var/www/html/ https://raw.githubusercontent.com/saltstack/libnacl/master/pkg/deb/python-libnacl.control https://pypi.python.org/packages/source/l/libnacl/libnacl-1.3.5.tar.gz
|
||||
|
||||
This example command should build the libnacl package for Debian using pbuilder
|
||||
and place it in /var/www/html/ on the minion
|
||||
'''
|
||||
ret = {}
|
||||
if not os.path.isdir(dest_dir):
|
||||
try:
|
||||
os.makedirs(dest_dir)
|
||||
except (IOError, OSError):
|
||||
pass
|
||||
dsc_dir = tempfile.mkdtemp()
|
||||
dscs = make_src_pkg(dsc_dir, spec, sources, template, saltenv)
|
||||
|
||||
# dscs should only contain salt orig and debian tarballs and dsc file
|
||||
for dsc in dscs:
|
||||
afile = os.path.basename(dsc)
|
||||
adist = os.path.join(dest_dir, afile)
|
||||
shutil.copy(dsc, adist)
|
||||
|
||||
if dsc.endswith('.dsc'):
|
||||
dbase = os.path.dirname(dsc)
|
||||
cmd = 'chown {0} -R {1}'.format(runas, dbase)
|
||||
__salt__['cmd.run'](cmd)
|
||||
|
||||
results_dir = tempfile.mkdtemp()
|
||||
cmd = 'chown {0} -R {1}'.format(runas, results_dir)
|
||||
__salt__['cmd.run'](cmd)
|
||||
|
||||
cmd = 'pbuilder create'
|
||||
__salt__['cmd.run'](cmd, runas=runas, python_shell=True)
|
||||
cmd = 'pbuilder --build --buildresult {1} {0}'.format(dsc, results_dir)
|
||||
__salt__['cmd.run'](cmd, runas=runas, python_shell=True)
|
||||
|
||||
for bfile in os.listdir(results_dir):
|
||||
full = os.path.join(results_dir, bfile)
|
||||
if bfile.endswith('.deb'):
|
||||
bdist = os.path.join(dest_dir, bfile)
|
||||
shutil.copy(full, bdist)
|
||||
else:
|
||||
with salt.utils.fopen(full, 'r') as fp_:
|
||||
ret[bfile] = fp_.read()
|
||||
shutil.rmtree(results_dir)
|
||||
shutil.rmtree(dsc_dir)
|
||||
return ret
|
||||
|
||||
|
||||
def make_repo(repodir):
|
||||
'''
|
||||
Given the repodir, create a Debian repository out of the dsc therein
|
||||
|
||||
CLI Example::
|
||||
|
||||
salt '*' pkgbuild.make_repo /var/www/html/
|
||||
'''
|
||||
repocfg_text = '''Origin: SaltStack
|
||||
Label: salt_debian
|
||||
Suite: unstable
|
||||
Codename: jessie
|
||||
Architectures: i386 amd64 source
|
||||
Components: contrib
|
||||
Description: SaltStack debian package repo
|
||||
Pull: jessie
|
||||
'''
|
||||
repoconf = os.path.join(repodir, 'conf')
|
||||
if not os.path.isdir(repoconf):
|
||||
os.makedirs(repoconf)
|
||||
|
||||
repoconfdist = os.path.join(repoconf, 'distributions')
|
||||
with open(repoconfdist, "w") as fow:
|
||||
fow.write('{0}'.format(repocfg_text))
|
||||
|
||||
for debfile in os.listdir(repodir):
|
||||
if debfile.endswith('.changes'):
|
||||
cmd = 'reprepro -Vb . include jessie {0}'.format(os.path.join(repodir, debfile))
|
||||
__salt__['cmd.run'](cmd, cwd=repodir)
|
||||
|
||||
if debfile.endswith('.deb'):
|
||||
cmd = 'reprepro -Vb . includedeb jessie {0}'.format(os.path.join(repodir, debfile))
|
||||
__salt__['cmd.run'](cmd, cwd=repodir)
|
||||
@ -19,6 +19,7 @@ from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: dis
|
||||
|
||||
# Import salt libs
|
||||
import salt.utils
|
||||
from salt.exceptions import SaltInvocationError
|
||||
|
||||
__virtualname__ = 'pkgbuild'
|
||||
|
||||
@ -32,6 +33,26 @@ def __virtual__():
|
||||
return False
|
||||
|
||||
|
||||
def _create_rpmmacros():
|
||||
'''
|
||||
Create the .rpmmacros file in user's home directory
|
||||
'''
|
||||
home = os.path.expanduser('~')
|
||||
rpmbuilddir = os.path.join(home, 'rpmbuild')
|
||||
if not os.path.isdir(rpmbuilddir):
|
||||
os.makedirs(rpmbuilddir)
|
||||
|
||||
mockdir = os.path.join(home, 'mock')
|
||||
if not os.path.isdir(mockdir):
|
||||
os.makedirs(mockdir)
|
||||
|
||||
rpmmacros = os.path.join(home, '.rpmmacros')
|
||||
with open(rpmmacros, "w") as afile:
|
||||
afile.write('%_topdir {0}\n'.format(rpmbuilddir))
|
||||
afile.write('%signature gpg\n')
|
||||
afile.write('%_gpg_name packaging@saltstack.com\n')
|
||||
|
||||
|
||||
def _mk_tree():
|
||||
'''
|
||||
Create the rpm build tree
|
||||
@ -51,10 +72,9 @@ def _get_spec(tree_base, spec, template, saltenv='base'):
|
||||
spec_tgt = os.path.basename(spec)
|
||||
dest = os.path.join(tree_base, 'SPECS', spec_tgt)
|
||||
return __salt__['cp.get_url'](
|
||||
spec,
|
||||
dest,
|
||||
saltenv=saltenv,
|
||||
template=template)
|
||||
spec,
|
||||
dest,
|
||||
saltenv=saltenv)
|
||||
|
||||
|
||||
def _get_src(tree_base, source, saltenv='base'):
|
||||
@ -70,6 +90,46 @@ def _get_src(tree_base, source, saltenv='base'):
|
||||
shutil.copy(source, dest)
|
||||
|
||||
|
||||
def _get_distset(tgt):
|
||||
'''
|
||||
Get the distribution string for use with rpmbuild and mock
|
||||
'''
|
||||
# Centos adds that string to rpm names, removing that to have
|
||||
# consistent naming on Centos and Redhat
|
||||
tgtattrs = tgt.split('-')
|
||||
if tgtattrs[1] in ['5', '6', '7']:
|
||||
distset = '--define "dist .el{0}"'.format(tgtattrs[1])
|
||||
else:
|
||||
distset = ""
|
||||
|
||||
return distset
|
||||
|
||||
|
||||
def _get_deps(deps, tree_base, saltenv='base'):
|
||||
'''
|
||||
Get include string for list of dependent rpms to build package
|
||||
'''
|
||||
deps_list = ""
|
||||
if deps is None:
|
||||
return deps_list
|
||||
if not isinstance(deps, list):
|
||||
raise SaltInvocationError(
|
||||
'\'deps\' must be a Python list or comma-separated string'
|
||||
)
|
||||
for deprpm in deps:
|
||||
parsed = _urlparse(deprpm)
|
||||
depbase = os.path.basename(deprpm)
|
||||
dest = os.path.join(tree_base, depbase)
|
||||
if parsed.scheme:
|
||||
__salt__['cp.get_url'](deprpm, dest, saltenv=saltenv)
|
||||
else:
|
||||
shutil.copy(deprpm, dest)
|
||||
|
||||
deps_list += ' --install {0}'.format(dest)
|
||||
|
||||
return deps_list
|
||||
|
||||
|
||||
def make_src_pkg(dest_dir, spec, sources, template=None, saltenv='base'):
|
||||
'''
|
||||
Create a source rpm from the given spec file and sources
|
||||
@ -81,13 +141,16 @@ def make_src_pkg(dest_dir, spec, sources, template=None, saltenv='base'):
|
||||
This example command should build the libnacl SOURCE package and place it in
|
||||
/var/www/html/ on the minion
|
||||
'''
|
||||
_create_rpmmacros()
|
||||
tree_base = _mk_tree()
|
||||
spec_path = _get_spec(tree_base, spec, template, saltenv)
|
||||
if isinstance(sources, str):
|
||||
sources = sources.split(',')
|
||||
for src in sources:
|
||||
_get_src(tree_base, src, saltenv)
|
||||
cmd = 'rpmbuild --define "_topdir {0}" -bs {1}'.format(tree_base, spec_path)
|
||||
|
||||
# make source rpms for dist el5, usable with mock on other dists
|
||||
cmd = 'rpmbuild --define "_topdir {0}" -bs --define "_source_filedigest_algorithm md5" --define "_binary_filedigest_algorithm md5" --define "dist .el5" {1}'.format(tree_base, spec_path)
|
||||
__salt__['cmd.run'](cmd)
|
||||
srpms = os.path.join(tree_base, 'SRPMS')
|
||||
ret = []
|
||||
@ -101,14 +164,14 @@ def make_src_pkg(dest_dir, spec, sources, template=None, saltenv='base'):
|
||||
return ret
|
||||
|
||||
|
||||
def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'):
|
||||
def build(runas, tgt, dest_dir, spec, sources, deps, template, saltenv='base'):
|
||||
'''
|
||||
Given the package destination directory, the spec file source and package
|
||||
sources, use mock to safely build the rpm defined in the spec file
|
||||
|
||||
CLI Example:
|
||||
|
||||
salt '*' pkgbuild.make_src_pkg mock epel-7-x86_64 /var/www/html/ https://raw.githubusercontent.com/saltstack/libnacl/master/pkg/rpm/python-libnacl.spec https://pypi.python.org/packages/source/l/libnacl/libnacl-1.3.5.tar.gz
|
||||
salt '*' pkgbuild.build mock epel-7-x86_64 /var/www/html/ https://raw.githubusercontent.com/saltstack/libnacl/master/pkg/rpm/python-libnacl.spec https://pypi.python.org/packages/source/l/libnacl/libnacl-1.3.5.tar.gz
|
||||
|
||||
This example command should build the libnacl package for rhel 7 using user
|
||||
"mock" and place it in /var/www/html/ on the minion
|
||||
@ -121,6 +184,17 @@ def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'):
|
||||
pass
|
||||
srpm_dir = tempfile.mkdtemp()
|
||||
srpms = make_src_pkg(srpm_dir, spec, sources, template, saltenv)
|
||||
|
||||
distset = _get_distset(tgt)
|
||||
|
||||
noclean = ""
|
||||
deps_dir = tempfile.mkdtemp()
|
||||
deps_list = _get_deps(deps, deps_dir, saltenv)
|
||||
if deps_list and not deps_list.isspace():
|
||||
cmd = 'mock --root={0} {1}'.format(tgt, deps_list)
|
||||
__salt__['cmd.run'](cmd, runas=runas)
|
||||
noclean += " --no-clean"
|
||||
|
||||
for srpm in srpms:
|
||||
dbase = os.path.dirname(srpm)
|
||||
cmd = 'chown {0} -R {1}'.format(runas, dbase)
|
||||
@ -128,10 +202,12 @@ def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'):
|
||||
results_dir = tempfile.mkdtemp()
|
||||
cmd = 'chown {0} -R {1}'.format(runas, results_dir)
|
||||
__salt__['cmd.run'](cmd)
|
||||
cmd = 'mock -r {0} --rebuild {1} --resultdir {2}'.format(
|
||||
tgt,
|
||||
srpm,
|
||||
results_dir)
|
||||
cmd = 'mock --root={0} --resultdir={1} {2} {3} {4}'.format(
|
||||
tgt,
|
||||
results_dir,
|
||||
distset,
|
||||
noclean,
|
||||
srpm)
|
||||
__salt__['cmd.run'](cmd, runas=runas)
|
||||
for rpm in os.listdir(results_dir):
|
||||
full = os.path.join(results_dir, rpm)
|
||||
@ -150,6 +226,7 @@ def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'):
|
||||
with salt.utils.fopen(full, 'r') as fp_:
|
||||
ret[rpm] = fp_.read()
|
||||
shutil.rmtree(results_dir)
|
||||
shutil.rmtree(deps_dir)
|
||||
shutil.rmtree(srpm_dir)
|
||||
return ret
|
||||
|
||||
|
||||
@ -53,6 +53,7 @@ def built(
|
||||
sources,
|
||||
template,
|
||||
tgt,
|
||||
deps=None,
|
||||
results=None,
|
||||
always=False,
|
||||
saltenv='base'):
|
||||
@ -80,6 +81,13 @@ def built(
|
||||
tgt
|
||||
The target platform to run the build on
|
||||
|
||||
deps
|
||||
Packages required to ensure that the named package is built
|
||||
can be hosted on either the salt master server or on an HTTP
|
||||
or FTP server. Both HTTPS and HTTP are supported as well as
|
||||
downloading directly from Amazon S3 compatible URLs with both
|
||||
pre-configured and automatic IAM credentials
|
||||
|
||||
results
|
||||
The names of the expected rpms that will be built
|
||||
|
||||
@ -101,7 +109,7 @@ def built(
|
||||
present = set()
|
||||
if os.path.isdir(dest_dir):
|
||||
for fn_ in os.listdir(dest_dir):
|
||||
present.appd(fn_)
|
||||
present.add(fn_)
|
||||
need = results.difference(present)
|
||||
if not need:
|
||||
ret['comment'] = 'All needed packages exist'
|
||||
@ -111,13 +119,14 @@ def built(
|
||||
ret['result'] = None
|
||||
return ret
|
||||
ret['changes'] = __salt__['pkgbuild.build'](
|
||||
runas,
|
||||
tgt,
|
||||
dest_dir,
|
||||
spec,
|
||||
sources,
|
||||
template,
|
||||
saltenv)
|
||||
runas,
|
||||
tgt,
|
||||
dest_dir,
|
||||
spec,
|
||||
sources,
|
||||
deps,
|
||||
template,
|
||||
saltenv)
|
||||
ret['comment'] = 'Packages Built'
|
||||
return ret
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user