valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-07 08:58:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #2 from saltstack/2015.8

Browse Source 
Sync 2015.8 with upstream
This commit is contained in:
Jorge Schrauwen 2015-08-01 01:37:14 +02:00
commit 59fcdf985f
7 changed files with 429 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
doc/topics/windows/windows-package-manager.rst
View File

@ -161,10 +161,10 @@ project's wiki_:
installer: salt://win/repo/7zip/7z920-x64.msi
full_name: 7-Zip 9.20 (x64 edition)
reboot: False
install_flags: ' /q '
install_flags: '/qn /norestart'
msiexec: True
uninstaller: salt://win/repo/7zip/7z920-x64.msi
uninstall_flags: ' /qn'
uninstaller: '{23170F69-40C1-2702-0920-000001000000}'
uninstall_flags: '/qn /norestart'
Add ``cache_dir: True`` when the installer requires multiple source files. The
directory containing the installer file will be recursively cached on the minion.
@ -177,7 +177,7 @@ Only applies to salt: installer URLs.
installer: 'salt://win/repo/sqlexpress/setup.exe'
full_name: Microsoft SQL Server 2014 Setup (English)
reboot: False
install_flags: ' /ACTION=install /IACCEPTSQLSERVERLICENSETERMS /Q'
install_flags: '/ACTION=install /IACCEPTSQLSERVERLICENSETERMS /Q'
cache_dir: True
Generate Repo Cache File

2
pkg/smartos/esky/build-tarball.sh
View File

@ -17,7 +17,7 @@ python2.7 setup.py bdist
python2.7 setup.py bdist_esky
rm salt/_syspaths.py
rm -f dist/*.tar.gz
mkdir -p $BUILD_DIR/{install,bin/appdata}
mkdir -p $BUILD_DIR/{bin/appdata,install,var/log/salt}
cp -r conf $BUILD_DIR/etc
cp $PKG_DIR/*.xml $PKG_DIR/install.sh $BUILD_DIR/install
chmod +x $BUILD_DIR/install/install.sh

5
salt/minion.py
View File

@ -484,7 +484,7 @@ class MultiMinion(MinionBase):
def __init__(self, opts):
super(MultiMinion, self).__init__(opts)
self.auth_wait = self.opts['acceptance_wait_time']
self.max_wait = self.opts['acceptance_wait_time_max']
self.max_auth_wait = self.opts['acceptance_wait_time_max']
self.io_loop = zmq.eventloop.ioloop.ZMQIOLoop()
@ -1985,6 +1985,9 @@ class MultiSyndic(MinionBase):
# sync (old behavior), cluster (only returns and publishes)
self.syndic_mode = self.opts.get('syndic_mode', 'sync')
self.auth_wait = self.opts['acceptance_wait_time']
self.max_auth_wait = self.opts['acceptance_wait_time_max']
self._has_master = threading.Event()
self.jid_forward_cache = set()

311
salt/modules/debbuild.py Normal file
View File

@ -0,0 +1,311 @@
# -*- coding: utf-8 -*-
'''
Debian Package builder system
.. versionadded:: Beryllium
This system allows for all of the components to build debs safely in chrooted
environments. This also provides a function to generate debian repositories
This module impliments the pkgbuild interface
'''
# import python libs
from __future__ import absolute_import, print_function
import os
import tempfile
import shutil
from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: disable=no-name-in-module,import-error
# Import salt libs
import salt.utils
# pylint: disable=import-error
__virtualname__ = 'pkgbuild'
def __virtual__():
'''
Confirm this module is on a Debian based system
'''
if __grains__.get('os_family', False) in ('Kali', 'Debian'):
return __virtualname__
return False
def _create_pbuilders():
'''
Create the .pbuilder family of files in user's home directory
'''
hook_text = '''#!/bin/sh
set -e
cat > "/etc/apt/preferences" << EOF
Package: python-abalaster
Pin: release a=testing
Pin-Priority: 950
Package: python-sphinx
Pin: release a=experimental
Pin-Priority: 900
Package: sphinx-common
Pin: release a=experimental
Pin-Priority: 900
Package: *
Pin: release a=jessie-backports
Pin-Priority: 750
Package: *
Pin: release a=stable
Pin-Priority: 700
Package: *
Pin: release a=testing
Pin-Priority: 650
Package: *
Pin: release a=unstable
Pin-Priority: 600
Package: *
Pin: release a=experimental
Pin-Priority: 550
EOF
'''
pbldrc_text = '''DIST="jessie"
if [ -n "${DIST}" ]; then
TMPDIR=/tmp
BASETGZ="`dirname $BASETGZ`/$DIST-base.tgz"
DISTRIBUTION=$DIST
APTCACHE="/var/cache/pbuilder/$DIST/aptcache"
fi
HOOKDIR="${HOME}/.pbuilder-hooks"
OTHERMIRROR="deb http://ftp.us.debian.org/debian/ testing main contrib non-free | deb http://ftp.us.debian.org/debian/ experimental main contrib non-free"
'''
home = os.path.expanduser('~')
pbuilder_hooksdir = os.path.join(home, '.pbuilder-hooks')
if not os.path.isdir(pbuilder_hooksdir):
os.makedirs(pbuilder_hooksdir)
d05hook = os.path.join(pbuilder_hooksdir, 'D05apt-preferences')
with open(d05hook, "w") as fow:
fow.write('{0}'.format(hook_text))
pbuilderrc = os.path.join(home, '.pbuilderrc')
with open(pbuilderrc, "w") as fow:
fow.write('{0}'.format(pbldrc_text))
def _mk_tree():
'''
Create the debian build area
'''
basedir = tempfile.mkdtemp()
return basedir
def _get_spec(tree_base, spec, template, saltenv='base'):
'''
Get the spec file (tarball of the debian sub-dir to use)
and place it in build area
'''
spec_tgt = os.path.basename(spec)
dest = os.path.join(tree_base, spec_tgt)
return __salt__['cp.get_url'](spec, dest, saltenv=saltenv)
def _get_src(tree_base, source, saltenv='base'):
'''
Get the named sources and place them into the tree_base
'''
parsed = _urlparse(source)
sbase = os.path.basename(source)
dest = os.path.join(tree_base, sbase)
if parsed.scheme:
__salt__['cp.get_url'](source, dest, saltenv=saltenv)
else:
shutil.copy(source, dest)
def make_src_pkg(dest_dir, spec, sources, template=None, saltenv='base'):
'''
Create a platform specific source package from the given platform spec/control file and sources
CLI Example:
Debian
salt '*' pkgbuild.make_src_pkg /var/www/html/ https://raw.githubusercontent.com/saltstack/libnacl/master/pkg/deb/python-libnacl.control.tar.xz https://pypi.python.org/packages/source/l/libnacl/libnacl-1.3.5.tar.gz
This example command should build the libnacl SOURCE package and place it in
/var/www/html/ on the minion
'''
_create_pbuilders()
tree_base = _mk_tree()
ret = []
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
spec_pathfile = _get_spec(tree_base, spec, template, saltenv)
# build salt equivalents from scratch
if isinstance(sources, str):
sources = sources.split(',')
for src in sources:
_get_src(tree_base, src, saltenv)
#.dsc then assumes sources already build
if spec_pathfile.endswith('.dsc'):
for efile in os.listdir(tree_base):
full = os.path.join(tree_base, efile)
trgt = os.path.join(dest_dir, efile)
shutil.copy(full, trgt)
ret.append(trgt)
trgt = os.path.join(dest_dir, os.path.basename(spec_pathfile))
shutil.copy(spec_pathfile, trgt)
ret.append(trgt)
return ret
# obtain name of 'python setup.py sdist' generated tarball, extract the version
# and manipulate the name for debian use (convert minix and add '+ds')
salttarball = None
for afile in os.listdir(tree_base):
if afile.startswith('salt-') and afile.endswith('.tar.gz'):
salttarball = afile
break
else:
return ret
frontname = salttarball.split('.tar.gz')
salttar_name = frontname[0]
debname = salttar_name.replace('-', '_')
debname += '+ds'
debname_orig = debname + '.orig.tar.gz'
abspath_debname = os.path.join(tree_base, debname)
cmd = 'tar -xvzf {0}'.format(salttarball)
__salt__['cmd.run'](cmd, cwd=tree_base)
cmd = 'mv {0} {1}'.format(salttar_name, debname)
__salt__['cmd.run'](cmd, cwd=tree_base)
cmd = 'tar -cvzf {0} {1}'.format(os.path.join(tree_base, debname_orig), debname)
__salt__['cmd.run'](cmd, cwd=tree_base)
cmd = 'rm -f {0}'.format(salttarball)
__salt__['cmd.run'](cmd, cwd=tree_base)
cmd = 'cp {0} {1}'.format(spec_pathfile, abspath_debname)
__salt__['cmd.run'](cmd, cwd=abspath_debname)
cmd = 'tar -xvJf {0}'.format(spec_pathfile)
__salt__['cmd.run'](cmd, cwd=abspath_debname)
cmd = 'rm -f {0}'.format(os.path.basename(spec_pathfile))
__salt__['cmd.run'](cmd, cwd=abspath_debname)
cmd = 'debuild -S -uc -us'
__salt__['cmd.run'](cmd, cwd=abspath_debname, python_shell=True)
cmd = 'rm -fR {0}'.format(abspath_debname)
__salt__['cmd.run'](cmd)
for dfile in os.listdir(tree_base):
if dfile.startswith('salt_'):
if not dfile.endswith('.build'):
full = os.path.join(tree_base, dfile)
trgt = os.path.join(dest_dir, dfile)
shutil.copy(full, trgt)
ret.append(trgt)
return ret
def build(runas, tgt, dest_dir, spec, sources, deps, template, saltenv='base'):
'''
Given the package destination directory, the tarball containing debian files (e.g. control)
and package sources, use pbuilder to safely build the platform package
CLI Example:
Debian
salt '*' pkgbuild.make_src_pkg deb-8-x86_64 /var/www/html/ https://raw.githubusercontent.com/saltstack/libnacl/master/pkg/deb/python-libnacl.control https://pypi.python.org/packages/source/l/libnacl/libnacl-1.3.5.tar.gz
This example command should build the libnacl package for Debian using pbuilder
and place it in /var/www/html/ on the minion
'''
ret = {}
if not os.path.isdir(dest_dir):
try:
os.makedirs(dest_dir)
except (IOError, OSError):
pass
dsc_dir = tempfile.mkdtemp()
dscs = make_src_pkg(dsc_dir, spec, sources, template, saltenv)
# dscs should only contain salt orig and debian tarballs and dsc file
for dsc in dscs:
afile = os.path.basename(dsc)
adist = os.path.join(dest_dir, afile)
shutil.copy(dsc, adist)
if dsc.endswith('.dsc'):
dbase = os.path.dirname(dsc)
cmd = 'chown {0} -R {1}'.format(runas, dbase)
__salt__['cmd.run'](cmd)
results_dir = tempfile.mkdtemp()
cmd = 'chown {0} -R {1}'.format(runas, results_dir)
__salt__['cmd.run'](cmd)
cmd = 'pbuilder create'
__salt__['cmd.run'](cmd, runas=runas, python_shell=True)
cmd = 'pbuilder --build --buildresult {1} {0}'.format(dsc, results_dir)
__salt__['cmd.run'](cmd, runas=runas, python_shell=True)
for bfile in os.listdir(results_dir):
full = os.path.join(results_dir, bfile)
if bfile.endswith('.deb'):
bdist = os.path.join(dest_dir, bfile)
shutil.copy(full, bdist)
else:
with salt.utils.fopen(full, 'r') as fp_:
ret[bfile] = fp_.read()
shutil.rmtree(results_dir)
shutil.rmtree(dsc_dir)
return ret
def make_repo(repodir):
'''
Given the repodir, create a Debian repository out of the dsc therein
CLI Example::
salt '*' pkgbuild.make_repo /var/www/html/
'''
repocfg_text = '''Origin: SaltStack
Label: salt_debian
Suite: unstable
Codename: jessie
Architectures: i386 amd64 source
Components: contrib
Description: SaltStack debian package repo
Pull: jessie
'''
repoconf = os.path.join(repodir, 'conf')
if not os.path.isdir(repoconf):
os.makedirs(repoconf)
repoconfdist = os.path.join(repoconf, 'distributions')
with open(repoconfdist, "w") as fow:
fow.write('{0}'.format(repocfg_text))
for debfile in os.listdir(repodir):
if debfile.endswith('.changes'):
cmd = 'reprepro -Vb . include jessie {0}'.format(os.path.join(repodir, debfile))
__salt__['cmd.run'](cmd, cwd=repodir)
if debfile.endswith('.deb'):
cmd = 'reprepro -Vb . includedeb jessie {0}'.format(os.path.join(repodir, debfile))
__salt__['cmd.run'](cmd, cwd=repodir)

99
salt/modules/rpmbuild.py
View File

@ -19,6 +19,7 @@ from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: dis
# Import salt libs
import salt.utils
from salt.exceptions import SaltInvocationError
__virtualname__ = 'pkgbuild'
@ -32,6 +33,26 @@ def __virtual__():
return False
def _create_rpmmacros():
'''
Create the .rpmmacros file in user's home directory
'''
home = os.path.expanduser('~')
rpmbuilddir = os.path.join(home, 'rpmbuild')
if not os.path.isdir(rpmbuilddir):
os.makedirs(rpmbuilddir)
mockdir = os.path.join(home, 'mock')
if not os.path.isdir(mockdir):
os.makedirs(mockdir)
rpmmacros = os.path.join(home, '.rpmmacros')
with open(rpmmacros, "w") as afile:
afile.write('%_topdir {0}\n'.format(rpmbuilddir))
afile.write('%signature gpg\n')
afile.write('%_gpg_name packaging@saltstack.com\n')
def _mk_tree():
'''
Create the rpm build tree
@ -51,10 +72,9 @@ def _get_spec(tree_base, spec, template, saltenv='base'):
spec_tgt = os.path.basename(spec)
dest = os.path.join(tree_base, 'SPECS', spec_tgt)
return __salt__['cp.get_url'](
spec,
dest,
saltenv=saltenv,
template=template)
spec,
dest,
saltenv=saltenv)
def _get_src(tree_base, source, saltenv='base'):
@ -70,6 +90,46 @@ def _get_src(tree_base, source, saltenv='base'):
shutil.copy(source, dest)
def _get_distset(tgt):
'''
Get the distribution string for use with rpmbuild and mock
'''
# Centos adds that string to rpm names, removing that to have
# consistent naming on Centos and Redhat
tgtattrs = tgt.split('-')
if tgtattrs[1] in ['5', '6', '7']:
distset = '--define "dist .el{0}"'.format(tgtattrs[1])
else:
distset = ""
return distset
def _get_deps(deps, tree_base, saltenv='base'):
'''
Get include string for list of dependent rpms to build package
'''
deps_list = ""
if deps is None:
return deps_list
if not isinstance(deps, list):
raise SaltInvocationError(
'\'deps\' must be a Python list or comma-separated string'
)
for deprpm in deps:
parsed = _urlparse(deprpm)
depbase = os.path.basename(deprpm)
dest = os.path.join(tree_base, depbase)
if parsed.scheme:
__salt__['cp.get_url'](deprpm, dest, saltenv=saltenv)
else:
shutil.copy(deprpm, dest)
deps_list += ' --install {0}'.format(dest)
return deps_list
def make_src_pkg(dest_dir, spec, sources, template=None, saltenv='base'):
'''
Create a source rpm from the given spec file and sources
@ -81,13 +141,16 @@ def make_src_pkg(dest_dir, spec, sources, template=None, saltenv='base'):
This example command should build the libnacl SOURCE package and place it in
/var/www/html/ on the minion
'''
_create_rpmmacros()
tree_base = _mk_tree()
spec_path = _get_spec(tree_base, spec, template, saltenv)
if isinstance(sources, str):
sources = sources.split(',')
for src in sources:
_get_src(tree_base, src, saltenv)
cmd = 'rpmbuild --define "_topdir {0}" -bs {1}'.format(tree_base, spec_path)
# make source rpms for dist el5, usable with mock on other dists
cmd = 'rpmbuild --define "_topdir {0}" -bs --define "_source_filedigest_algorithm md5" --define "_binary_filedigest_algorithm md5" --define "dist .el5" {1}'.format(tree_base, spec_path)
__salt__['cmd.run'](cmd)
srpms = os.path.join(tree_base, 'SRPMS')
ret = []
@ -101,14 +164,14 @@ def make_src_pkg(dest_dir, spec, sources, template=None, saltenv='base'):
return ret
def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'):
def build(runas, tgt, dest_dir, spec, sources, deps, template, saltenv='base'):
'''
Given the package destination directory, the spec file source and package
sources, use mock to safely build the rpm defined in the spec file
CLI Example:
salt '*' pkgbuild.make_src_pkg mock epel-7-x86_64 /var/www/html/ https://raw.githubusercontent.com/saltstack/libnacl/master/pkg/rpm/python-libnacl.spec https://pypi.python.org/packages/source/l/libnacl/libnacl-1.3.5.tar.gz
salt '*' pkgbuild.build mock epel-7-x86_64 /var/www/html/ https://raw.githubusercontent.com/saltstack/libnacl/master/pkg/rpm/python-libnacl.spec https://pypi.python.org/packages/source/l/libnacl/libnacl-1.3.5.tar.gz
This example command should build the libnacl package for rhel 7 using user
"mock" and place it in /var/www/html/ on the minion
@ -121,6 +184,17 @@ def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'):
pass
srpm_dir = tempfile.mkdtemp()
srpms = make_src_pkg(srpm_dir, spec, sources, template, saltenv)
distset = _get_distset(tgt)
noclean = ""
deps_dir = tempfile.mkdtemp()
deps_list = _get_deps(deps, deps_dir, saltenv)
if deps_list and not deps_list.isspace():
cmd = 'mock --root={0} {1}'.format(tgt, deps_list)
__salt__['cmd.run'](cmd, runas=runas)
noclean += " --no-clean"
for srpm in srpms:
dbase = os.path.dirname(srpm)
cmd = 'chown {0} -R {1}'.format(runas, dbase)
@ -128,10 +202,12 @@ def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'):
results_dir = tempfile.mkdtemp()
cmd = 'chown {0} -R {1}'.format(runas, results_dir)
__salt__['cmd.run'](cmd)
cmd = 'mock -r {0} --rebuild {1} --resultdir {2}'.format(
tgt,
srpm,
results_dir)
cmd = 'mock --root={0} --resultdir={1} {2} {3} {4}'.format(
tgt,
results_dir,
distset,
noclean,
srpm)
__salt__['cmd.run'](cmd, runas=runas)
for rpm in os.listdir(results_dir):
full = os.path.join(results_dir, rpm)
@ -150,6 +226,7 @@ def build(runas, tgt, dest_dir, spec, sources, template, saltenv='base'):
with salt.utils.fopen(full, 'r') as fp_:
ret[rpm] = fp_.read()
shutil.rmtree(results_dir)
shutil.rmtree(deps_dir)
shutil.rmtree(srpm_dir)
return ret

5
salt/state.py
View File

@ -632,7 +632,10 @@ class State(object):
mod_init function in the state module.
'''
# ensure that the module is loaded
self.states['{0}.{1}'.format(low['state'], low['fun'])] # pylint: disable=W0106
try:
self.states['{0}.{1}'.format(low['state'], low['fun'])] # pylint: disable=W0106
except KeyError:
return
minit = '{0}.mod_init'.format(low['state'])
if low['state'] not in self.mod_init:
if minit in self.states._dict:

25
salt/states/pkgbuild.py
View File

@ -53,6 +53,7 @@ def built(
sources,
template,
tgt,
deps=None,
results=None,
always=False,
saltenv='base'):
@ -80,6 +81,13 @@ def built(
tgt
The target platform to run the build on
deps
Packages required to ensure that the named package is built
can be hosted on either the salt master server or on an HTTP
or FTP server. Both HTTPS and HTTP are supported as well as
downloading directly from Amazon S3 compatible URLs with both
pre-configured and automatic IAM credentials
results
The names of the expected rpms that will be built
@ -101,7 +109,7 @@ def built(
present = set()
if os.path.isdir(dest_dir):
for fn_ in os.listdir(dest_dir):
present.appd(fn_)
present.add(fn_)
need = results.difference(present)
if not need:
ret['comment'] = 'All needed packages exist'
@ -111,13 +119,14 @@ def built(
ret['result'] = None
return ret
ret['changes'] = __salt__['pkgbuild.build'](
runas,
tgt,
dest_dir,
spec,
sources,
template,
saltenv)
runas,
tgt,
dest_dir,
spec,
sources,
deps,
template,
saltenv)
ret['comment'] = 'Packages Built'
return ret