Merge branch 'develop' into zfs

pkg/arch/git/PKGBUILD

@@ -0,0 +1,88 @@
+# Maintainer: Christer Edwards <christer.edwards@gmail.com>
+pkgname=salt-git
+_gitname="salt"
+pkgver=v0.15.0.786.g0a800ec
+pkgrel=1
+pkgdesc="A remote execution and communication system built on zeromq"
+arch=('any')
+url="https://github.com/saltstack/salt"
+license=('APACHE')
+groups=()
+depends=('python2'
+         'python2-yaml'
+         'python2-jinja'
+         'python2-pyzmq'
+         'python2-crypto'
+         'python2-psutil'
+         'python2-msgpack'
+         'python2-m2crypto'
+         'logrotate'
+         'bash-completion')
+
+backup=('etc/salt/master'
+        'etc/salt/minion')
+
+makedepends=('git')
+conflicts=('salt')
+provides=('salt' 'bash-completion-salt')
+install="salt.install"
+
+# makepkg 4.1 knows about git and will pull main branch
+source=("git://github.com/saltstack/salt.git")
+
+# makepkg knows it's a git repo because the url starts with 'git'
+# it then knows to checkout the branch upon cloning, expediating versioning.
+#branch="develop"
+#source=("git://github.com/saltstack/salt.git#branch=$branch")
+
+# makepkg also knows about tags
+#tags="v0.14.1"
+#source=("git://github.com/saltstack/salt.git#tag=$tag")
+
+# because the sources are not static, skip checksums
+md5sums=('SKIP')
+
+pkgver() {
+  cd "$srcdir/$_gitname"
+  echo $(git describe --always | sed 's/-/./g')
+  # for git, if the repo has no tags, comment out the above and uncomment the next line:
+  #echo "0.$(git rev-list --count $branch).$(git describe --always)"
+  # This will give you a count of the total commits and the hash of the commit you are on.
+  # Useful if you're making a repository with git packages so that they can have sequential
+  # version numbers. (Else a pacman -Syu may not update the package)
+}
+
+#build() {
+#  cd "${srcdir}/${_gitname}"
+#  python2 setup.py build 
+   # no need to build setup.py install will do this
+#}
+
+package() {
+  cd "${srcdir}/${_gitname}"
+
+  ## build salt
+  python2 setup.py install --root=${pkgdir}/ --optimize=1
+
+  ## install salt systemd service files
+  install -Dm644 ${srcdir}/salt/pkg/arch/salt-master.service ${pkgdir}/usr/lib/systemd/system/salt-master.service
+  install -Dm644 ${srcdir}/salt/pkg/arch/salt-syndic.service ${pkgdir}/usr/lib/systemd/system/salt-syndic.service
+  install -Dm644 ${srcdir}/salt/pkg/arch/salt-minion.service ${pkgdir}/usr/lib/systemd/system/salt-minion.service
+
+  ## install salt config files
+  mkdir -p ${pkgdir}/etc/salt/master.d
+  mkdir -p ${pkgdir}/etc/salt/minion.d
+  cp ${srcdir}/salt/conf/master ${pkgdir}/etc/salt/
+  cp ${srcdir}/salt/conf/minion ${pkgdir}/etc/salt/
+
+  ## install logrotate:
+  mkdir -p ${pkgdir}/etc/logrotate.d/
+  install -Dm644 ${srcdir}/salt/pkg/salt-common.logrotate ${pkgdir}/etc/logrotate.d/salt
+
+  ## install bash-completion
+  mkdir -p ${pkgdir}usr/share/bash-completion/completion/
+  install -Dm644 ${srcdir}/salt/pkg/salt.bash ${pkgdir}usr/share/bash-completion/completion/salt
+
+  # remove vcs leftovers
+  find "$pkgdir" -type d -name .git -exec rm -r '{}' +
+}

pkg/arch/git/salt.install

@@ -0,0 +1,104 @@
+# Salt: Installer: Arch
+# Maintainer: Niels Abspoel
+
+pre_install(){
+  # create salt user
+  getent passwd salt &>/dev/null || \
+    echo "salt master user doesn't exist, creating..."; \
+    useradd -r -d /srv/salt -s /sbin/nologin -c "Salt" salt &>/dev/null || :
+}
+
+pre_upgrade () {
+  pre_install
+  salthomedir=`getent passwd salt | cut -d: -f6`
+  saltdir=/srv/salt/
+  if [[ $salthomedir != $saltdir ]]; then
+    echo "setting salt master user homedir to /srv/salt/"
+    usermod -d /srv/salt/ salt &>/dev/null || :
+  fi
+}
+
+post_install() {
+  # set user permissions on directories needed for salt
+  getent passwd salt &>/dev/null && chown -R salt /var/cache/salt
+  getent passwd salt &>/dev/null && chown -R salt /var/log/salt
+  getent passwd salt &>/dev/null && chown -R salt /etc/salt/pki
+  getent passwd salt &>/dev/null && chown -R salt /srv/salt
+
+  # set salt master user in config
+  # and verify environment
+  if [[ ! -f /etc/salt/master.d/salt-user.conf ]]; then
+    if [[ ! -d /etc/salt/master.d ]]; then
+      mkdir -p /etc/salt/master.d
+    fi
+    echo "configure salt-master to run as salt master user"
+    cat << EOF1 > /etc/salt/master.d/salt-user.conf
+user: salt
+verify_env: True
+EOF1
+  fi
+
+  # set salt user limits
+  if [[ ! -f /etc/security/limits.d/20-salt.conf ]]; then
+    echo "raising file limits for salt master user"
+    cat << EOF2 > /etc/security/limits.d/20-salt.conf
+salt  soft  nofile  100000
+salt  hard  nofile  100000
+EOF2
+  fi
+}
+
+post_upgrade () {
+  # if salt-master/salt-minion daemon is running reinitialise
+  if [[ -f /var/run/salt-master.pid ]]; then
+    if [ "`systemctl is-active salt-master`" == "active" ]; then
+      echo "salt-master is running system daemons are reloaded"
+      getent passwd salt &>/dev/null && systemctl daemon-reexec
+      getent passwd salt &>/dev/null && systemctl daemon-reload
+    fi
+  fi
+  if [[ -f /var/run/salt-minion.pid ]]; then
+    if [ "`systemctl is-active salt-minion`" == "active" ]; then
+      echo "salt-minion was running system daemons are reloaded"
+      getent passwd salt &>/dev/null && systemctl daemon-reexec
+      getent passwd salt &>/dev/null && systemctl daemon-reload
+    fi
+  fi
+}
+
+pre_remove (){
+  # Stop salt-master daemon and remove it
+  if [[ -f /var/run/salt-master.pid ]]; then
+    if [ "`systemctl is-active salt-master`" == "active" ]; then
+      echo "stopping salt-master and removing it"
+      systemctl stop salt-master
+      systemctl disable salt-master
+    fi
+  fi
+
+  # Stop salt-minion daemon and remove it
+  if [[ -f /var/run/salt-minion.pid ]]; then
+    if [ "`systemctl is-active salt-minion`" == "active" ]; then
+      echo "stopping salt-minion and removing it"
+      systemctl stop salt-minion
+      systemctl disable salt-minion
+    fi
+  fi
+}
+
+post_remove (){
+  # remove shared job cache and other runtime directories
+  rm -rf \
+    /var/cache/salt \
+    /var/log/salt \
+    2> /dev/null
+  echo "shared job cache and runtime directories removed"
+  # remove salt user and group but leave /srv/salt
+  getent passwd salt &>/dev/null && userdel salt && echo "salt master user removed"
+  echo "salt has been removed but /srv/salt is still available"
+}
+
+op=$1
+shift
+
+$op "$@"

salt/modules/gem.py

@@ -5,6 +5,9 @@ Manage ruby gems.
 # Import python libs
 import re
 
+__func_alias__ = {
+    'list_': 'list'
+}
 
 def _gem(command, ruby=None, runas=None):
     cmdline = 'gem {command}'.format(command=command)
@@ -117,7 +120,7 @@ def update_system(version='', ruby=None, runas=None):
                 format(version=version), ruby, runas=runas)
 
 
-def list(prefix='', ruby=None, runas=None):
+def list_(prefix='', ruby=None, runas=None):
     '''
     List locally installed gems.
 

salt/modules/match.py

@@ -5,6 +5,9 @@ specs.
 
 import salt.minion
 
+__func_alias__ = {
+    'list_': 'list'
+}
 
 def compound(tgt):
     '''
@@ -96,7 +99,7 @@ def grain(tgt):
         return False
 
 
-def list(tgt):
+def list_(tgt):
     '''
     Return True if the minion matches the given list target
 

salt/modules/nova.py

@@ -23,6 +23,10 @@ except ImportError:
 # Import salt libs
 import salt.utils
 
+# Function alias to not shadow built-in's
+__func_alias__ = {
+    'list_': 'list'
+}
 
 def __virtual__():
     '''
@@ -248,7 +252,7 @@ def image_meta_delete(id=None,     # pylint: disable-msg=C0103
     return {id: 'Deleted: {0}'.format(pairs)}
 
 
-def list():
+def list_():
     '''
     To maintain the feel of the nova command line, this function simply calls
     the server_list function.

salt/modules/npm.py

@@ -14,6 +14,10 @@ from salt.exceptions import CommandExecutionError
 
 log = logging.getLogger(__name__)
 
+# Function alias to make sure not to shadow built-in's
+__func_alias__ = {
+    'list_': 'list'
+}
 
 def __virtual__():
     '''
@@ -140,7 +144,7 @@ def uninstall(pkg,
     return True
 
 
-def list(pkg=None,
+def list_(pkg=None,
          dir=None):
     '''
     List installed NPM packages.

salt/modules/nzbget.py

@@ -5,6 +5,9 @@ Support for nzbget
 # Import salt libs
 import salt.utils
 
+__func_alias__ = {
+    'list_': 'list'
+}
 
 def __virtual__():
     '''
@@ -82,7 +85,7 @@ def stop(user=None):
     return out
 
 
-def list(user=None):
+def list_(user=None):
     '''
     Return list of active downloads using nzbget -L.
     Default user is root.

salt/modules/pecl.py

@@ -9,6 +9,9 @@ import re
 __opts__ = {}
 __pillar__ = {}
 
+__func_alias__ = {
+    'list_': 'list'
+}
 
 def _pecl(command):
     '''
@@ -66,7 +69,7 @@ def update(pecls):
     return _pecl('install -U {0}'.format(pecls))
 
 
-def list():
+def list_():
     '''
     List installed pecl extensions.
 

salt/modules/pip.py

@@ -18,6 +18,10 @@ from salt.exceptions import CommandExecutionError, CommandNotFoundError
 
 logger = logging.getLogger(__name__)  # pylint: disable-msg=C0103
 
+# Don't shadow built-in's. 
+__func_alias__ = {
+    'list_': 'list'
+}
 
 VALID_PROTOS = ['http', 'https', 'ftp']
 
@@ -526,7 +530,7 @@ def freeze(bin_env=None,
     return result['stdout'].splitlines()
 
 
-def list(prefix='',
+def list_(prefix='',
          bin_env=None,
          runas=None,
          cwd=None):

salt/modules/pkgin.py

@@ -27,7 +27,7 @@ def __virtual__():
 
     if __grains__['os'] in supported and _check_pkgin():
         return 'pkg'
-    else:
+
     return False
 
 

salt/modules/rvm.py

@@ -6,6 +6,11 @@ Manage ruby installations and gemsets with RVM, the Ruby Version Manager.
 import re
 import os
 
+# Don't shadow built-in's.
+__func_alias__ = {
+    'list_': 'list'
+}
+
 __opts__ = {
     'rvm.runas': None,
 }
@@ -102,7 +107,7 @@ def reinstall_ruby(ruby, runas=None):
     return _rvm('reinstall', ruby, runas=runas)
 
 
-def list(runas=None):
+def list_(runas=None):
     '''
     List all rvm installed rubies.
 

salt/modules/smartos_vmadm.py

@@ -4,6 +4,7 @@ Module for managing VMs on SmartOS
 
 # Import Python libs
 import logging
+import json
 
 # Import Salt libs
 from salt.exceptions import CommandExecutionError
@@ -37,6 +38,43 @@ def _exit_status(retcode):
     return ret
 
 
+def _gen_zone_json(**kwargs):
+    ''' 
+    Generate the JSON for OS virtualization creation
+    
+    Example layout (all keys are mandatory) :
+
+       {"brand": "joyent",
+        "dataset_uuid": "9eac5c0c-a941-11e2-a7dc-57a6b041988f",
+        "alias": "myname",
+        "hostname": "www.domain.com",
+        "max_physical_memory": 2048,
+        "quota": 10,
+        "nics": [
+            {
+                "nic_tag": "admin",
+                "ip": "192.168.0.1",
+                "netmask": "255.255.255.0",
+                "gateway": "192.168.0.254"
+            }
+        ]}
+    '''
+    ret = {}
+    check_args = (
+        'dataset_uuid','alias','hostname',
+        'max_physical_memory','quota','nics')
+    # Lazy check of arguments
+    if not all (key in kwargs for key in check_args):
+        raise CommandExecutionError('Not all arguments are given')
+    # This one is mandatory for OS virt
+    ret.update(brand='joyent')
+    ret.update((key, kwargs[key])
+        for key in check_args 
+        if key in kwargs)
+
+    return json.dumps(ret)
+
+
 def list_vms():
     '''
     Return a list of virtual machine names on the minion

salt/modules/virt.py

@@ -109,13 +109,14 @@ def _get_target(target, ssh):
     return ' %s://%s/%s' % (proto, target, 'system')
 
 
-def _gen_xml(name, cpu, mem, vda, nicp, **kwargs):
+def _gen_xml(name, cpu, mem, vda, nicp, emulator, **kwargs):
     '''
     Generate the XML string to define a libvirt vm
     '''
     mem = mem * 1024
+    print 'emulator: {}'.format(emulator)
     data = '''
-<domain type='kvm'>
+<domain type='%%EMULATOR%%'>
         <name>%%NAME%%</name>
         <vcpu>%%CPU%%</vcpu>
         <memory>%%MEM%%</memory>
@@ -137,6 +138,7 @@ def _gen_xml(name, cpu, mem, vda, nicp, **kwargs):
         </features>
 </domain>
 '''
+    data = data.replace('%%EMULATOR%%', emulator)
     data = data.replace('%%NAME%%', name)
     data = data.replace('%%CPU%%', str(cpu))
     data = data.replace('%%MEM%%', str(mem))
@@ -173,8 +175,8 @@ def _image_type(vda):
     '''
     Detect what driver needs to be used for the given image
     '''
-    out = __salt__['cmd.run']('file {0}'.format(vda))
+    out = __salt__['cmd.run']('qemu-img {0}'.format(vda))
-    if 'Qcow' in out and 'Version: 2' in out:
+    if 'qcow2' in out:
         return 'qcow2'
     else:
         return 'raw'
@@ -188,7 +190,7 @@ def _nic_profile(nic):
     return __salt__['config.option']('virt.nic', {}).get(nic, default)
 
 
-def init(name, cpu, mem, image, nic='default', **kwargs):
+def init(name, cpu, mem, image, nic='default', emulator='kvm', **kwargs):
     '''
     Initialize a new vm
 
@@ -206,7 +208,7 @@ def init(name, cpu, mem, image, nic='default', **kwargs):
         os.makedirs(img_dir)
     nicp = _nic_profile(nic)
     salt.utils.copyfile(sfn, img_dest)
-    xml = _gen_xml(name, cpu, mem, img_dest, nicp, **kwargs)
+    xml = _gen_xml(name, cpu, mem, img_dest, nicp, emulator, **kwargs)
     define_xml_str(xml)
     if kwargs.get('seed'):
         __salt__['img.seed'](img_dest, name, kwargs.get('config'))

salt/modules/xapi.py

@@ -7,33 +7,49 @@ Xen Source, NOT XenServer nor Xen Cloud Platform. As a matter of fact it
 to adapt this code to XS/XCP, mostly playing with XenAPI version, but as
 XCP is not taking precedence on Xen Source on many platforms, please keep
 compatibility in mind.
+
+Useful documentation:
+
+. http://downloads.xen.org/Wiki/XenAPI/xenapi-1.0.6.pdf
+. http://docs.vmd.citrix.com/XenServer/6.0.0/1.0/en_gb/api/
+. https://github.com/xen-org/xen-api/tree/master/scripts/examples/python
+. http://xenbits.xen.org/gitweb/?p=xen.git;a=tree;f=tools/python/xen/xm;hb=HEAD
 '''
 
 # Import python libs
 import sys
 import contextlib
+import os
+import importlib
 
 # Import salt libs
 from salt.exceptions import CommandExecutionError
 import salt.utils
 
+
 # This module has only been tested on Debian GNU/Linux and NetBSD, it
 # probably needs more path appending for other distributions.
 # The path to append is the path to python Xen libraries, where resides
 # XenAPI.
-sys.path.append('/usr/lib/xen-default/lib/python') # Debian
+def _check_xenapi():
-
+    if __grains__['os'] == 'Debian':
+        debian_xen_version = '/usr/lib/xen-common/bin/xen-version'
+        if os.path.isfile(debian_xen_version):
+            # __salt__ is not available in __virtual__
+            xenversion =  salt.modules.cmdmod._run_quiet(debian_xen_version)
+            xapipath = '/usr/lib/xen-{0}/lib/python'.format(xenversion)
+            if os.path.isdir(xapipath):
+                sys.path.append(xapipath)
     try:
-    import xen.xm.XenAPI as XenAPI
+        return importlib.import_module('xen.xm.XenAPI')
-    HAS_XENAPI = True
     except ImportError:
-    HAS_XENAPI = False
+        return False
 
 
 def __virtual__():
-    if HAS_XENAPI is False:
+    if _check_xenapi() is not False:
-        return False
         return 'virt'
+    return False
 
 
 @contextlib.contextmanager
@@ -41,12 +57,14 @@ def _get_xapi_session():
     '''
     Get a session to XenAPI. By default, use the local UNIX socket.
     '''
+    _xenapi = _check_xenapi()
+
     xapi_uri = __salt__['config.option']('xapi.uri')
     xapi_login = __salt__['config.option']('xapi.login')
     xapi_password = __salt__['config.option']('xapi.password')
 
     if not xapi_uri:
-        # "old" xend method
+        # xend local UNIX socket
         xapi_uri = 'httpu:///var/run/xend/xen-api.sock'
     if not xapi_login:
         xapi_login = ''
@@ -54,14 +72,14 @@ def _get_xapi_session():
         xapi_password = ''
 
     try:
-        session = XenAPI.Session(xapi_uri)
+        session = _xenapi.Session(xapi_uri)
         session.xenapi.login_with_password(xapi_login, xapi_password)
 
         yield session.xenapi
     except Exception:
         raise CommandExecutionError('Failed to connect to XenAPI socket.')
     finally:
-        session.logout()
+        session.xenapi.session.logout()
 
 
 # Used rectypes (Record types):
@@ -73,6 +91,16 @@ def _get_xapi_session():
 # VBD
 
 
+def _get_xtool():
+    '''
+    Internal, returns xl or xm command line path
+    '''
+    for xtool in ['xl', 'xm']:
+        path = salt.utils.which(xtool)
+        if path is not None:
+            return path
+
+
 def _get_all(xapi, rectype):
     '''
     Internal, returns all members of rectype
@@ -410,6 +438,58 @@ def setvcpus(vm_, vcpus):
             return False
 
 
+def vcpu_pin(vm_, vcpu, cpus):
+    '''
+    Set which CPUs a VCPU can use.
+
+    CLI Example::
+
+        salt 'foo' virt.vcpu_pin domU-id 2 1
+        salt 'foo' virt.vcpu_pin domU-id 2 2-6
+    '''
+    with _get_xapi_session() as xapi:
+
+        vm_uuid = _get_label_uuid(xapi, 'VM', vm_)
+        if vm_uuid is False:
+            return False
+
+        # from xm's main
+        def cpu_make_map(cpulist):
+            cpus = []
+            for c in cpulist.split(','):
+                if c == '':
+                    continue
+                if c.find('-') != -1:
+                    (x,y) = c.split('-')
+                    for i in range(int(x),int(y)+1):
+                        cpus.append(int(i))
+                else:
+                    # remove this element from the list
+                    if c[0] == '^':
+                        cpus = [x for x in cpus if x != int(c[1:])]
+                    else:
+                        cpus.append(int(c))
+            cpus.sort()
+            return ",".join(map(str, cpus))
+    
+        if cpus == 'all':
+            cpumap = cpu_make_map('0-63')
+        else:
+            cpumap = cpu_make_map('{0}'.format(cpus))
+
+        try:
+            xapi.VM.add_to_VCPUs_params_live(vm_uuid,
+                                            'cpumap{0}'.format(vcpu), cpumap)
+            return True
+        # VM.add_to_VCPUs_params_live() implementation in xend 4.1+ has
+        # a bug which makes the client call fail.
+        # That code is accurate for all others XenAPI implementations, but
+        # for that particular one, fallback to xm / xl instead.
+        except:
+            return __salt__['cmd.run']('{0} vcpu-pin {1} {2} {3}'.format(
+                                            _get_xtool(), vm_, vcpu, cpus))
+
+
 def freemem():
     '''
     Return an int representing the amount of memory that has not been given
@@ -515,13 +595,7 @@ def create(config_):
 
         salt '*' virt.create <path to Xen cfg file>
     '''
-    def get_xtool():
+    return __salt__['cmd.run']('{0} create {1}'.format(_get_xtool(), config_))
-        for xtool in ['xl', 'xm']:
-            path = salt.utils.which(xtool)
-            if path is not None:
-                return path
-
-    return __salt__['cmd.run']('{0} create {1}'.format(get_xtool(), config_))
 
 def start(config_):
     '''