From 8e4413064f78b85a645924c051ebd6fd113b0ebd Mon Sep 17 00:00:00 2001 From: Markus Gattol Date: Wed, 23 Nov 2011 20:22:18 +0000 Subject: [PATCH 01/18] make it easier for folks to start contributing Signed-off-by: Markus Gattol --- README.rst | 22 ++++++++++++++++++++++ setuprepo.sh | 28 ++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100755 setuprepo.sh diff --git a/README.rst b/README.rst index 323b0360cf..8ee83cbb8a 100644 --- a/README.rst +++ b/README.rst @@ -88,3 +88,25 @@ So, please feel free to sprinkle some of this around your systems and let the deliciousness come forth. .. _`Apache 2.0 licence`: http://www.apache.org/licenses/LICENSE-2.0.html + +Contribute +========== +This world is not salty enough yet... help us make it a saltier place: + +* install gitflow e.g. aptitude install git-flow on Debian or from source https://github.com/nvie/gitflow +* fork the salt repository to you your github account +* git clone git@github.com:/salt.git +* cd salt; ./setuprepo.sh + +Start contributing... write a test, make sure the test fails, write +the actual code, make the test pass and if it does, make an atomic +commit (referencing the issue(s) if this is a fix) and git push to +your fork. Issue a pull request so one of the saltstack members can +review it and accept or require/advice a change. Lather, rinse, +repeat... + +* git pull upstream develop +* write the test, make it fail... +* pep8, pylint, pychecker +* commit, push +* pull request diff --git a/setuprepo.sh b/setuprepo.sh new file mode 100755 index 0000000000..dbc9c46c65 --- /dev/null +++ b/setuprepo.sh @@ -0,0 +1,28 @@ +# run this script right after you cloned/forked the repo + +which git-flow 2>/dev/null +has_gitflow=$? +if [ ${has_gitflow} -gt 0 -a ! -x /usr/lib/git-core/git-flow ]; then + echo + echo "*************************************" + echo + echo "You need gitflow to hack on salt" + echo " - https://github.com/nvie/gitflow" + echo " - aptitude install git-flow" + echo +exit 1 +fi + +git checkout master +git remote add upstream https://github.com/saltstack/salt.git +git config push.default tracking # only push the current branch +git config branch.autosetuprebase always # we want a linear history + +echo "Configuring gitflow for this repository..." +git flow init -d +echo +echo "gitflow has been setup successfully!" +echo "See Contribute section at https://github.com/saltstack/salt for further information" + +git checkout develop +git push -u origin develop From cf4ac6844b7539c348df3f41268deb9481160ccd Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 23 Nov 2011 13:55:40 -0700 Subject: [PATCH 02/18] fix path issue in recurse function for deep dirs on the salt master --- salt/states/file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/states/file.py b/salt/states/file.py index a3334c6f10..26f2c75986 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -625,7 +625,8 @@ def recurse(name, source, __env__='base'): os.path.join( __opts__['cachedir'], 'files', - __env__ + __env__, + source[7:] ) ) ) From 31c8168ed38ba89b3a4ce75d18498159533a8f1e Mon Sep 17 00:00:00 2001 From: Tor Hveem Date: Wed, 23 Nov 2011 22:29:34 +0100 Subject: [PATCH 03/18] Add correct usage parameter for OptionParser --- salt/cli/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index a3167b02a0..25f5642e6a 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -39,7 +39,8 @@ class SaltCMD(object): ''' Parse the command line ''' - parser = optparse.OptionParser(version="%%prog %s" % VERSION) + usage = "%prog [options] '' [arguments]" + parser = optparse.OptionParser(version="%%prog %s" % VERSION, usage=usage) parser.add_option('-t', '--timeout', From 55e5f4705912eb9da9e80fd0f69b04c211156dc8 Mon Sep 17 00:00:00 2001 From: Carlo Pires Date: Wed, 23 Nov 2011 19:39:42 -0200 Subject: [PATCH 04/18] Fix kwargs for salt/loader.call() --- salt/loader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/loader.py b/salt/loader.py index f10ce54903..5ab601cf01 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -105,12 +105,12 @@ def grains(opts): return grains -# FIXME: mutable types as default parameter values, NO! -# http://goo.gl/ToU2z -def call(fun, args=[], dirs=[]): +def call(fun, **kwargs): ''' Directly call a function inside a loader directory ''' + args = kwargs.get('args', []) + dirs = kwargs.get('dirs', []) module_dirs = [ os.path.join(salt_base_path, 'modules'), ] + dirs From a76d31edbf4eaa2e47329e19955d7705e825b423 Mon Sep 17 00:00:00 2001 From: Carlo Pires Date: Wed, 23 Nov 2011 19:55:15 -0200 Subject: [PATCH 05/18] Optimization fix for AESFuncs class --- salt/master.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/salt/master.py b/salt/master.py index 9060d263db..50c971fcbe 100644 --- a/salt/master.py +++ b/salt/master.py @@ -545,14 +545,16 @@ class AESFuncs(object): # Don't honor private functions if func.startswith('__'): return self.crypticle.dumps({}) - # Run the func - ret = getattr(self, func)(load) - # Don't encrypt the return value for the _return func - # (we don't care about the return value, so why encrypt it?) - if func == '_return': + elif func == '_return': + # Don't encrypt the return value for the _return func + # (we don't care about the return value, so why encrypt it?) return ret - # AES Encrypt the return - return self.crypticle.dumps(ret) + else: + # Run the func + ret = getattr(self, func)(load) + + # AES Encrypt the return + return self.crypticle.dumps(ret) class ClearFuncs(object): From a5c85752954a922bcf1bae62853d24fc8d9b45b9 Mon Sep 17 00:00:00 2001 From: Tor Hveem Date: Wed, 23 Nov 2011 22:57:41 +0100 Subject: [PATCH 06/18] Fix syntax error --- salt/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cli/__init__.py b/salt/cli/__init__.py index 25f5642e6a..688a590a19 100644 --- a/salt/cli/__init__.py +++ b/salt/cli/__init__.py @@ -153,7 +153,7 @@ class SaltCMD(object): if len(args) < 1: err = ('Please pass in a command to query the old salt ' 'calls for.') - sys.stderr.write(err, + '\n') + sys.stderr.write(err + '\n') sys.exit('2') opts['cmd'] = args[0] else: From 42550c9c0fd58e73b04bfa431de19e442ea76df1 Mon Sep 17 00:00:00 2001 From: Eric Poelke Date: Mon, 21 Nov 2011 10:06:00 -0800 Subject: [PATCH 07/18] updated salt/modules/yum.py to use rpm API for list_pkgs function added a filter option to list_pkgs fucntion used when called from version function. Now returns only a single package and version instead of all packages and versions on system. --- salt/modules/yum.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/salt/modules/yum.py b/salt/modules/yum.py index 8fd30dad97..9d4156c795 100644 --- a/salt/modules/yum.py +++ b/salt/modules/yum.py @@ -55,14 +55,14 @@ def version(name): salt '*' pkg.version ''' - pkgs = list_pkgs() + pkgs = list_pkgs(name) if name in pkgs: return pkgs[name] else: return '' -def list_pkgs(): +def list_pkgs(*args): ''' List the packages currently installed in a dict:: @@ -72,15 +72,18 @@ def list_pkgs(): salt '*' pkg.list_pkgs ''' - cmd = "rpm -qa --qf '%{NAME}:%{VERSION}-%{RELEASE};'" - ret = {} - out = __salt__['cmd.run_stdout'](cmd) - for line in out.split(';'): - if not line.count(':'): - continue - comps = line.split(':') - ret[comps[0]] = comps[1] - return ret + import rpm + ts = rpm.TransactionSet() + pkgs = {} + + if len(args) == 0: + for h in ts.dbMatch(): + pkgs[h['name']] = '-'.join([h['version'],h['release']]) + else: + for h in ts.dbMatch('name', args[0]): + pkgs[h['name']] = '-'.join([h['version'],h['release']]) + + return pkgs def refresh_db(): From 58c34ee4d4cb58f3ba274d49b60ed39eff7bd623 Mon Sep 17 00:00:00 2001 From: Eric Poelke Date: Tue, 22 Nov 2011 00:10:49 -0800 Subject: [PATCH 08/18] yum.py now uses yum API, also ignores packages that do not match the basearch of the system. --- salt/modules/yum.py | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/salt/modules/yum.py b/salt/modules/yum.py index 9d4156c795..58dd5ae3c4 100644 --- a/salt/modules/yum.py +++ b/salt/modules/yum.py @@ -32,18 +32,21 @@ def available_version(name): salt '*' pkg.available_version ''' - out = __salt__['cmd.run_stdout']('yum list {0} -q'.format(name)) - for line in out.split('\n'): - if not line.strip(): - continue - # Itterate through the output - comps = line.split() - if comps[0].split('.')[0] == name: - if len(comps) < 2: - continue - # found it! - return comps[1][:comps[1].rindex('.')] - # Package not available + import yum + from rpmUtils.arch import getBaseArch + + yb = yum.YumBase() + # look for available packages only, if package is already installed with + # latest version it will not show up here. + pl = yb.doPackageLists('available') + exactmatch, matched, unmatched = yum.packages.parsePackages(pl.available, + [name]) + + for pkg in exactmatch: + # ignore packages that do not match base arch + if pkg.arch == getBaseArch(): + return '-'.join([pkg.version, pkg.release]) + return '' @@ -75,13 +78,15 @@ def list_pkgs(*args): import rpm ts = rpm.TransactionSet() pkgs = {} - + # if no args are passed in get all packages if len(args) == 0: for h in ts.dbMatch(): pkgs[h['name']] = '-'.join([h['version'],h['release']]) else: - for h in ts.dbMatch('name', args[0]): - pkgs[h['name']] = '-'.join([h['version'],h['release']]) + # get package version for each package in *args + for arg in args: + for h in ts.dbMatch('name', arg): + pkgs[h['name']] = '-'.join([h['version'],h['release']]) return pkgs @@ -95,9 +100,11 @@ def refresh_db(): salt '*' pkg.refresh_db ''' - cmd = 'yum clean dbcache' - __salt__['cmd.retcode'](cmd) - return True + import yum + yb = yum.YumBase() + yb.cleanMetadata() + return true + def install(pkg, refresh=False): From 996c8191c85dd418a0774dd34045fdaccb8b2bfc Mon Sep 17 00:00:00 2001 From: Eric Poelke Date: Tue, 22 Nov 2011 17:15:45 -0800 Subject: [PATCH 09/18] Renamed yum.py to yumcmd.py to avoid namespace conflict with the real yum module. Updated list_pkgs to handle any number of packages to be passed in for filtering purposes. Added clean_metadata function to keep options similar to the actual yum cli tool. More enhancements to available_version, now will return the version regardless of the package being an update to existing package or the version of a currently not installed package. --- doc/man/salt.7 | 22 +++---- doc/ref/modules/all/salt.modules.yum.rst | 6 -- doc/ref/modules/all/salt.modules.yumpkg.rst | 6 ++ doc/ref/modules/index.rst | 2 +- salt/modules/{yum.py => yumpkg.py} | 66 ++++++++++++++++----- 5 files changed, 70 insertions(+), 32 deletions(-) delete mode 100644 doc/ref/modules/all/salt.modules.yum.rst create mode 100644 doc/ref/modules/all/salt.modules.yumpkg.rst rename salt/modules/{yum.py => yumpkg.py} (72%) diff --git a/doc/man/salt.7 b/doc/man/salt.7 index 0e33f032be..77aec33cf6 100644 --- a/doc/man/salt.7 +++ b/doc/man/salt.7 @@ -1265,7 +1265,7 @@ regardless of what the actual module is named. The package manager modules are the best example of using the \fB__virtual__\fP function: \fI\%https://github.com/thatch45/salt/blob/v0.9.3/salt/modules/pacman.py\fP -\fI\%https://github.com/thatch45/salt/blob/v0.9.3/salt/modules/yum.py\fP +\fI\%https://github.com/thatch45/salt/blob/v0.9.3/salt/modules/yumpkg.py\fP \fI\%https://github.com/thatch45/salt/blob/v0.9.3/salt/modules/apt.py\fP .SS Documentation .sp @@ -1432,7 +1432,7 @@ def ping(): .IP \(bu 2 \fI\%salt.modules.virt\fP .IP \(bu 2 -\fI\%salt.modules.yum\fP +\fI\%salt.modules.yumpkg\fP .UNINDENT .SS \fBsalt.modules.apache\fP .sp @@ -3714,12 +3714,12 @@ Return detailed information about the vms on this hyper in a dict: CLI Example: salt \(aq*\(aq virt.vm_info .UNINDENT -.SS \fBsalt.modules.yum\fP +.SS \fBsalt.modules.yumpkg\fP .sp Support for YUM .INDENT 0.0 .TP -.B salt.modules.yum.available_version(name) +.B salt.modules.yumpkg.available_version(name) The available version of the package in the repository .sp CLI Example: @@ -3727,7 +3727,7 @@ salt \(aq*\(aq pkg.available_version .UNINDENT .INDENT 0.0 .TP -.B salt.modules.yum.install(pkg, refresh=False) +.B salt.modules.yumpkg.install(pkg, refresh=False) Install the passed package, add refresh=True to clean out the yum database before executing .sp @@ -3745,7 +3745,7 @@ salt \(aq*\(aq pkg.install .UNINDENT .INDENT 0.0 .TP -.B salt.modules.yum.list_pkgs() +.B salt.modules.yumpkg.list_pkgs() List the packages currently installed in a dict: {\(aq\(aq: \(aq\(aq} .sp @@ -3754,7 +3754,7 @@ salt \(aq*\(aq pkg.list_pkgs .UNINDENT .INDENT 0.0 .TP -.B salt.modules.yum.purge(pkg) +.B salt.modules.yumpkg.purge(pkg) Yum does not have a purge, this function calls remove .sp Return a list containing the removed packages: @@ -3764,7 +3764,7 @@ salt \(aq*\(aq pkg.purge .UNINDENT .INDENT 0.0 .TP -.B salt.modules.yum.refresh_db() +.B salt.modules.yumpkg.refresh_db() Since yum refreshes the database automatically, this runs a yum clean, so that the next yum operation will have a clean database .sp @@ -3773,7 +3773,7 @@ salt \(aq*\(aq pkg.refresh_db .UNINDENT .INDENT 0.0 .TP -.B salt.modules.yum.remove(pkg) +.B salt.modules.yumpkg.remove(pkg) Remove a single package with yum remove .sp Return a list containing the removed packages: @@ -3783,7 +3783,7 @@ salt \(aq*\(aq pkg.remove .UNINDENT .INDENT 0.0 .TP -.B salt.modules.yum.upgrade() +.B salt.modules.yumpkg.upgrade() Run a full system upgrade, a yum upgrade .sp Return a dict containing the new package names and versions: @@ -3800,7 +3800,7 @@ salt \(aq*\(aq pkg.upgrade .UNINDENT .INDENT 0.0 .TP -.B salt.modules.yum.version(name) +.B salt.modules.yumpkg.version(name) Returns a version if the package is installed, else returns an empty string .sp CLI Example: diff --git a/doc/ref/modules/all/salt.modules.yum.rst b/doc/ref/modules/all/salt.modules.yum.rst deleted file mode 100644 index fd3e8b9db3..0000000000 --- a/doc/ref/modules/all/salt.modules.yum.rst +++ /dev/null @@ -1,6 +0,0 @@ -================ -salt.modules.yum -================ - -.. automodule:: salt.modules.yum - :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.yumpkg.rst b/doc/ref/modules/all/salt.modules.yumpkg.rst new file mode 100644 index 0000000000..db2d843e27 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.yumpkg.rst @@ -0,0 +1,6 @@ +=================== +salt.modules.yumpkg +=================== + +.. automodule:: salt.modules.yumpkg + :members: diff --git a/doc/ref/modules/index.rst b/doc/ref/modules/index.rst index 0a8f17f76a..2680e7ac9f 100644 --- a/doc/ref/modules/index.rst +++ b/doc/ref/modules/index.rst @@ -142,7 +142,7 @@ regardless of what the actual module is named. The package manager modules are the best example of using the ``__virtual__`` function: :blob:`salt/modules/pacman.py` -:blob:`salt/modules/yum.py` +:blob:`salt/modules/yumpkg.py` :blob:`salt/modules/apt.py` Documentation diff --git a/salt/modules/yum.py b/salt/modules/yumpkg.py similarity index 72% rename from salt/modules/yum.py rename to salt/modules/yumpkg.py index 58dd5ae3c4..5628091ec5 100644 --- a/salt/modules/yum.py +++ b/salt/modules/yumpkg.py @@ -37,17 +37,32 @@ def available_version(name): yb = yum.YumBase() # look for available packages only, if package is already installed with - # latest version it will not show up here. - pl = yb.doPackageLists('available') - exactmatch, matched, unmatched = yum.packages.parsePackages(pl.available, - [name]) + # latest version it will not show up here. If we want to use wildcards + # here we can, but for now its exactmatch only. - for pkg in exactmatch: - # ignore packages that do not match base arch - if pkg.arch == getBaseArch(): - return '-'.join([pkg.version, pkg.release]) + versions_list = [] + for pkgtype in ['available', 'updates']: + + pl = yb.doPackageLists(pkgtype) + exactmatch, matched, unmatched = yum.packages.parsePackages(pl, [name]) + # build a list of available packages from either available or updates + # this will result in a double match for a package that is already + # installed. Maybe we should just return the value if we get a hit + # on available, and only iterate though updates if we don't.. + for pkg in exactmatch: + if pkg.arch == getBaseArch(): + versions_list.append('-'.join([pkg.version, pkg.release])) + + if len(versions_list) == 0: + # if versions_list is empty return empty string. It may make sense + # to also check if a package is installed and on latest version + # already and return a message saying 'up to date' or something along + # those lines. + return '' + + # remove the duplicate items from the list and return the first one + return list(set(versions_list))[0] - return '' def version(name): @@ -103,9 +118,19 @@ def refresh_db(): import yum yb = yum.YumBase() yb.cleanMetadata() - return true + return True +def clean_metadata(): + ''' + Cleans local yum metadata. + + CLI Example:: + + salt '*' pkg.clean_metadata + ''' + return refresh_db() + def install(pkg, refresh=False): ''' @@ -121,13 +146,24 @@ def install(pkg, refresh=False): salt '*' pkg.install ''' - old = list_pkgs() - cmd = 'yum -y install ' + pkg + + # WIP only commiting this so I can keep working on it at home later... + import yum + if refresh: refresh_db() - __salt__['cmd.retcode'](cmd) - new = list_pkgs() + + yb = yum.YumBase() + + try: + yb.install(name=pkg) + yb.resolveDeps() + except yum.Errors.InstallError, e: + return False + + pkgs = {} + for npkg in new: if npkg in old: if old[npkg] == new[npkg]: @@ -188,7 +224,9 @@ def remove(pkg): salt '*' pkg.remove ''' +# import subprocess as sp old = list_pkgs() +# sp.Popen(['yum', '-y', 'remove', pkg]) cmd = 'yum -y remove ' + pkg __salt__['cmd.retcode'](cmd) new = list_pkgs() From 996c075d7c2b482ca4b2e019cf97a986901af2b3 Mon Sep 17 00:00:00 2001 From: Eric Poelke Date: Wed, 23 Nov 2011 13:28:42 -0800 Subject: [PATCH 10/18] updated install/upgrade/remove functions in yumpkg.py to use yum API as well as allow for multiple packages to be installed/removed in one transaction. --- salt/modules/yumpkg.py | 142 ++++++++++++++++++++++++++--------------- 1 file changed, 90 insertions(+), 52 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 5628091ec5..18c1f78c1a 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -2,7 +2,6 @@ Support for YUM ''' - def __virtual__(): ''' Confine this module to yum based systems @@ -21,6 +20,29 @@ def _list_removed(old, new): for pkg in old: if pkg not in new: pkgs.append(pkg) + + return pkgs + + +def _compare_versions(old, new): + ''' + Returns a dict that that displays old and new versions for a package after + install/upgrade of package. + ''' + pkgs = {} + for npkg in new: + if npkg in old: + if old[npkg] == new[npkg]: + # no change in the package + continue + else: + # the package was here before and the version has changed + pkgs[npkg] = {'old': old[npkg], + 'new': new[npkg]} + else: + # the package is freshly installed + pkgs[npkg] = {'old': '', + 'new': new[npkg]} return pkgs @@ -132,7 +154,7 @@ def clean_metadata(): return refresh_db() -def install(pkg, refresh=False): +def install(*args, **kwargs): ''' Install the passed package, add refresh=True to clean out the yum database before executing @@ -146,38 +168,40 @@ def install(pkg, refresh=False): salt '*' pkg.install ''' - - # WIP only commiting this so I can keep working on it at home later... import yum - + import rpm + + # If the kwarg refresh exists get it, otherwise set it to False + refresh = kwargs.get('refresh', False) + if refresh: refresh_db() - + + old = list_pkgs(*args) + yb = yum.YumBase() - - try: - yb.install(name=pkg) + setattr(yb.conf, 'assumeyes', True) + + try: + for pkg in args: + yb.install(name=pkg) + # Resolve Deps before attempting install. This needs to be improved + # by also tracking any deps that may get upgraded/installed during this + # process. For now only the version of the package(s) you request be + # installed is tracked. yb.resolveDeps() + yb.processTransaction(rpmDisplay=yum.rpmtrans.NoOutputCallBack()) + yb.closeRpmDB() + except yum.Errors.InstallError, e: - return False - + # returns a empyt string in the event of a failure. Any ideas on what + # we should do here? Do we want to return a failure message of some + # sort? + return '' - pkgs = {} - - for npkg in new: - if npkg in old: - if old[npkg] == new[npkg]: - # no change in the package - continue - else: - # the package was here before and the version has changed - pkgs[npkg] = {'old': old[npkg], - 'new': new[npkg]} - else: - # the package is freshly installed - pkgs[npkg] = {'old': '', - 'new': new[npkg]} - return pkgs + new = list_pkgs(*args) + + return _compare_versions(old, new) def upgrade(): @@ -193,30 +217,31 @@ def upgrade(): salt '*' pkg.upgrade ''' + import yum + + yb = yum.YumBase() + setattr(yb.conf, 'assumeyes', True) + old = list_pkgs() - cmd = 'yum -y upgrade' - __salt__['cmd.retcode'](cmd) + + try: + # ideally we would look in the yum transaction and get info on all the + # packages that are going to be upgraded and only look up old/new + # version info on those packages. + yb.update() + yb.resolveDeps() + yb.processTransaction(rpmDisplay=yum.rpmtrans.NoOutputCallBack()) + yb.closeRpmDB() + except yum.Errors.InstallError, e: + return '' + new = list_pkgs() - pkgs = {} - for npkg in new: - if npkg in old: - if old[npkg] == new[npkg]: - # no change in the package - continue - else: - # the package was here before and the version has changed - pkgs[npkg] = {'old': old[npkg], - 'new': new[npkg]} - else: - # the package is freshly installed - pkgs[npkg] = {'old': '', - 'new': new[npkg]} - return pkgs + return _compare_versions(old, new) -def remove(pkg): +def remove(*args): ''' - Remove a single package with yum remove + Removes packages with yum remove Return a list containing the removed packages: @@ -224,12 +249,25 @@ def remove(pkg): salt '*' pkg.remove ''' -# import subprocess as sp - old = list_pkgs() -# sp.Popen(['yum', '-y', 'remove', pkg]) - cmd = 'yum -y remove ' + pkg - __salt__['cmd.retcode'](cmd) - new = list_pkgs() + import yum + + yb = yum.YumBase() + setattr(yb.conf, 'assumeyes', True) + old = list_pkgs(*args) + + try: + # same comments as in upgrade for remove. + for pkg in args: + yb.remove(name=pkg) + + yb.resolveDeps() + yb.processTransaction(rpmDisplay=yum.rpmtrans.NoOutputCallBack()) + yb.closeRpmDB() + except yum.Errors.RemoveError, e: + return '' + + new = list_pkgs(*args) + return _list_removed(old, new) From b82aac41f41be69572e2fbf4813a4da2d2a75f1e Mon Sep 17 00:00:00 2001 From: Eric Poelke Date: Wed, 23 Nov 2011 13:49:21 -0800 Subject: [PATCH 11/18] moved imports out of functions, removed exception handler for upgrade/install/remove --- salt/modules/yumpkg.py | 84 ++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 53 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 18c1f78c1a..4aeeaaa6b9 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -1,6 +1,9 @@ ''' Support for YUM ''' +import yum +import rpm +from rpmUtils.arch import getBaseArch def __virtual__(): ''' @@ -53,15 +56,11 @@ def available_version(name): CLI Example:: salt '*' pkg.available_version - ''' - import yum - from rpmUtils.arch import getBaseArch - + ''' yb = yum.YumBase() # look for available packages only, if package is already installed with # latest version it will not show up here. If we want to use wildcards # here we can, but for now its exactmatch only. - versions_list = [] for pkgtype in ['available', 'updates']: @@ -86,7 +85,6 @@ def available_version(name): return list(set(versions_list))[0] - def version(name): ''' Returns a version if the package is installed, else returns an empty string @@ -112,7 +110,6 @@ def list_pkgs(*args): salt '*' pkg.list_pkgs ''' - import rpm ts = rpm.TransactionSet() pkgs = {} # if no args are passed in get all packages @@ -137,7 +134,6 @@ def refresh_db(): salt '*' pkg.refresh_db ''' - import yum yb = yum.YumBase() yb.cleanMetadata() return True @@ -168,9 +164,6 @@ def install(*args, **kwargs): salt '*' pkg.install ''' - import yum - import rpm - # If the kwarg refresh exists get it, otherwise set it to False refresh = kwargs.get('refresh', False) @@ -182,23 +175,16 @@ def install(*args, **kwargs): yb = yum.YumBase() setattr(yb.conf, 'assumeyes', True) - try: - for pkg in args: - yb.install(name=pkg) - # Resolve Deps before attempting install. This needs to be improved - # by also tracking any deps that may get upgraded/installed during this - # process. For now only the version of the package(s) you request be - # installed is tracked. - yb.resolveDeps() - yb.processTransaction(rpmDisplay=yum.rpmtrans.NoOutputCallBack()) - yb.closeRpmDB() - - except yum.Errors.InstallError, e: - # returns a empyt string in the event of a failure. Any ideas on what - # we should do here? Do we want to return a failure message of some - # sort? - return '' - + for pkg in args: + yb.install(name=pkg) + # Resolve Deps before attempting install. This needs to be improved + # by also tracking any deps that may get upgraded/installed during this + # process. For now only the version of the package(s) you request be + # installed is tracked. + yb.resolveDeps() + yb.processTransaction(rpmDisplay=yum.rpmtrans.NoOutputCallBack()) + yb.closeRpmDB() + new = list_pkgs(*args) return _compare_versions(old, new) @@ -217,23 +203,19 @@ def upgrade(): salt '*' pkg.upgrade ''' - import yum - + yb = yum.YumBase() setattr(yb.conf, 'assumeyes', True) old = list_pkgs() - try: - # ideally we would look in the yum transaction and get info on all the - # packages that are going to be upgraded and only look up old/new - # version info on those packages. - yb.update() - yb.resolveDeps() - yb.processTransaction(rpmDisplay=yum.rpmtrans.NoOutputCallBack()) - yb.closeRpmDB() - except yum.Errors.InstallError, e: - return '' + # ideally we would look in the yum transaction and get info on all the + # packages that are going to be upgraded and only look up old/new version + # info on those packages. + yb.update() + yb.resolveDeps() + yb.processTransaction(rpmDisplay=yum.rpmtrans.NoOutputCallBack()) + yb.closeRpmDB() new = list_pkgs() return _compare_versions(old, new) @@ -249,29 +231,25 @@ def remove(*args): salt '*' pkg.remove ''' - import yum yb = yum.YumBase() setattr(yb.conf, 'assumeyes', True) old = list_pkgs(*args) - try: - # same comments as in upgrade for remove. - for pkg in args: - yb.remove(name=pkg) - - yb.resolveDeps() - yb.processTransaction(rpmDisplay=yum.rpmtrans.NoOutputCallBack()) - yb.closeRpmDB() - except yum.Errors.RemoveError, e: - return '' + # same comments as in upgrade for remove. + for pkg in args: + yb.remove(name=pkg) + + yb.resolveDeps() + yb.processTransaction(rpmDisplay=yum.rpmtrans.NoOutputCallBack()) + yb.closeRpmDB() new = list_pkgs(*args) return _list_removed(old, new) -def purge(pkg): +def purge(*args): ''' Yum does not have a purge, this function calls remove @@ -281,4 +259,4 @@ def purge(pkg): salt '*' pkg.purge ''' - return remove(pkg) + return remove(*args) From b9a325c9d9e7550babb46e6fff81e0d4339ae0dd Mon Sep 17 00:00:00 2001 From: root Date: Wed, 23 Nov 2011 15:08:47 -0800 Subject: [PATCH 12/18] Changed install/remove functions to use comma delimited list as package list instead of *args, due to compatibility issues with passing in the refresh flag at runtime. --- salt/modules/yumpkg.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 4aeeaaa6b9..7e0a1063e5 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -150,10 +150,10 @@ def clean_metadata(): return refresh_db() -def install(*args, **kwargs): +def install(pkgs, refresh=False): ''' - Install the passed package, add refresh=True to clean out the yum database - before executing + Install the passed package(s), add refresh=True to clean out the yum + database before executing Return a dict containing the new package names and versions:: @@ -162,20 +162,21 @@ def install(*args, **kwargs): CLI Example:: - salt '*' pkg.install + salt '*' pkg.install ''' # If the kwarg refresh exists get it, otherwise set it to False - refresh = kwargs.get('refresh', False) + #refresh = kwargs.get('refresh', False) if refresh: refresh_db() - old = list_pkgs(*args) + pkgs = pkgs.split(',') + old = list_pkgs(*pkgs) yb = yum.YumBase() setattr(yb.conf, 'assumeyes', True) - - for pkg in args: + + for pkg in pkgs: yb.install(name=pkg) # Resolve Deps before attempting install. This needs to be improved # by also tracking any deps that may get upgraded/installed during this @@ -185,7 +186,7 @@ def install(*args, **kwargs): yb.processTransaction(rpmDisplay=yum.rpmtrans.NoOutputCallBack()) yb.closeRpmDB() - new = list_pkgs(*args) + new = list_pkgs(*pkgs) return _compare_versions(old, new) @@ -221,7 +222,7 @@ def upgrade(): return _compare_versions(old, new) -def remove(*args): +def remove(pkgs): ''' Removes packages with yum remove @@ -229,27 +230,28 @@ def remove(*args): CLI Example:: - salt '*' pkg.remove + salt '*' pkg.remove ''' yb = yum.YumBase() setattr(yb.conf, 'assumeyes', True) - old = list_pkgs(*args) + pkgs = pkgs.split(',') + old = list_pkgs(*pkgs) # same comments as in upgrade for remove. - for pkg in args: + for pkg in pkgs: yb.remove(name=pkg) yb.resolveDeps() yb.processTransaction(rpmDisplay=yum.rpmtrans.NoOutputCallBack()) yb.closeRpmDB() - new = list_pkgs(*args) + new = list_pkgs(*pkgs) return _list_removed(old, new) -def purge(*args): +def purge(pkgs): ''' Yum does not have a purge, this function calls remove @@ -259,4 +261,4 @@ def purge(*args): salt '*' pkg.purge ''' - return remove(*args) + return remove(pkgs) From 5570836e1b020a9a036b8ef3fe81fbb0314b2cda Mon Sep 17 00:00:00 2001 From: Jed Glazner Date: Wed, 23 Nov 2011 23:12:05 +0000 Subject: [PATCH 13/18] solr verserion 0.2 --- salt/modules/solr.py | 934 +++++++++++++++++++++++++++++-------------- 1 file changed, 624 insertions(+), 310 deletions(-) diff --git a/salt/modules/solr.py b/salt/modules/solr.py index 7a914b7d1c..e705b21735 100644 --- a/salt/modules/solr.py +++ b/salt/modules/solr.py @@ -1,49 +1,93 @@ -""" -Manage Apache Solr +############################################################################### +# APACHE SOLR SALT MODULE # +# Author: Jed Glazner # +# Version: 0.2 # +# Modified: 9/20/2011 # +# # +# This module uses http requests to talk to the apache solr request handlers # +# to gather information and report errors. Because of this the minion doesn't # +# nescessarily need to reside on the actual slave. However if you want to # +# use the signal function the minion must reside on the physical solr host. # +# # +# This module supports multi-core and standard setups. Certain methods are # +# master/slave specific. Make sure you set the solr.type. If you have # +# questions or want a feature request please ask. # +# # +# ############################################################################# -This module uses http requests to talk to the Apache Solr request handlers to -gather information and report errors. Because of this the minion doesn't -nescessarily need to reside on the actual slave. However if you want to use -the signal function the minion must reside on the physical Solr host. - -This module supports multi-core and standard setups. Certain methods are -master/slave specific. Make sure you set the ``solr.type``. If you have -questions or want a feature request please ask. - -""" +####################### COMING FEATURES IN 0.3 ############################### +# +# 1. Add command for checking for replication failures on slaves +# 2. Improve match_index_versions since it's pointless on busy solr masters +# 3. Add additional local fs checks for backups to make sure they succeeded +# +############################################################################## +import urllib2 import json -import urllib +import socket +import os - -# Override these in the minion config. solr.cores as an empty list indicates -# this is not a multi-core setup. +# Override these in the minion config. +''' +solr.cores: A list of core names eg ['core1','core2']. + An empty list indicates non-multicore setup. +solr.baseurl: The root level url to access solr via http +solr.request_timeout: The number of seconds before timing out an http/https/ftp + request. If nothing is specified then the python global + timeout setting is used. +solr.type: Possible values are 'master' or 'slave' +solr.backup_path: The path to store your backups. If you are using cores and + you can specify to append the core name to the path in the + backup method. +solr.num_backups: For versions of solr >= 3.5. Indicates the number of backups + to keep. This option is ignored if your version is less. +solr.init_script: The full path to your init script with start/stop options +solr.dih.options: A list of options to pass to the dih. +------------------- +Required Options for DIH +clean (False): Clear the index before importing +commit (True): Commit the documents to the index upon completion +optimize (True): Optimize the index after commit is complete +verbose (True): Get verbose output +''' +#sane defaults __opts__ = {'solr.cores': [], 'solr.baseurl': 'http://localhost:8983/solr', - 'solr.type': 'master', - 'solr.init_script': '/etc/rc.d/solr'} + 'solr.type':'master', + 'solr.request_timeout': None, + 'solr.init_script': '/etc/rc.d/solr', + 'solr.dih.import_options': {'clean':False, 'optimize':True, + 'commit':True, 'verbose':False}, + 'solr.backup_path': None, + 'solr.num_backups':1 + } +########################## PRIVATE METHODS ############################## def __virtual__(): ''' PRIVATE METHOD Solr needs to be installed to use this. - Return: str/bool Indicates whether solr is present or not + Return: str/bool Indicates weather solr is present or not - TODO: currently __salt__ is not available to call in this method because + TODO:// currently __salt__ is not available to call in this method because all the salt modules have not been loaded yet. Use a grains module? ''' - # FIXME: this module should only be available if it is installed return 'solr' + names = ['solr', 'apache-solr'] + for name in names: + if __salt__['pkg.version'](name): + return 'solr' + return False -def __check_for_cores__(): +def _check_for_cores(): ''' PRIVATE METHOD - Checks to see if using_cores has been set or not. - If it has been set return it, otherwise figure it out and set it, - then return it. + Checks to see if using_cores has been set or not. if it's been set + return it, otherwise figure it out and set it. Then return it Return: bool True indicates that cores are used. ''' @@ -52,7 +96,6 @@ def __check_for_cores__(): else: return False - def _get_return_dict(success=True, data={}, errors=[], warnings=[]): ''' PRIVATE METHOD @@ -63,20 +106,19 @@ def _get_return_dict(success=True, data={}, errors=[], warnings=[]): Param: list errors Default = [] Param: list warnings Default= [] ''' - ret = {'success': success, - 'data':data, - 'errors':errors, + ret = {'success':success, + 'data':data, + 'errors':errors, 'warnings':warnings} return ret - def _update_return_dict(ret, success, data, errors, warnings=[]): ''' PRIVATE METHOD Updates the return dictionary and returns it. - Param: dict ret: The origional returning dictionary to update + Param: dict ret: The original returning dictionary to update Param: bool success: Indicates if the call was successful. Param: dict data: The data to update. Param: list errors: Errors list to append to the return @@ -86,87 +128,190 @@ def _update_return_dict(ret, success, data, errors, warnings=[]): ret['data'].update(data) ret['errors'] = ret['errors'] + errors ret['warnings'] = ret['warnings'] + warnings - return ret + return ret -def _format_url(handler, core_name=None, extra=[]): +def _format_url(handler,core_name=None,extra=[]): ''' PRIVATE METHOD Formats the url based on parameters, and if cores are used or not Param: str request_handler: The request handler to hit - Param: str core_name (None): The name of the solr core if using cores. + Param: str core_name (None): The name of the solr core if using cores. Leave this blank if you are not using cores or - if you want to check all cores. + if you want to check all cores. Param: list extra ([]): A list of additional name value pairs ['name=value] Return: str formatted url ''' baseurl = __opts__['solr.baseurl'] if core_name is None: - return "{0}/{1}?wt=json".format(baseurl, handler) + if extra is None: + return "{0}/{1}?wt=json".format(baseurl, handler) + else: + return "{0}/{1}?wt=json&{2}".format(baseurl, handler,"&".join(extra)) else: if extra is None: return "{0}/{1}/{2}?wt=json".format(baseurl, core_name, handler) else: - return "{0}/{1}/{2}?wt=json&{3}".format(baseurl, core_name, - handler, "&".join(extra)) - + return "{0}/{1}/{2}?wt=json&{3}".format(baseurl, core_name, + handler,"&".join(extra)) def _http_request(url): ''' PRIVATE METHOD Uses json.load to fetch the json results from the solr api. - Param: str Url (A formatted url to send to urllib) + Param: str Url (A formatted url to and to urllib) Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} TODO://Add a timeout param. ''' try: - data = json.load(urllib.urlopen(url)) - return _get_return_dict(True, data, []) + request_timeout = __opts__['solr.request_timeout'] + if request_timeout is None: + data = json.load(urllib2.urlopen(url)) + else: + data = json.load(urllib2.urlopen(url,timeout=request_timeout)) + return _get_return_dict(True, data,[]) except Exception as e: - return _get_return_dict(False, {}, ["{0} : {1}".format(url, e)]) - + return _get_return_dict(False, {}, ["{0} : {1}".format(url,e)]) def _replication_request(replication_command, core_name=None, params=[]): ''' PRIVATE METHOD - Performs the requested replication command and returns a dictionary with - success, errors and data as keys. The data object will contain the json + Performs the requested replication command and returns a dictionary with + success, errors and data as keys. The data object will contain the json response. Param: str replication_command: The replication command to execute - Param: str core_name (None): The name of the solr core if using cores. + Param: str core_name (None): The name of the solr core if using cores. Leave this blank if you are not using cores or if you want to check all cores. - Param: list params ([]): Any additional parameters you want send. + Param: list params ([]): Any additional parameters you want send. Should be a list of strings in name=value format. Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} ''' extra = ["command={0}".format(replication_command)] + params - url = _format_url('replication', core_name=core_name, extra=extra) + url = _format_url('replication',core_name=core_name,extra=extra) return _http_request(url) - def _get_admin_info(command, core_name=None): ''' PRIVATE METHOD - Calls the _http_request method, passes the admin command to execute, - and stores the data. This data is fairly static but should be refreshed - periodically to make sure everying this ok. The data object will contain - the json response. + Calls the _http_request method and passes the admin command to execute + and stores the data. This data is fairly static but should be refreshed + periodically to make sure everything this OK. The data object will contain + the json response. Param: str command: The admin command to run - Param: str core_name (None): The name of the solr core if using cores. + Param: str core_name (None): The name of the solr core if using cores. Leave this blank if you are not using cores or if you want to check all cores. Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} ''' url = _format_url("admin/{0}".format(command), core_name=core_name) - resp = _http_request(url) + resp = _http_request(url) return resp +def _is_master(): + ''' + PRIVATE METHOD + Simple method to determine if the minion is configured as master or slave + + Return: bool + ''' + if __opts__['solr.type'] == 'master': + return True + return False + +def _merge_options(options): + ''' + PRIVATE METHOD + updates the default import options from __opts__['solr.dih.import_options'] + with the dictionary passed in. Also converts booleans to strings + to pass to solr. + + Param: dict options {str:bool}: Dictionary that over rides default options + Return: dict {str:str} + ''' + + + defaults = __opts__['solr.dih.import_options'] + if type(options) == dict: + defaults.update(options) + for (k,v) in defaults.items(): + if type(v) is bool: + defaults[k] = str(bool(v)).lower() + return defaults + + +def _pre_index_check(handler, core_name=None): + ''' + PRIVATE METHOD + Does a pre-check to make sure that all the options are set and that + we can talk to solr before trying to send a command to solr. + ''' + #make sure that it's a master minion + if not _is_master(): + err = ['solr.pre_indexing_check can only be called by "master" minions'] + return _get_return_dict(False, err) + ''' + solr can run out of memory quickly if the dih is processing multiple + handlers at the same time, so if it's a multicore setup require a + core_name param. + ''' + if core_name is None and _check_for_cores(): + errors = ['solr.full_import is not safe to multiple handlers at once'] + return _get_return_dict(False, errors=errors) + #check to make sure that we're not already indexing + resp = import_status(handler, core_name) + if resp['success']: + status = resp['data']['status'] + if status == 'busy': + warn = ['An indexing process is already running.'] + return _get_return_dict(True, warnings=warn) + if status != 'idle': + errors = ['Unknown status: "{0}"'.format(status)] + return _get_return_dict(False, data=resp['data'],errors=errors) + else: + errors = ['Status check failed. Response details: {0}'.format(resp)] + return _get_return_dict(False, data=resp['data'],errors=errors) + + return resp + +def _find_value(ret_dict, key, path=None): + ''' + PRIVATE METHOD + Traverses a dictionary of dictionaries/lists to find key + and return the value stored. + TODO:// this method doesn't really work very well, and it's not really very + useful in it's current state. The purpose for this method is to + simplify parsing the json ouput so you can just pass the key + you want to find and have it return the value. + + Param: dict return_dict: The return dictionary + Param: str key: The key to find in the dictionary. + + Return: list [{path:path, value:value}] + ''' + if path is None: + path = key + else: + path = "{0}:{1}".format(path, key) + + ret = [] + for (k, v) in ret_dict.items(): + if k == key: + ret.append({path:v}) + if type(v) is list: + for x in v: + if type(x) is dict: + ret = ret + _find_value(x, key, path) + if type(v) is dict: + ret = ret + _find_value(v, key, path) + return ret + +########################## PUBLIC METHODS ############################## def lucene_version(core_name=None): ''' @@ -174,451 +319,620 @@ def lucene_version(core_name=None): setup you should specify a core name since all the cores run under the same servlet container, they will all have the same version. - core_name : str (None) - The name of the solr core if using cores. Leave this blank if you are - not using cores or if you want to check all cores. + Param: str core_name (None): The name of the solr core if using cores. + Leave this blank if you are not using cores or + if you want to check all cores. + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - Return : dict:: - - {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - - CLI Example:: - - salt '*' solr.lucene_version + CLI Example: + salt '*' solr.lucene_version ''' ret = _get_return_dict() #do we want to check for all the cores? - if core_name is None and __check_for_cores__(): - success = True + if core_name is None and _check_for_cores(): + success=True for name in __opts__['solr.cores']: - resp = _get_admin_info('system', core_name=name) + resp = _get_admin_info('system', core_name=name ) if resp['success']: version = resp['data']['lucene']['lucene-spec-version'] - data = {name: {'lucene_version': version}} - #generally this means that an exception happened. - else: - data = {name: {'lucene_version': None}} - success = False - ret = _update_return_dict(ret, success, data, resp['errors']) + data = {name: {'lucene_version':version}} + else:#generally this means that an exception happened. + data = {name:{'lucene_version':None}} + success=False + ret = _update_return_dict(ret,success, data, resp['errors']) return ret else: resp = _get_admin_info('system', core_name=core_name) if resp['success']: version = resp['data']['lucene']['lucene-spec-version'] - return _get_return_dict(True, {'version': version}, resp['errors']) + return _get_return_dict(True, {'version':version}, resp['errors']) else: return resp - def version(core_name=None): ''' - Gets the solr version for the specified core. You should specify a core - here as all the cores will run under the same servlet container and so + Gets the solr version for the core specified. You should specify a core + here as all the cores will run under the same servlet container and so will all have the same version. - core_name : str (None) - The name of the solr core if using cores. Leave this blank if you are - not using cores or if you want to check all cores. + Param: str core_name (None): The name of the solr core if using cores. + Leave this blank if you are not using cores or + if you want to check all cores. + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - Return : dict:: - - {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - - CLI Example:: - - alt '*' solr.version + CLI Example: + alt '*' solr.version ''' ret = _get_return_dict() #do we want to check for all the cores? - if core_name is None and __check_for_cores__(): - success = True + if core_name is None and _check_for_cores(): + success=True for name in __opts__['solr.cores']: - resp = _get_admin_info('system', core_name=name) + resp = _get_admin_info('system', core_name=name ) if resp['success']: lucene = resp['data']['lucene'] - data = {name: {'version': lucene['solr-spec-version']}} + data = {name:{'version':lucene['solr-spec-version']}} else: - success = False - data = {name: {'version': None}} - ret = _update_return_dict(ret, success, data, + success=False + data = {name:{'version':None}} + ret = _update_return_dict(ret, success, data, resp['errors'], resp['warnings']) return ret else: resp = _get_admin_info('system', core_name=core_name) if resp['success']: version = resp['data']['lucene']['solr-spec-version'] - return _get_return_dict(True, {'version': version}, + return _get_return_dict(True, {'version':version}, reps['errors'], resp['warnings']) else: return resp - def optimize(core_name=None): ''' - RUN ON THE MASTER ONLY - Optimize the solr index. This should be done on a daily basis and only on - solr masters. It may take a LONG time to run and depending on timeout - settings may time out the http request. - - core_name : str (None) - The name of the solr core if using cores. Leave this blank if you are - not using cores or if you want to check all cores. - - Return : dict:: - - {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - - CLI Example:: - - salt '*' solr.optimize music + Optimize the solr index. Optimizing the index is a good way to keep + Search queries fast, but it is a very expensive operation. The ideal + process is to run this with a master/slave configuration. Then you + can optimize the master, and push the optimized index to the slaves. + If you are running a single solr instance, or if you are going to run + this on a slave be aware than search performance will be horrible + while this command is being run. Additionally it can take a LONG time + to run and your http request may timeout. If that happens adjust your + timeout settings. + + Param: str core_name (None): The name of the solr core if using cores. + Leave this blank if you are not using cores or + if you want to check all cores. + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} + + CLI Example: + salt '*' solr.optimize music ''' ret = _get_return_dict() - # since only masters can call this let's check the config: - if __opts__['solr.type'] != 'master': - errors = ['Only minions configured as solr masters can run this'] - return ret.update({'success': False, 'errors': errors}) - - if core_name is None and __check_for_cores__(): - success = True + if core_name is None and _check_for_cores(): + success=True for name in __opts__['solr.cores']: - url = _format_url('update', core_name=name, - extra=["optimize=true"]) + url = _format_url('update',core_name=name,extra=["optimize=true"]) resp = _http_request(url) if resp['success']: - data = {name: {'data': resp['data']}} - ret = _update_return_dict(ret, success, data, - resp['errors'], resp['warnings']) - else: - success = False - data = {name: {'data': resp['data']}} - ret = _update_return_dict(ret, success, data, + data = {name : {'data':resp['data']}} + ret = _update_return_dict(ret, success, data, resp['errors'], resp['warnings']) - return ret + else: + success=False + data = {name : {'data':resp['data']}} + ret = _update_return_dict(ret, success, data, + resp['errors'], resp['warnings']) + return ret else: - url = _format_url('update', core_name=core_name, - extra=["optimize=true"]) + url = _format_url('update',core_name=core_name,extra=["optimize=true"]) return _http_request(url) - def ping(core_name=None): ''' - Does a health check on solr, makes sure solr can talk to the indexes. + Does a health check on solr, makes sure solr can talk to the indexes. - core_name : str (None) - The name of the solr core if using cores. Leave this blank if you are - not using cores or if you want to check all cores. + Param: str core_name (None): The name of the solr core if using cores. + Leave this blank if you are not using cores or + if you want to check all cores. + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - Return : dict:: - - {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - - CLI Example:: - - salt '*' solr.ping music + CLI Example: + salt '*' solr.ping music ''' ret = _get_return_dict() - if core_name is None and __check_for_cores__(): - success = True + if core_name is None and _check_for_cores(): + success=True for name in __opts__['solr.cores']: resp = _get_admin_info('ping', core_name=name) if resp['success']: - data = {name: {'status': resp['data']['status']}} + data = {name:{'status':resp['data']['status']}} else: - success = False - data = {name: {'status': None}} - ret = _update_return_dict(ret, success, data, resp['errors']) + success=False + data = {name:{'status':None}} + ret = _update_return_dict(ret,success, data, resp['errors']) return ret else: resp = _get_admin_info('ping', core_name=core_name) - if resp['success']: - return _get_return_dict(ret, True, resp['data'], resp['errors']) - else: - return resp - + return resp def is_replication_enabled(core_name=None): ''' USED ONLY BY SLAVES Check for errors, and determine if a slave is replicating or not. - core_name : str (None) - The name of the solr core if using cores. Leave this blank if you are - not using cores or if you want to check all cores. + Param: str core_name (None): The name of the solr core if using cores. + Leave this blank if you are not using cores or + if you want to check all cores. + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - Return : dict:: - - {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - - CLI Example:: - - salt '*' solr.is_replication_enabled music + CLI Example: + salt '*' solr.is_replication_enabled music ''' ret = _get_return_dict() success = True # since only slaves can call this let's check the config: if __opts__['solr.type'] != 'slave': - errors = ['Only minions configured as solr slaves can run this'] - return ret.update({'success': False, 'errors': errors}) - + errors = ['Only "slave" minions can run "is_replication_enabled"'] + return ret.update({'success':False, 'errors':errors}) #define a convenience method so we don't duplicate code - def _checks(ret, success, resp, core): + def _checks(ret, success, resp,core): if response['success']: slave = resp['data']['details']['slave'] # we need to initialize this to false in case there is an error # on the master and we can't get this info. - replication_enabled = 'false' + replication_enabled = 'false' master_url = slave['masterUrl'] #check for errors on the slave - if 'ERROR' in slave: - success = False + if slave.has_key('ERROR'): + success=False err = "{0}: {1} - {2}".format(name, slave['ERROR'], master_url) resp['errors'].append(err) #if there is an error return everything - data = slave if core is None else {core: {'data': slave}} + data = slave if core is None else {core : {'data':slave}} else: - enabled = (slave['masterDetails'] - ['master']['replicationEnabled']) - #if replication is turned off on the master, or polling is - #isabled we need to return false. These may not not errors, - #but the purpose of this call is to check to see if the slaves - #can replicate. + enabled = slave['masterDetails']['master']['replicationEnabled'] + ''' + if replication is turned off on the master, or polling is + disabled we need to return false. These may not not errors, + but the purpose of this call is to check to see if the slaves + can replicate. + ''' if enabled == 'false': - resp['warnings'].append("Replicaiton is disabled on master.") + resp['warnings'].append("Replication is disabled on master.") success = False if slave['isPollingDisabled'] == 'true': success = False resp['warning'].append("Polling is disabled") #update the return - ret = _update_return_dict(ret, success, data, + ret = _update_return_dict(ret, success, data, resp['errors'], resp['warnings']) return (ret, success) - if core_name is None and __check_for_cores__(): + if core_name is None and _check_for_cores(): for name in __opts__['solr.cores']: response = _replication_request('details', core_name=name) - ret, success = _checks(ret, success, response, name) + ret, success = _checks(ret, success, response,name) else: response = _replication_request('details', core_name=core_name) - ret, success = _checks(ret, success, response, core_name) + ret, success = _checks(ret, success, response,core_name) return ret - def match_index_versions(core_name=None): ''' SLAVE ONLY - Verifies that the master and the slave versions are in sync by - comparing the index version. + Verifies that the master and the slave versions are in sync by + comparing the index version. If you are constantly pushing updates + the index the master and slave versions will seldom match. - core_name : str (None) - The name of the solr core if using cores. Leave this blank if you are - not using cores or if you want to check all cores. + Param: str core_name (None): The name of the solr core if using cores. + Leave this blank if you are not using cores or + if you want to check all cores. + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - Return : dict:: - - {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - - CLI Example:: - - salt '*' solr.match_index_versions music + CLI Example: + salt '*' solr.match_index_versions music ''' - #get the defualt return dict + # since only slaves can call this let's check the config: + if _is_master(): + e = ['solr.match_index_versions can only be called by "slave" minions'] + return ret.update({'success':False, 'errors':e}) + #get the default return dict ret = _get_return_dict() success = True - # since only slaves can call this let's check the config: - if __opts__['solr.type'] != 'slave': - errors = ['Only minions configured as solr slaves can run this'] - return ret.update({'success': False, 'errors': errors}) def _match(ret, success, resp, core): if response['success']: slave = resp['data']['details']['slave'] master_url = resp['data']['details']['slave']['masterUrl'] - if 'ERROR' in slave: + if slave.has_key('ERROR'): error = slave['ERROR'] - success = False + success=False err = "{0}: {1} - {2}".format(name, error, master_url) resp['errors'].append(err) - #if there was an error return the entire response so the + #if there was an error return the entire response so the #alterer can get what it wants - data = slave if core is None else {core: {'data': slave}} + data = slave if core is None else {core : {'data': slave}} else: - versions = {'master': slave['masterDetails']['indexVersion'], - 'slave': resp['data']['details']['indexVersion'], - 'next_replication': slave['nextExecutionAt'], - 'failed_list': slave['replicationFailedAtList'] + versions = {'master':slave['masterDetails']['indexVersion'], + 'slave' : resp['data']['details']['indexVersion'], + 'next_replication' : slave['nextExecutionAt'], + 'failed_list' : slave['replicationFailedAtList'] } - #check the index versions + #check the index versions if index_versions['master'] != index_versions['slave']: success = False err = "Master and Slave index versions do not match." resp['errors'].append(err) - data = versions if core is None else {core: {'data': versions}} - ret = _update_return_dict(ret, success, data, + data = versions if core is None else {core:{'data':versions}} + ret = _update_return_dict(ret, success, data, resp['errors'], resp['warnings']) return (ret, success) - - #check all cores? - if core_name is None and __check_for_cores__(): + + #check all cores? + if core_name is None and _check_for_cores(): success = True for name in __opts__['solr.cores']: response = _replication_request('details', core_name=name) - ret, success = _match(ret, success, response, name) + ret, success = _match(ret, success, response, name) else: response = _replication_request('details', core_name=core_name) - ret, success = _match(ret, success, response, core_name) + ret, success = _match(ret , success, response, core_name) return ret - def replication_details(core_name=None): ''' Get the full replication details. - core_name : str (None) - The name of the solr core if using cores. Leave this blank if you are - not using cores or if you want to check all cores. + Param: str core_name (None): The name of the solr core if using cores. + Leave this blank if you are not using cores or + if you want to check all cores. + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - Return : dict:: - - {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - - CLI Example:: - - salt '*' solr.replication_details music + CLI Example: + salt '*' solr.replication_details music ''' ret = _get_return_dict() if core_name is None: - success = True + success=True for name in __opts__['solr.cores']: resp = _replication_request('details', core_name=name) - data = {name: {'data': resp['data']}} - ret = _update_return_dict(ret, success, data, + data = {name : {'data':resp['data']}} + ret = _update_return_dict(ret, success, data, resp['errors'], resp['warnings']) else: resp = _replication_request('details', core_name=core_name) if resp['success']: - ret = _update_return_dict(ret, success, resp['data'], - resp['errors'], resp['warnings']) + ret = _update_return_dict(ret, success, resp['data'], + resp['errors'], resp['warnings']) else: return resp return ret - - -def backup_master(core_name=None, path=None): + +def backup(core_name=None, append_core_to_path=False): ''' - Tell the master to make a backup. If you don't pass a core name and you are - using cores it will backup all cores and append the name of the core to the - backup path. + Tell solr make a backup. This method can be mis-leading since it uses + the backup api. If an error happens during the backup you are not + notified. The status: 'OK' in the response simply means that solr received + the request successfully. - core_name : str (None) - The name of the solr core if using cores. Leave this blank if you are - not using cores or if you want to check all cores. + Param: str core_name (None): The name of the solr core if using cores. + Leave this blank if you are not using cores or + if you want to check all cores. + Param: str append_core_to_path (False): If True add the name of the core + to the backup path. Assumes that + minion backup path is not None. + + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - path : str (/srv/media/solr/backup) - The base backup path. - DO NOT INCLUDE THE CORE NAME! - If the core name is specified or if you are using cores and leave - core_name blank the name of the core will be appened to it. - - Return : dict:: - - {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - - CLI Example:: - - salt '*' solr.backup_master music + CLI Example: + salt '*' solr.backup music ''' - if path is None: - path = "/srv/media/solr/backup{0}" - else: - path = path + "{0}" + path = __opts__['solr.backup_path'] + print path + numBackups = __opts__['solr.num_backups'] + print numBackups + if path is not None: + if not path.endswith(os.path.sep): + path += os.path.sep + ret = _get_return_dict() - if core_name is None and __check_for_cores__(): - success = True + if core_name is None and _check_for_cores(): + success=True for name in __opts__['solr.cores']: - extra = "&location={0}".format(path + name) - resp = _replication_request('backup', core_name=name, extra=extra) + params = [] + if path is not None: + path = path + name if append_core_to_path else path + params.append("&location={0}".format(path + name)) + params.append("&numberToKeep={0}".format(numBackups)) + resp = _replication_request('backup', core_name=name, params=params) if not resp['success']: - success = False - data = {name: {'data': resp['data']}} - ret = _update_return_dict(ret, success, data, + success=False + data = {name : {'data': resp['data']}} + ret = _update_return_dict(ret, success, data, resp['errors'], resp['warnings']) return ret else: - if core_name is None: - path = path.format("") - else: - path = path.format("/{0}".format(core_name)) - params = ["location={0}".format(path)] - resp = _replication_request('backup', core_name=core_name, - params=params) + if core_name is not None and path is not None: + if append_core_to_path: + path += core_name + if path is not None: + params = ["location={0}".format(path)] + params.append("&numberToKeep={0}".format(numBackups)) + resp = _replication_request('backup',core_name=core_name,params=params) return resp - def set_is_polling(polling, core_name=None): ''' SLAVE ONLY Prevent the slaves from polling the master for updates. - - polling : bool - True will enable polling. False will disable it. - - core_name : str (None) - The name of the solr core if using cores. Leave this blank if you are - not using cores or if you want to check all cores. - - Return : dict:: - - {'success':bool, 'data':dict, 'errors':list, 'warnings':list} - - CLI Example:: - - salt '*' solr.set_is_polling False + + Param: bool polling: True will enable polling. False will disable it. + Param: str core_name (None): The name of the solr core if using cores. + Leave this blank if you are not using cores or + if you want to check all cores. + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} + + CLI Example: + salt '*' solr.set_is_polling False ''' - ret = _get_return_dict() + ret = _get_return_dict() # since only slaves can call this let's check the config: - if __opts__['solr.type'] != 'slave': - err = ['Only minions configured as solr slaves can run this'] - return ret.update({'success': False, 'errors': err}) + if _is_master(): + err = ['solr.set_is_polling can only be called by "slave" minions'] + return ret.update({'success':False, 'errors':err}) cmd = "enablepoll" if polling else "disapblepoll" - if core_name is None and __check_for_cores__(): - success = True + if core_name is None and _check_for_cores(): + success=True for name in __opts__['solr.cores']: - resp = _replication_request(cmd, core_name=name) + resp = set_is_polling(cmd, core_name=name) if not resp['success']: success = False - data = {name: {'data': resp['data']}} - ret = _update_return_dict(ret, success, data, + data = {name : {'data' : resp['data']}} + ret = _update_return_dict(ret, success, data, resp['errors'], resp['warnings']) return ret else: resp = _replication_request(cmd, core_name=name) return resp +def set_replication_enabled(status, core_name): + ''' + MASTER ONLY + Sets the master to ignore poll requests from the slaves. Useful when + you don't want the slaves replicating during indexing or when clearing + the index. + + Param bool status: Sets the replication status to the specified state. + Param str core_name (None): The name of the solr core if using cores. + Leave this blank if you are not using cores or + if you want to set the status on all cores. + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} + ''' + if __opts__['solr.type'] != 'master': + return _get_return_dict(False, + errors=['Only minions configured as master can run this']) + cmd = 'enablereplication' if status else 'disablereplication' + if core_name is None and _check_for_cores(): + ret = _get_return_dict() + success=True + for name in __opts__['solr.cores']: + resp = set_replication_enabled(status, name) + if not resp['success']: + success = False + data = {name : {'data' : resp['data']}} + ret = _update_return_dict(ret, success, data, + resp['errors'], resp['warnings']) + return ret + else: + if status: + return _replication_request(cmd, core_name=core_name) + else: + return _replication_request(cmd, core_name=core_name) def signal(signal=None): ''' - Signals Apache Solr to start, stop, or restart. Obvioulsy this is only - going to work if the minion resides on the solr host. Additionally + Signals Apache Solr to start, stop, or restart. Obviously this is only + going to work if the minion resides on the solr host. Additionally Solr doesn't ship with an init script so one must be created. - signal : str (None) - The command to pass to the apache solr init valid values are 'start', - 'stop', and 'restart' + Param: str signal (None): The command to pass to the apache solr init + valid values are 'start', 'stop', and 'restart' - CLI Example:: - - salt '*' solr.signal restart + CLI Example: + salt '*' solr.signal restart ''' - + ret = _get_return_dict() valid_signals = 'start stop restart' if not valid_signals.count(signal): return - + cmd = "{0} {1}".format(__opts__['solr.init_script'], signal) out = __salt__['cmd.run'](cmd) + +def reload_core(core_name): + ''' + MULTI-CORE HOSTS ONLY + Load a new core from the same configuration as an existing registered core. + While the "new" core is initializing, the "old" one will continue to + accept requests. Once it has finished, all new request will go to the + "new" core, and the "old" core will be unloaded. + + Param: str core_name: The name of the core to reload + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} + ''' + extra = ['action=RELOAD', 'core={0}'.format(core_name)] + url = _format_url('admin/cores', None, extra=extra) + return _http_request(url) + +def core_status(core_name): + ''' + MULTI-CORE HOSTS ONLY + Get the status for a given core or all cores if no core is specified + + Param: str core_name: The name of the core to reload + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} + ''' + extra = ['action=STATUS', 'core={0}'.format(core_name)] + url = _format_url('admin/cores', None, extra=extra) + print url + return _http_request(url) + +################### DIH (Direct Import Handler) COMMANDS ##################### + +def reload_import_config(handler, core_name=None, verbose=False): + ''' + MASTER ONLY + re-loads the handler config XML file. + This command can only be run if the minion is a 'master' type + + Param: str handler: The name of the data import handler. + Param: str core (None): The core the handler belongs to. + Param: bool verbose (False): Run the command with verbose output. + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} + + CLI Example: + salt '*' solr.reload_import_config dataimport music {'clean':True} + ''' + + #make sure that it's a master minion + if not _is_master(): + err = ['solr.pre_indexing_check can only be called by "master" minions'] + return _get_return_dict(False, err) + + params = ['command=reload-config'] + if verbose: + params.append("verbose=true") + url = _format_url(handler,core_name=core_name,extra=params) + return _http_request(url) + +def abort_import(handler, core_name=None, verbose=False): + ''' + MASTER ONLY + Aborts an existing import command to the specified handler. + This command can only be run if the minion is is configured with + solr.type=master + + Param: str handler: The name of the data import handler. + Param: str core (None): The core the handler belongs to. + Param: bool verbose (False): Run the command with verbose output. + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} + + CLI Example: + salt '*' solr.abort_import dataimport music {'clean':True} + ''' + if not _is_master(): + err = ['solr.abort_import can only be called on "master" minions'] + return _get_return_dict(False, errors=err) + + params = ['command=abort'] + if verbose: + params.append("verbose=true") + url = _format_url(handler,core_name=core_name,extra=params) + return _http_request(url) + +def full_import(handler, core_name=None, options={}, extra=[]): + ''' + MASTER ONLY + Submits an import command to the specified handler using specified options. + This command can only be run if the minion is is configured with + solr.type=master + + Param: str handler: The name of the data import handler. + Param: str core (None): The core the handler belongs to. + Param: dict options (__opts__): A list of options such as clean, optimize + commit, verbose, and pause_replication. + leave blank to use __opts__ defaults. + options will be merged with __opts__ + Param: dict extra ([]): Extra name value pairs to pass to the handler. + e.g. ["name=value"] + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} + + CLI Example: + salt '*' solr.full_import dataimport music {'clean':True} + ''' + if not _is_master(): + err = ['solr.full_import can only be called on "master" minions'] + return _get_return_dict(False, errors=err) + + resp = _pre_index_check(handler, core_name) + if not resp['success']: + return resp + options = _merge_options(options) + if options['clean']: + resp = set_replication_enabled(False, core_name) + if not resp['success']: + errors = ['Failed to set the replication status on the master.'] + return _get_return_dict(False, errors=errors) + params = ['command=full-import'] + for (k,v) in options.items(): + params.append("&{0}={1}".format(k,v)) + url = _format_url(handler,core_name=core_name,extra=params + extra) + return _http_request(url) + +def delta_import(handler, core_name=None, options={}, extra=[]): + ''' + Submits an import command to the specified handler using specified options. + This command can only be run if the minion is is configured with + solr.type=master + + Param: str handler: The name of the data import handler. + Param: str core (None): The core the handler belongs to. + Param: dict options (__opts__): A list of options such as clean, optimize + commit, verbose, and pause_replication. + leave blank to use __opts__ defaults. + options will be merged with __opts__ + Param dict extra ([]): Extra name value pairs to pass to the handler. + eg ["name=value"] + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} + + CLI Example: + salt '*' solr.delta_import dataimport music {'clean':True} + ''' + if not _is_master(): + err = ['solr.delta_import can only be called on "master" minions'] + return _get_return_dict(False, errors=err) + + resp = _pre_index_check(handler, core_name) + if not resp['success']: + return resp + options = _merge_options(options) + if options['clean']: + resp = set_replication_enabled(False, core_name) + if not resp['success']: + errors = ['Failed to set the replication status on the master.'] + return _get_return_dict(False, errors=errors) + params = ['command=delta-import'] + for (k,v) in options.items(): + params.append("{0}={1}".format(k,v)) + url = _format_url(handler, core_name=core_name, extra=params + extra) + return _http_request(url) + +def import_status(handler, core_name=None, verbose=False): + ''' + Submits an import command to the specified handler using specified options. + This command can only be run if the minion is is configured with + solr.type: 'master' + + Param: str handler: The name of the data import handler. + Param: str core (None): The core the handler belongs to. + Param: bool verbose (False): Specifies verbose output + Return: dict {'success':bool, 'data':dict, 'errors':list, 'warnings':list} + + CLI Example + salt '*' solr.import_status dataimport music False + ''' + if not _is_master(): + errors = ['solr.import_status can only be called by "master" minions'] + return _get_return_dict(False, errors=errors) + + extra = ["command=status"] + if verbose: + extra.append("verbose=true") + url = _format_url(handler,core_name=core_name,extra=extra) + return _http_request(url) From 23f7a018ca316cd912d98e13a613d1b7ad53c575 Mon Sep 17 00:00:00 2001 From: Eric Poelke Date: Wed, 23 Nov 2011 16:21:00 -0800 Subject: [PATCH 14/18] merge --- salt/modules/yumpkg.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 7e0a1063e5..fbe9fe3c40 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -164,9 +164,6 @@ def install(pkgs, refresh=False): salt '*' pkg.install ''' - # If the kwarg refresh exists get it, otherwise set it to False - #refresh = kwargs.get('refresh', False) - if refresh: refresh_db() From 8ba76c7e8d5dc0d2e71dd9923fda8ea515d176cc Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 24 Nov 2011 00:07:55 -0700 Subject: [PATCH 15/18] Add support for failing hard --- salt/state.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/salt/state.py b/salt/state.py index 29889360e4..77b9c2308e 100644 --- a/salt/state.py +++ b/salt/state.py @@ -409,11 +409,26 @@ class State(object): ''' running = {} for low in chunks: - if (low['state'] + '.' + low['name'] + - '.' + low['fun'] not in running): + if '__FAILHARD__' in running: + running.pop('__FAILHARD__') + return running + tag = '{0[state]}.{0[name]}.{0[fun]}'.format(low) + if tag not in running: running = self.call_chunk(low, running, chunks) + if self.check_failhard(low, running): + return running return running + def check_failhard(self, low, running): + ''' + Check if the low data chunk should send a failhard signal + ''' + tag = '{0[state]}.{0[name]}.{0[fun]}'.format(low) + if low.get('failhard', False) and tag in running: + if not running[tag]['result']: + return True + return False + def check_requires(self, low, running, chunks): ''' Look into the running data to see if the requirement has been met @@ -430,7 +445,7 @@ class State(object): reqs.append(chunk) fun_stats = [] for req in reqs: - tag = req['state'] + '.' + req['name'] + '.' + req['fun'] + tag = '{0[state]}.{0[name]}.{0[fun]}'.format(req) if tag not in running: fun_stats.append('unmet') else: @@ -458,7 +473,7 @@ class State(object): reqs.append(chunk) fun_stats = [] for req in reqs: - tag = req['state'] + '.' + req['name'] + '.' + req['fun'] + tag = '{0[state]}.{0[name]}.{0[fun]}'.format(req) if tag not in running: fun_stats.append('unmet') else: @@ -476,7 +491,7 @@ class State(object): Check if a chunk has any requires, execute the requires and then the chunk ''' - tag = low['state'] + '.' + low['name'] + '.' + low['fun'] + tag = '{0[state]}.{0[name]}.{0[fun]}'.format(low) if 'require' in low: status = self.check_requires(low, running, chunks) if status == 'unmet': @@ -493,7 +508,13 @@ class State(object): if (chunk['state'] + '.' + chunk['name'] + '.' + chunk['fun'] not in running): running = self.call_chunk(chunk, running, chunks) + if self.check_failhard(chunk, running): + running['__FAILHARD__'] = True + return running running = self.call_chunk(low, running, chunks) + if self.check_failhard(chunk, running): + running['__FAILHARD__'] = True + return running elif status == 'met': running[tag] = self.call(low) elif status == 'fail': @@ -516,7 +537,13 @@ class State(object): if (chunk['state'] + '.' + chunk['name'] + '.' + chunk['fun'] not in running): running = self.call_chunk(chunk, running, chunks) + if self.check_failhard(chunk, running): + running['__FAILHARD__'] = True + return running running = self.call_chunk(low, running, chunks) + if self.check_failhard(chunk, running): + running['__FAILHARD__'] = True + return running elif status == 'nochange': running[tag] = self.call(low) elif status == 'change': From 32fd2028a46c9587999f5066fe9b92ec954dfa4c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 24 Nov 2011 00:22:31 -0700 Subject: [PATCH 16/18] add failhard to the master config file --- conf/master | 4 ++++ salt/config.py | 2 ++ salt/state.py | 5 ++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/conf/master b/conf/master index 0c29f99e4c..830a310ce7 100644 --- a/conf/master +++ b/conf/master @@ -50,6 +50,10 @@ # # The renderer to use on the minions to render the state data #renderer: yaml_jinja +# +# The failhard option tells the minions to stop immediately after the first +# failure detected in the state execution, defaults to False +#failhard: False ##### File Server settings ##### ########################################## diff --git a/salt/config.py b/salt/config.py index d7c6ac8c01..d0be9411b9 100644 --- a/salt/config.py +++ b/salt/config.py @@ -61,6 +61,7 @@ def minion_config(path): 'cachedir': '/var/cache/salt', 'conf_file': path, 'renderer': 'yaml_jinja', + 'failhard': False, 'disable_modules': [], 'disable_returners': [], 'module_dirs': [], @@ -122,6 +123,7 @@ def master_config(path): 'open_mode': False, 'auto_accept': False, 'renderer': 'yaml_jinja', + 'failhard': False, 'state_top': 'top.sls', 'order_masters': False, 'log_file': '/var/log/salt/master', diff --git a/salt/state.py b/salt/state.py index 77b9c2308e..a4fa1acc34 100644 --- a/salt/state.py +++ b/salt/state.py @@ -424,7 +424,9 @@ class State(object): Check if the low data chunk should send a failhard signal ''' tag = '{0[state]}.{0[name]}.{0[fun]}'.format(low) - if low.get('failhard', False) and tag in running: + if low.get('failhard', False) \ + or self.opts['failhard'] \ + and tag in running: if not running[tag]['result']: return True return False @@ -623,6 +625,7 @@ class HighState(object): return opts mopts = self.client.master_opts() opts['renderer'] = mopts['renderer'] + opts['failhard'] = mopts['failhard'] if mopts['state_top'].startswith('salt://'): opts['state_top'] = mopts['state_top'] elif mopts['state_top'].startswith('/'): From e2617bb51d5900757074ea0e4d5fb1e3182504a5 Mon Sep 17 00:00:00 2001 From: Tor Hveem Date: Thu, 24 Nov 2011 15:37:47 +0100 Subject: [PATCH 17/18] Fix bug in /proc/net/dev parsing --- salt/modules/status.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/modules/status.py b/salt/modules/status.py index 0d39a3e459..636fcb6641 100644 --- a/salt/modules/status.py +++ b/salt/modules/status.py @@ -317,6 +317,10 @@ def netdev(): if line.find(':') < 0: continue comps = line.split() + # Fix lines like eth0:9999..' + comps[0] = line.split(':')[0].strip() + #Support lines both like eth0:999 and eth0: 9999 + comps[1] = line.split(':')[1].strip().split()[0] ret[comps[0]] = {'iface': comps[0], 'rx_bytes': _number(comps[1]), 'rx_compressed': _number(comps[7]), From 1fa51ff2008ccf1f9766b8b05f0ec313226d0ee2 Mon Sep 17 00:00:00 2001 From: Markus Gattol Date: Thu, 24 Nov 2011 18:37:58 +0000 Subject: [PATCH 18/18] using gitflow is optional Signed-off-by: Markus Gattol --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 8ee83cbb8a..53098394ad 100644 --- a/README.rst +++ b/README.rst @@ -110,3 +110,8 @@ repeat... * pep8, pylint, pychecker * commit, push * pull request + +Following the gitflow branching model is optional i.e. if you prefer a +different branching model then that's fine. All that setuprepo.sh does +is provide you with a quick and hassle-free way to setup the same +branching model that's used for the main repository.