From ff8d8df1d29e81efde6aeb27297f2a79a8625ec8 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 12 Aug 2014 16:25:43 -0600 Subject: [PATCH 01/17] Add additional example for datetime YAML idiosyncrasy Fixes #14860 --- .../troubleshooting/yaml_idiosyncrasies.rst | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst index fcbc70aef0..dfbd2e87db 100644 --- a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -311,3 +311,32 @@ strings to force YAML to serialize them as strings: datetime.datetime(2014, 1, 20, 14, 23, 23) >>> yaml.safe_load('"2014-01-20 14:23:23"') '2014-01-20 14:23:23' + +Additionally, numbers formatted like ``XXXX-XX-XX`` will also be converted (or +YAML will attempt to convert them, and error out if it doesn't think the date +is a real one). Thus, for example, if you were to have a minion ID of +``4017-16-20`` the minion would not start because YAML would complain that the +date was out of range. The workaround is the same, surround the offending +string with quotes: + +.. code-block:: python + + >>> import yaml + >>> yaml.safe_load('4017-16-20') + Traceback (most recent call last): + File "", line 1, in + File "/usr/local/lib/python2.7/site-packages/yaml/__init__.py", line 93, in safe_load + return load(stream, SafeLoader) + File "/usr/local/lib/python2.7/site-packages/yaml/__init__.py", line 71, in load + return loader.get_single_data() + File "/usr/local/lib/python2.7/site-packages/yaml/constructor.py", line 39, in get_single_data + return self.construct_document(node) + File "/usr/local/lib/python2.7/site-packages/yaml/constructor.py", line 43, in construct_document + data = self.construct_object(node) + File "/usr/local/lib/python2.7/site-packages/yaml/constructor.py", line 88, in construct_object + data = constructor(self, node) + File "/usr/local/lib/python2.7/site-packages/yaml/constructor.py", line 312, in construct_yaml_timestamp + return datetime.date(year, month, day) + ValueError: month must be in 1..12 + >>> yaml.safe_load('"4017-16-20"') + '4017-16-20' From 92742f7e08bc052ddd3e2a2627849a7af26c4f02 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Wed, 13 Aug 2014 17:33:06 -0600 Subject: [PATCH 02/17] No second person --- doc/topics/troubleshooting/yaml_idiosyncrasies.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst index dfbd2e87db..a1ae67f9b7 100644 --- a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -314,7 +314,7 @@ strings to force YAML to serialize them as strings: Additionally, numbers formatted like ``XXXX-XX-XX`` will also be converted (or YAML will attempt to convert them, and error out if it doesn't think the date -is a real one). Thus, for example, if you were to have a minion ID of +is a real one). Thus, for example, if a minion were to have an ID of ``4017-16-20`` the minion would not start because YAML would complain that the date was out of range. The workaround is the same, surround the offending string with quotes: From 53c1cd432b6bf718184a1bdbceb371f3755147a4 Mon Sep 17 00:00:00 2001 From: Nahum Shalman Date: Thu, 14 Aug 2014 15:42:15 -0400 Subject: [PATCH 03/17] lack of ssh_py_shim.py breaks esky builds use the .pyc file instead --- salt/client/ssh/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index 006be54691..609546044e 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -151,7 +151,11 @@ EOF'''.format( ) if not is_windows(): - with open(os.path.join(os.path.dirname(__file__), 'ssh_py_shim.py')) as ssh_py_shim: + shim_file = os.path.join(os.path.dirname(__file__), 'ssh_py_shim.py') + if not os.path.exists(shim_file): + # On esky builds we only have the .pyc file + shim_file += "c" + with open(shim_file) as ssh_py_shim: SSH_PY_SHIM = ''.join(ssh_py_shim.readlines()).encode('base64') log = logging.getLogger(__name__) From 2cd229d32b791cd631021c502009e6c0e4c4ebdd Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 14 Aug 2014 13:43:29 -0600 Subject: [PATCH 04/17] Change self.wheelclient.w_funcs to self.wheelclient.functions Fixes #14591 --- salt/client/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/client/api.py b/salt/client/api.py index 69e62615a7..77020a3bcc 100644 --- a/salt/client/api.py +++ b/salt/client/api.py @@ -223,7 +223,7 @@ class APIClient(object): client = parts[0] module = '.'.join(parts[1:]) # strip prefix if client == 'wheel': - functions = self.wheelClient.w_funcs + functions = self.wheelClient.functions elif client == 'runner': functions = self.runnerClient.functions result = {'master': salt.utils.argspec_report(functions, module)} From 63ddb3500f3516a28002c24c6b39eb2dce78193f Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 14 Aug 2014 13:45:28 -0600 Subject: [PATCH 05/17] Do not use non_blocking mode on master event bus In my testing, using the NOBLOCK option in ZeroMQ resulted in an approx. mesasge loss of between 0.5% and 1.5% across 1M messages delivered at high-speed. Removing that option resulted in a slight performance hit but 100% of messages were delivered. --- salt/utils/event.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/salt/utils/event.py b/salt/utils/event.py index de2734e613..b6ca4658c6 100644 --- a/salt/utils/event.py +++ b/salt/utils/event.py @@ -290,7 +290,8 @@ class SaltEvent(object): continue try: - ret = self.get_event_noblock() + ret = self.get_event_block() # Please do not use non-blocking mode here. + # Reliability is more important than pure speed on the event bus. except zmq.ZMQError as ex: if ex.errno == errno.EAGAIN or ex.errno == errno.EINTR: continue @@ -353,6 +354,14 @@ class SaltEvent(object): mtag, data = self.unpack(raw, self.serial) return {'data': data, 'tag': mtag} + def get_event_block(self): + '''Get the raw event in a blocking fashion + Slower, but decreases the possibility of dropped events + ''' + raw = self.sub.recv() + mtag, data = self.unpack(raw, self.serial) + return {'data': data, 'tag': mtag} + def iter_events(self, tag='', full=False): ''' Creates a generator that continuously listens for events From f3b5c088ef3594cce4005f595fe6fbeae62c21f5 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 14 Aug 2014 16:00:45 -0600 Subject: [PATCH 06/17] Add checks to see if psutil version is new enough --- salt/modules/ps.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/salt/modules/ps.py b/salt/modules/ps.py index ca50f07763..3d16624c55 100644 --- a/salt/modules/ps.py +++ b/salt/modules/ps.py @@ -12,7 +12,7 @@ import time import datetime # Import salt libs -from salt.exceptions import SaltInvocationError +from salt.exceptions import SaltInvocationError, CommandExecutionError # Import third party libs try: @@ -347,12 +347,19 @@ def virtual_memory(): Return a dict that describes statistics about system memory usage. + .. note:: + + This function is only available in psutil version 0.6.0 and above. + CLI Example: .. code-block:: bash salt '*' ps.virtual_memory ''' + if psutil.version_info < (0, 6, 0): + msg = 'virtual_memory is only available in psutil 0.6.0 or greater' + raise CommandExecutionError(msg) return dict(psutil.virtual_memory()._asdict()) @@ -362,12 +369,19 @@ def swap_memory(): Return a dict that describes swap memory statistics. + .. note:: + + This function is only available in psutil version 0.6.0 and above. + CLI Example: .. code-block:: bash salt '*' ps.swap_memory ''' + if psutil.version_info < (0, 6, 0): + msg = 'swap_memory is only available in psutil 0.6.0 or greater' + raise CommandExecutionError(msg) return dict(psutil.swap_memory()._asdict()) From 8dca2515eef142e286438ef6c6fc7a9f4ee1f90c Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 14 Aug 2014 16:16:38 -0600 Subject: [PATCH 07/17] Skip virtual_memory and swap_memory tests if psutil < 0.6.0 --- tests/unit/modules/ps_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/modules/ps_test.py b/tests/unit/modules/ps_test.py index ebadd4c854..c863aec602 100644 --- a/tests/unit/modules/ps_test.py +++ b/tests/unit/modules/ps_test.py @@ -56,6 +56,10 @@ try: except ImportError: HAS_UTMP = False +HAS_PSUTIL_VERSION = False +if psutil.version_info >= (0, 6, 0): + HAS_PSUTIL_VERSION = True + def _get_proc_name(proc): return proc.name() if PSUTIL2 else proc.name @@ -104,11 +108,13 @@ class PsTestCase(TestCase): def test_cpu_times(self): self.assertDictEqual({'idle': 4, 'nice': 2, 'system': 3, 'user': 1}, ps.cpu_times()) + @skipIf(HAS_PSUTIL_VERSION is False, 'psutil 0.6.0 or greater is required for this test') @patch('psutil.virtual_memory', new=MagicMock(return_value=STUB_VIRT_MEM)) def test_virtual_memory(self): self.assertDictEqual({'used': 500, 'total': 1000, 'available': 500, 'percent': 50, 'free': 500}, ps.virtual_memory()) + @skipIf(HAS_PSUTIL_VERSION is False, 'psutil 0.6.0 or greater is required for this test') @patch('psutil.swap_memory', new=MagicMock(return_value=STUB_SWAP_MEM)) def test_swap_memory(self): self.assertDictEqual({'used': 500, 'total': 1000, 'percent': 50, 'free': 500, 'sin': 0, 'sout': 0}, From e9318e5f6c4fa1bd86b16f156c19338bb210a240 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Thu, 14 Aug 2014 16:48:36 -0600 Subject: [PATCH 08/17] use unicode strings Fixes #14695 --- salt/output/key.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/output/key.py b/salt/output/key.py index 910992e776..a77c559e74 100644 --- a/salt/output/key.py +++ b/salt/output/key.py @@ -32,34 +32,34 @@ def output(data): rej: color['BLUE'], 'local': color['PURPLE']} - trans = {pend: '{0}Unaccepted Keys:{1}'.format( + trans = {pend: u'{0}Unaccepted Keys:{1}'.format( color['LIGHT_RED'], color['ENDC']), - acc: '{0}Accepted Keys:{1}'.format( + acc: u'{0}Accepted Keys:{1}'.format( color['LIGHT_GREEN'], color['ENDC']), - rej: '{0}Rejected Keys:{1}'.format( + rej: u'{0}Rejected Keys:{1}'.format( color['LIGHT_BLUE'], color['ENDC']), - 'local': '{0}Local Keys:{1}'.format( + 'local': u'{0}Local Keys:{1}'.format( color['LIGHT_PURPLE'], color['ENDC'])} ret = '' for status in sorted(data): - ret += '{0}\n'.format(trans[status]) + ret += u'{0}\n'.format(trans[status]) for key in data[status]: skey = key if strip_colors: skey = salt.output.strip_esc_sequence(key) if isinstance(data[status], list): - ret += '{0}{1}{2}\n'.format( + ret += u'{0}{1}{2}\n'.format( cmap[status], skey, color['ENDC']) if isinstance(data[status], dict): - ret += '{0}{1}: {2}{3}\n'.format( + ret += u'{0}{1}: {2}{3}\n'.format( cmap[status], skey, data[status][key], From 6589d4dd1bdb30ef33ea918071f4f7f906f03b25 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 14 Aug 2014 17:13:15 -0600 Subject: [PATCH 09/17] Optimize auth We were traversing the entire minion data cache on every auth unnecessarily. This should speed auths up substantially. --- salt/master.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/master.py b/salt/master.py index 31b4edd437..9224c42ba4 100644 --- a/salt/master.py +++ b/salt/master.py @@ -1317,10 +1317,9 @@ class ClearFuncs(object): 'load': {'ret': False}} log.info('Authentication request from {id}'.format(**load)) - minions = salt.utils.minions.CkMinions(self.opts).connected_ids() - # 0 is default which should be 'unlimited' if self.opts['max_minions'] > 0: + minions = salt.utils.minions.CkMinions(self.opts).connected_ids() if not len(minions) < self.opts['max_minions']: # we reject new minions, minions that are already # connected must be allowed for the mine, highstate, etc. From 0683c704ddfe98197d30c1e0bc167700f6513ec2 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 14 Aug 2014 21:20:25 -0600 Subject: [PATCH 10/17] Move psutil version check --- tests/unit/modules/ps_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/modules/ps_test.py b/tests/unit/modules/ps_test.py index c863aec602..b34ac3c9ad 100644 --- a/tests/unit/modules/ps_test.py +++ b/tests/unit/modules/ps_test.py @@ -35,6 +35,10 @@ if HAS_PSUTIL: 1000, 2000, 500, 600, 2000, 3000) STUB_USER = psutil._compat.namedtuple('user', 'name, terminal, host, started')('bdobbs', 'ttys000', 'localhost', 0.0) + HAS_PSUTIL_VERSION = False + if psutil.version_info >= (0, 6, 0): + HAS_PSUTIL_VERSION = True + else: (STUB_CPU_TIMES, STUB_VIRT_MEM, @@ -56,10 +60,6 @@ try: except ImportError: HAS_UTMP = False -HAS_PSUTIL_VERSION = False -if psutil.version_info >= (0, 6, 0): - HAS_PSUTIL_VERSION = True - def _get_proc_name(proc): return proc.name() if PSUTIL2 else proc.name From e8fab3d28ac76c21c96e0d895973534a366fa995 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 14 Aug 2014 21:22:55 -0600 Subject: [PATCH 11/17] Fix psutil version --- tests/unit/modules/ps_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/modules/ps_test.py b/tests/unit/modules/ps_test.py index b34ac3c9ad..9e0aa6b777 100644 --- a/tests/unit/modules/ps_test.py +++ b/tests/unit/modules/ps_test.py @@ -13,6 +13,7 @@ ensure_in_syspath('../../') from salt.modules import ps HAS_PSUTIL = ps.__virtual__() +HAS_PSUTIL_VERSION = False if HAS_PSUTIL: import psutil @@ -35,7 +36,6 @@ if HAS_PSUTIL: 1000, 2000, 500, 600, 2000, 3000) STUB_USER = psutil._compat.namedtuple('user', 'name, terminal, host, started')('bdobbs', 'ttys000', 'localhost', 0.0) - HAS_PSUTIL_VERSION = False if psutil.version_info >= (0, 6, 0): HAS_PSUTIL_VERSION = True From 12e87f3545bb36318684f18b6b3bd300d8498db8 Mon Sep 17 00:00:00 2001 From: Seth House Date: Thu, 14 Aug 2014 20:01:17 -0400 Subject: [PATCH 12/17] Don't document the state.event function aliases --- doc/ref/states/all/salt.states.event.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/ref/states/all/salt.states.event.rst b/doc/ref/states/all/salt.states.event.rst index 8d52394e31..18f7f997bf 100644 --- a/doc/ref/states/all/salt.states.event.rst +++ b/doc/ref/states/all/salt.states.event.rst @@ -3,4 +3,5 @@ salt.states.event ================= .. automodule:: salt.states.event - :members: \ No newline at end of file + :members: + :exclude-members: fire_master, mod_watch From da8fc07a7cd6e63fdb366fed6a1610f4994981df Mon Sep 17 00:00:00 2001 From: Seth House Date: Fri, 15 Aug 2014 08:30:22 -0400 Subject: [PATCH 13/17] Fix incorrect function name in event state module example --- salt/states/event.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/states/event.py b/salt/states/event.py index d20d711974..86dac90ab8 100644 --- a/salt/states/event.py +++ b/salt/states/event.py @@ -26,8 +26,7 @@ def send(name, # ...snip bunch of states above mycompany/mystaterun/status/update: - event: - - fire_master + event.send: - data: status: "Half-way through the state run!" From bcdbaab71690c9217eaea2fbe3cfed6feab74930 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Fri, 15 Aug 2014 14:36:22 +0200 Subject: [PATCH 14/17] return only one url needed for test=True where compare one URL with an array of URLs failed --- salt/modules/zypper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index 6215c950b8..e84008f0e6 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -378,7 +378,7 @@ class _RepoInfo(object): @property def url(self): - return [url.asCompleteString() for url in self.zypp.baseUrls()] + return self.zypp.url().asCompleteString() @url.setter def url(self, value): From 396a4de7c516b79edf04cb1acb35c4266bfa319d Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 15 Aug 2014 11:09:20 -0600 Subject: [PATCH 15/17] Make lxc tests run! --- tests/integration/modules/lxc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/modules/lxc.py b/tests/integration/modules/lxc.py index d08d32d0fe..909fbbd57b 100644 --- a/tests/integration/modules/lxc.py +++ b/tests/integration/modules/lxc.py @@ -14,7 +14,7 @@ import integration import salt.utils -@skipIf(salt.utils.which('lxc') is None, 'Skipping - lxc must be installed.') +@skipIf(salt.utils.which('lxc-start') is None, 'Skipping - lxc must be installed.') class LXCModuleTest(integration.ModuleCase): ''' Test the lxc module From b8db5ff56b6bc2d03f68febf7d85bfe2f682ed68 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 15 Aug 2014 14:21:09 -0600 Subject: [PATCH 16/17] Change lxc skipIf message to be more helpful --- tests/integration/modules/lxc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/modules/lxc.py b/tests/integration/modules/lxc.py index 909fbbd57b..6087efe883 100644 --- a/tests/integration/modules/lxc.py +++ b/tests/integration/modules/lxc.py @@ -14,7 +14,7 @@ import integration import salt.utils -@skipIf(salt.utils.which('lxc-start') is None, 'Skipping - lxc must be installed.') +@skipIf(salt.utils.which('lxc-start') is None, 'LXC is not installed or minimal version not met') class LXCModuleTest(integration.ModuleCase): ''' Test the lxc module From 3bb602104c6d51159a17fb2a67c96f894921c966 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 15 Aug 2014 14:30:44 -0600 Subject: [PATCH 17/17] Fix IPv6 typo --- salt/payload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/payload.py b/salt/payload.py index 9855730405..cc0b5868a2 100644 --- a/salt/payload.py +++ b/salt/payload.py @@ -171,7 +171,7 @@ class SREQ(object): zmq.RECONNECT_IVL_MAX, 5000 ) - if self.master.startswith('tcp://[') and hasattr(zmq, 'IPV4ONLY'): + if self.master.startswith('tcp://') and hasattr(zmq, 'IPV4ONLY'): # IPv6 sockets work for both IPv6 and IPv4 addresses self._socket.setsockopt(zmq.IPV4ONLY, 0) self._socket.linger = self.linger