From 08a5f5a1283c2f5cede201c060616ed2bd916df7 Mon Sep 17 00:00:00 2001 From: obimod Date: Sat, 5 Apr 2014 01:46:26 -0700 Subject: [PATCH 01/55] Prevent a keyerror when cloud mapping removed the requirement that 'minion' must be in nodedata if 'minion' is in overrides. this requirement threw a keyerror on me. --- salt/cloud/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index 2b05422f11..a25c75bdfa 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -1650,7 +1650,7 @@ class Map(Cloud): overrides[setting] = overrides.pop(deprecated) # merge minion grains from map file - if 'minion' in overrides: + if 'minion' in overrides and 'minion' in nodedata: if 'grains' in overrides['minion']: if 'grains' in nodedata['minion']: nodedata['minion']['grains'].update( From 1e83b2012be1335ff06a825e63a6dde5bfb7065b Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 20 Jun 2014 15:29:18 -0600 Subject: [PATCH 02/55] Unit tests for is_provider_configured function in salt.config.py --- tests/unit/config_test.py | 84 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 3 deletions(-) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 425e73d068..8ea68e7212 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -524,7 +524,8 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn): profiles_config_path='foo', profiles_config='bar') @patch('salt.config.load_config', MagicMock(return_value={})) - @patch('salt.config.apply_cloud_config', MagicMock(return_value={'providers': 'foo'})) + @patch('salt.config.apply_cloud_config', + MagicMock(return_value={'providers': 'foo'})) def test_cloud_config_providers_in_opts(self): ''' Tests mixing old cloud providers with pre-configured providers configurations @@ -534,7 +535,8 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn): providers_config='bar') @patch('salt.config.load_config', MagicMock(return_value={})) - @patch('salt.config.apply_cloud_config', MagicMock(return_value={'providers': 'foo'})) + @patch('salt.config.apply_cloud_config', + MagicMock(return_value={'providers': 'foo'})) @patch('os.path.isfile', MagicMock(return_value=True)) def test_cloud_config_providers_in_opts_path(self): ''' @@ -686,7 +688,10 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn): 'provider': 'ec2', 'id': 'ABCDEFGHIJKLMNOP', 'user': 'user@mycorp.com'}}} - self.assertEqual(ret, sconfig.apply_cloud_providers_config(overrides, defaults=DEFAULT)) + self.assertEqual(ret, + sconfig.apply_cloud_providers_config( + overrides, + defaults=DEFAULT)) def test_apply_cloud_providers_config_extend_multiple(self): ''' @@ -821,6 +826,79 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn): overrides, DEFAULT) + def test_is_provider_configured_no_alias(self): + ''' + Tests when provider alias is not in opts + ''' + opts = {'providers': 'test'} + provider = 'foo:bar' + self.assertFalse(sconfig.is_provider_configured(opts, provider)) + + def test_is_provider_configured_no_driver(self): + ''' + Tests when provider driver is not in opts + ''' + opts = {'providers': {'foo': 'baz'}} + provider = 'foo:bar' + self.assertFalse(sconfig.is_provider_configured(opts, provider)) + + def test_is_provider_configured_key_is_none(self): + ''' + Tests when a required configuration key is not set + ''' + opts = {'providers': {'foo': {'bar': {'api_key': None}}}} + provider = 'foo:bar' + self.assertFalse( + sconfig.is_provider_configured(opts, + provider, + required_keys=('api_key',))) + + def test_is_provider_configured_success(self): + ''' + Tests successful cloud provider configuration + ''' + opts = {'providers': {'foo': {'bar': {'api_key': 'baz'}}}} + provider = 'foo:bar' + ret = {'api_key': 'baz'} + self.assertEqual( + sconfig.is_provider_configured(opts, + provider, + required_keys=('api_key',)), ret) + + def test_is_provider_configured_multiple_driver_not_provider(self): + ''' + Tests when the drive is not the same as the provider when + searching through multiple providers + ''' + opts = {'providers': {'foo': {'bar': {'api_key': 'baz'}}}} + provider = 'foo' + self.assertFalse(sconfig.is_provider_configured(opts, provider)) + + def test_is_provider_configured_multiple_key_is_none(self): + ''' + Tests when a required configuration key is not set when + searching through multiple providers + ''' + opts = {'providers': {'foo': {'bar': {'api_key': None}}}} + provider = 'bar' + self.assertFalse( + sconfig.is_provider_configured(opts, + provider, + required_keys=('api_key',))) + + def test_is_provider_configured_multiple_success(self): + ''' + Tests successful cloud provider configuration when searching + through multiple providers + ''' + opts = {'providers': {'foo': {'bar': {'api_key': 'baz'}}}} + provider = 'bar' + ret = {'api_key': 'baz'} + self.assertEqual( + sconfig.is_provider_configured(opts, + provider, + required_keys=('api_key',)), ret) + if __name__ == '__main__': from integration import run_tests run_tests(ConfigTestCase, needs_daemon=False) From d9c929795416c9107455f1648e831842a8273202 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 20 Jun 2014 15:47:00 -0600 Subject: [PATCH 03/55] Make cloud configuration unit tests more organized --- tests/unit/config_test.py | 176 ++++++++++++++++++++------------------ 1 file changed, 94 insertions(+), 82 deletions(-) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 8ea68e7212..29c2c9ea32 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -474,6 +474,10 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn): sconfig.get_id(cache=False), (MOCK_HOSTNAME, False) ) +# <---- Salt Cloud Configuration Tests --------------------------------------------- + + # cloud_config tests + def test_cloud_config_vm_profiles_config(self): ''' Tests passing in vm_config and profiles_config. @@ -546,88 +550,7 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn): self.assertRaises(SaltCloudConfigError, sconfig.cloud_config, PATH, providers_config_path='bar') - def test_load_cloud_config_from_environ_var(self): - original_environ = os.environ.copy() - - tempdir = tempfile.mkdtemp(dir=integration.SYS_TMP_DIR) - try: - env_root_dir = os.path.join(tempdir, 'foo', 'env') - os.makedirs(env_root_dir) - env_fpath = os.path.join(env_root_dir, 'config-env') - - salt.utils.fopen(env_fpath, 'w').write( - 'root_dir: {0}\n' - 'log_file: {1}\n'.format(env_root_dir, env_fpath) - ) - - os.environ['SALT_CLOUD_CONFIG'] = env_fpath - # Should load from env variable, not the default configuration file - config = sconfig.cloud_config('/etc/salt/cloud') - self.assertEqual(config['log_file'], env_fpath) - os.environ.clear() - os.environ.update(original_environ) - - root_dir = os.path.join(tempdir, 'foo', 'bar') - os.makedirs(root_dir) - fpath = os.path.join(root_dir, 'config') - salt.utils.fopen(fpath, 'w').write( - 'root_dir: {0}\n' - 'log_file: {1}\n'.format(root_dir, fpath) - ) - # Let's set the environment variable, yet, since the configuration - # file path is not the default one, ie, the user has passed an - # alternative configuration file form the CLI parser, the - # environment variable will be ignored. - os.environ['SALT_CLOUD_CONFIG'] = env_fpath - config = sconfig.cloud_config(fpath) - self.assertEqual(config['log_file'], fpath) - finally: - # Reset the environ - os.environ.clear() - os.environ.update(original_environ) - - if os.path.isdir(tempdir): - shutil.rmtree(tempdir) - - def test_deploy_search_path_as_string(self): - temp_conf_dir = os.path.join(integration.TMP, 'issue-8863') - config_file_path = os.path.join(temp_conf_dir, 'cloud') - deploy_dir_path = os.path.join(temp_conf_dir, 'test-deploy.d') - try: - for directory in (temp_conf_dir, deploy_dir_path): - if not os.path.isdir(directory): - os.makedirs(directory) - - default_config = sconfig.cloud_config(config_file_path) - default_config['deploy_scripts_search_path'] = deploy_dir_path - with salt.utils.fopen(config_file_path, 'w') as cfd: - cfd.write(yaml.dump(default_config)) - - default_config = sconfig.cloud_config(config_file_path) - - # Our custom deploy scripts path was correctly added to the list - self.assertIn( - deploy_dir_path, - default_config['deploy_scripts_search_path'] - ) - - # And it's even the first occurrence as it should - self.assertEqual( - deploy_dir_path, - default_config['deploy_scripts_search_path'][0] - ) - finally: - if os.path.isdir(temp_conf_dir): - shutil.rmtree(temp_conf_dir) - - def test_includes_load(self): - ''' - Tests that cloud.{providers,profiles}.d directories are loaded, even if not - directly passed in through path - ''' - config = sconfig.cloud_config(self.get_config_file_path('cloud')) - self.assertIn('ec2-config', config['providers']) - self.assertIn('Ubuntu-13.04-AMD64', config['profiles']) + # apply_cloud_providers_config tests def test_apply_cloud_providers_config_same_providers(self): ''' @@ -826,6 +749,8 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn): overrides, DEFAULT) + # is_provider_configured tests + def test_is_provider_configured_no_alias(self): ''' Tests when provider alias is not in opts @@ -899,6 +824,93 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn): provider, required_keys=('api_key',)), ret) + # other cloud configuration tests + + def test_load_cloud_config_from_environ_var(self): + original_environ = os.environ.copy() + + tempdir = tempfile.mkdtemp(dir=integration.SYS_TMP_DIR) + try: + env_root_dir = os.path.join(tempdir, 'foo', 'env') + os.makedirs(env_root_dir) + env_fpath = os.path.join(env_root_dir, 'config-env') + + salt.utils.fopen(env_fpath, 'w').write( + 'root_dir: {0}\n' + 'log_file: {1}\n'.format(env_root_dir, env_fpath) + ) + + os.environ['SALT_CLOUD_CONFIG'] = env_fpath + # Should load from env variable, not the default configuration file + config = sconfig.cloud_config('/etc/salt/cloud') + self.assertEqual(config['log_file'], env_fpath) + os.environ.clear() + os.environ.update(original_environ) + + root_dir = os.path.join(tempdir, 'foo', 'bar') + os.makedirs(root_dir) + fpath = os.path.join(root_dir, 'config') + salt.utils.fopen(fpath, 'w').write( + 'root_dir: {0}\n' + 'log_file: {1}\n'.format(root_dir, fpath) + ) + # Let's set the environment variable, yet, since the configuration + # file path is not the default one, ie, the user has passed an + # alternative configuration file form the CLI parser, the + # environment variable will be ignored. + os.environ['SALT_CLOUD_CONFIG'] = env_fpath + config = sconfig.cloud_config(fpath) + self.assertEqual(config['log_file'], fpath) + finally: + # Reset the environ + os.environ.clear() + os.environ.update(original_environ) + + if os.path.isdir(tempdir): + shutil.rmtree(tempdir) + + def test_deploy_search_path_as_string(self): + temp_conf_dir = os.path.join(integration.TMP, 'issue-8863') + config_file_path = os.path.join(temp_conf_dir, 'cloud') + deploy_dir_path = os.path.join(temp_conf_dir, 'test-deploy.d') + try: + for directory in (temp_conf_dir, deploy_dir_path): + if not os.path.isdir(directory): + os.makedirs(directory) + + default_config = sconfig.cloud_config(config_file_path) + default_config['deploy_scripts_search_path'] = deploy_dir_path + with salt.utils.fopen(config_file_path, 'w') as cfd: + cfd.write(yaml.dump(default_config)) + + default_config = sconfig.cloud_config(config_file_path) + + # Our custom deploy scripts path was correctly added to the list + self.assertIn( + deploy_dir_path, + default_config['deploy_scripts_search_path'] + ) + + # And it's even the first occurrence as it should + self.assertEqual( + deploy_dir_path, + default_config['deploy_scripts_search_path'][0] + ) + finally: + if os.path.isdir(temp_conf_dir): + shutil.rmtree(temp_conf_dir) + + def test_includes_load(self): + ''' + Tests that cloud.{providers,profiles}.d directories are loaded, even if not + directly passed in through path + ''' + config = sconfig.cloud_config(self.get_config_file_path('cloud')) + self.assertIn('ec2-config', config['providers']) + self.assertIn('Ubuntu-13.04-AMD64', config['profiles']) + +# <---- Salt Cloud Configuration Tests --------------------------------------------- + if __name__ == '__main__': from integration import run_tests run_tests(ConfigTestCase, needs_daemon=False) From c721d9d664ca83c9f50e765f45f074ba8d6bb1e2 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 20 Jun 2014 15:10:56 -0700 Subject: [PATCH 04/55] Seperate saltnado_websockets from the rest of the implementation. --- .../netapi/all/salt.netapi.rest_tornado.rst | 2 + salt/netapi/rest_tornado/__init__.py | 28 +- salt/netapi/rest_tornado/event_processor.py | 2 +- salt/netapi/rest_tornado/saltnado.py | 387 +---------------- .../rest_tornado/saltnado_websockets.py | 400 ++++++++++++++++++ 5 files changed, 437 insertions(+), 382 deletions(-) create mode 100644 salt/netapi/rest_tornado/saltnado_websockets.py diff --git a/doc/ref/netapi/all/salt.netapi.rest_tornado.rst b/doc/ref/netapi/all/salt.netapi.rest_tornado.rst index 5ee57bbae1..cd65ea4ff3 100644 --- a/doc/ref/netapi/all/salt.netapi.rest_tornado.rst +++ b/doc/ref/netapi/all/salt.netapi.rest_tornado.rst @@ -4,4 +4,6 @@ rest_tornado .. automodule:: salt.netapi.rest_tornado.saltnado +.. automodule:: salt.netapi.rest_tornado.saltnado_websockets + .. ............................................................................ diff --git a/salt/netapi/rest_tornado/__init__.py b/salt/netapi/rest_tornado/__init__.py index f7d1f55ad9..f7b7813fd3 100644 --- a/salt/netapi/rest_tornado/__init__.py +++ b/salt/netapi/rest_tornado/__init__.py @@ -37,6 +37,9 @@ def start(): mod_opts = __opts__.get(__virtualname__, {}) + if mod_opts.get('websockets', False): + from . import saltnado_websockets + if 'num_processes' not in mod_opts: mod_opts['num_processes'] = 1 @@ -46,7 +49,7 @@ def start(): formatted_events_pattern = r"/formatted_events/{0}".format(token_pattern) logger.debug("All events URL pattern is {0}".format(all_events_pattern)) - application = tornado.web.Application([ + paths = [ (r"/", saltnado.SaltAPIHandler), (r"/login", saltnado.SaltAuthHandler), (r"/minions/(.*)", saltnado.MinionSaltAPIHandler), @@ -56,14 +59,21 @@ def start(): (r"/run", saltnado.RunSaltAPIHandler), (r"/events", saltnado.EventsSaltAPIHandler), (r"/hook(/.*)?", saltnado.WebhookSaltAPIHandler), - # Matches /all_events/[0-9A-Fa-f]{n} - # Where n is the length of hexdigest - # for the current hashing algorithm. - # This algorithm is specified in the - # salt master config file. - (all_events_pattern, saltnado.AllEventsHandler), - (formatted_events_pattern, saltnado.FormattedEventsHandler), - ], debug=mod_opts.get('debug', False)) + ] + + # if you have enabled websockets, add them! + if mod_opts.get('websockets', False): + paths += [ + # Matches /all_events/[0-9A-Fa-f]{n} + # Where n is the length of hexdigest + # for the current hashing algorithm. + # This algorithm is specified in the + # salt master config file. + (all_events_pattern, saltnado_websockets.AllEventsHandler), + (formatted_events_pattern, saltnado_websockets.FormattedEventsHandler), + ] + + application = tornado.web.Application(paths, debug=mod_opts.get('debug', False)) application.opts = __opts__ application.mod_opts = mod_opts diff --git a/salt/netapi/rest_tornado/event_processor.py b/salt/netapi/rest_tornado/event_processor.py index 1f4ee234a8..4e10026fc7 100644 --- a/salt/netapi/rest_tornado/event_processor.py +++ b/salt/netapi/rest_tornado/event_processor.py @@ -1,6 +1,7 @@ # encoding: utf-8 import json import logging +import threading import salt.netapi @@ -201,7 +202,6 @@ class SaltInfo(object): ''' Process events and publish data ''' - import threading logger.debug('In process {0}'.format(threading.current_thread())) logger.debug(salt_data['tag']) logger.debug(salt_data) diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index 61d1d4b66a..62a16e68be 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -7,271 +7,7 @@ A REST API for Salt :depends: - tornado Python module -All Events ----------- -Exposes ``all`` "real-time" events from Salt's event bus on a websocket connection. -It should be noted that "Real-time" here means these events are made available -to the server as soon as any salt related action (changes to minions, new jobs etc) happens. -Clients are however assumed to be able to tolerate any network transport related latencies. -Functionality provided by this endpoint is similar to the ``/events`` end point. - -The event bus on the Salt master exposes a large variety of things, notably -when executions are started on the master and also when minions ultimately -return their results. This URL provides a real-time window into a running -Salt infrastructure. Uses websocket as the transport mechanism. - -Exposes GET method to return websocket connections. -All requests should include an auth token. -A way to obtain obtain authentication tokens is shown below. - -.. code-block:: bash - - % curl -si localhost:8000/login \\ - -H "Accept: application/json" \\ - -d username='salt' \\ - -d password='salt' \\ - -d eauth='pam' - -Which results in the response - -.. code-block:: json - - { - "return": [{ - "perms": [".*", "@runner", "@wheel"], - "start": 1400556492.277421, - "token": "d0ce6c1a37e99dcc0374392f272fe19c0090cca7", - "expire": 1400599692.277422, - "user": "salt", - "eauth": "pam" - }] - } - -In this example the ``token`` returned is ``d0ce6c1a37e99dcc0374392f272fe19c0090cca7`` and can be included -in subsequent websocket requests (as part of the URL). - -The event stream can be easily consumed via JavaScript: - -.. code-block:: javascript - - // Note, you must be authenticated! - - // Get the Websocket connection to Salt - var source = new Websocket('wss://localhost:8000/all_events/d0ce6c1a37e99dcc0374392f272fe19c0090cca7'); - - // Get Salt's "real time" event stream. - source.onopen = function() { source.send('websocket client ready'); }; - - // Other handlers - source.onerror = function(e) { console.debug('error!', e); }; - - // e.data represents Salt's "real time" event data as serialized JSON. - source.onmessage = function(e) { console.debug(e.data); }; - - // Terminates websocket connection and Salt's "real time" event stream on the server. - source.close(); - -Or via Python, using the Python module -`websocket-client `_ for example. -Or the tornado -`client `_. - -.. code-block:: python - - # Note, you must be authenticated! - - from websocket import create_connection - - # Get the Websocket connection to Salt - ws = create_connection('wss://localhost:8000/all_events/d0ce6c1a37e99dcc0374392f272fe19c0090cca7') - - # Get Salt's "real time" event stream. - ws.send('websocket client ready') - - - # Simple listener to print results of Salt's "real time" event stream. - # Look at https://pypi.python.org/pypi/websocket-client/ for more examples. - while listening_to_events: - print ws.recv() # Salt's "real time" event data as serialized JSON. - - # Terminates websocket connection and Salt's "real time" event stream on the server. - ws.close() - - # Please refer to https://github.com/liris/websocket-client/issues/81 when using a self signed cert - -Above examples show how to establish a websocket connection to Salt and activating -real time updates from Salt's event stream by signaling ``websocket client ready``. - - -Formatted Events ------------------ - -Exposes ``formatted`` "real-time" events from Salt's event bus on a websocket connection. -It should be noted that "Real-time" here means these events are made available -to the server as soon as any salt related action (changes to minions, new jobs etc) happens. -Clients are however assumed to be able to tolerate any network transport related latencies. -Functionality provided by this endpoint is similar to the ``/events`` end point. - -The event bus on the Salt master exposes a large variety of things, notably -when executions are started on the master and also when minions ultimately -return their results. This URL provides a real-time window into a running -Salt infrastructure. Uses websocket as the transport mechanism. - -Formatted events parses the raw "real time" event stream and maintains -a current view of the following: - -- minions -- jobs - -A change to the minions (such as addition, removal of keys or connection drops) -or jobs is processed and clients are updated. -Since we use salt's presence events to track minions, -please enable ``presence_events`` -and set a small value for the ``loop_interval`` -in the salt master config file. - -Exposes GET method to return websocket connections. -All requests should include an auth token. -A way to obtain obtain authentication tokens is shown below. - -.. code-block:: bash - - % curl -si localhost:8000/login \\ - -H "Accept: application/json" \\ - -d username='salt' \\ - -d password='salt' \\ - -d eauth='pam' - -Which results in the response - -.. code-block:: json - - { - "return": [{ - "perms": [".*", "@runner", "@wheel"], - "start": 1400556492.277421, - "token": "d0ce6c1a37e99dcc0374392f272fe19c0090cca7", - "expire": 1400599692.277422, - "user": "salt", - "eauth": "pam" - }] - } - -In this example the ``token`` returned is ``d0ce6c1a37e99dcc0374392f272fe19c0090cca7`` and can be included -in subsequent websocket requests (as part of the URL). - -The event stream can be easily consumed via JavaScript: - -.. code-block:: javascript - - // Note, you must be authenticated! - - // Get the Websocket connection to Salt - var source = new Websocket('wss://localhost:8000/formatted_events/d0ce6c1a37e99dcc0374392f272fe19c0090cca7'); - - // Get Salt's "real time" event stream. - source.onopen = function() { source.send('websocket client ready'); }; - - // Other handlers - source.onerror = function(e) { console.debug('error!', e); }; - - // e.data represents Salt's "real time" event data as serialized JSON. - source.onmessage = function(e) { console.debug(e.data); }; - - // Terminates websocket connection and Salt's "real time" event stream on the server. - source.close(); - -Or via Python, using the Python module -`websocket-client `_ for example. -Or the tornado -`client `_. - -.. code-block:: python - - # Note, you must be authenticated! - - from websocket import create_connection - - # Get the Websocket connection to Salt - ws = create_connection('wss://localhost:8000/formatted_events/d0ce6c1a37e99dcc0374392f272fe19c0090cca7') - - # Get Salt's "real time" event stream. - ws.send('websocket client ready') - - - # Simple listener to print results of Salt's "real time" event stream. - # Look at https://pypi.python.org/pypi/websocket-client/ for more examples. - while listening_to_events: - print ws.recv() # Salt's "real time" event data as serialized JSON. - - # Terminates websocket connection and Salt's "real time" event stream on the server. - ws.close() - - # Please refer to https://github.com/liris/websocket-client/issues/81 when using a self signed cert - -Above examples show how to establish a websocket connection to Salt and activating -real time updates from Salt's event stream by signaling ``websocket client ready``. - -Example responses ------------------ - -``Minion information`` is a dictionary keyed by each connected minion's ``id`` (``mid``), -grains information for each minion is also included. - -Minion information is sent in response to the following minion events: - -- connection drops - - requires running ``manage.present`` periodically every ``loop_interval`` seconds -- minion addition -- minon removal - -.. code-block:: python - - # Not all grains are shown - data: { - "minions": { - "minion1": { - "id": "minion1", - "grains": { - "kernel": "Darwin", - "domain": "local", - "zmqversion": "4.0.3", - "kernelrelease": "13.2.0" - } - } - } - } - -``Job information`` is also tracked and delivered. - -Job information is also a dictionary -in which each job's information is keyed by salt's ``jid``. - -.. code-block:: python - - data: { - "jobs": { - "20140609153646699137": { - "tgt_type": "glob", - "jid": "20140609153646699137", - "tgt": "*", - "start_time": "2014-06-09T15:36:46.700315", - "state": "complete", - "fun": "test.ping", - "minions": { - "minion1": { - "return": true, - "retcode": 0, - "success": true - } - } - } - } - } - -Setup -===== In order to run rest_tornado with the salt-master add the following to your salt master config file. @@ -320,9 +56,8 @@ import tornado.httpserver import tornado.ioloop import tornado.web import tornado.gen -import tornado.websocket + from tornado.concurrent import Future -from . import event_processor from collections import defaultdict @@ -358,7 +93,9 @@ logger = logging.getLogger() class SaltClientsMixIn(object): - + ''' + MixIn class to container all of the salt clients that the API needs + ''' @property def saltclients(self): if not hasattr(self, '__saltclients'): @@ -398,6 +135,12 @@ class Any(Future): class EventListener(object): + ''' + Class responsible for listening to the salt master event bus and updating + futures. This is the core of what makes this async, this allows us to do + non-blocking work in the main processes and "wait" for an event to happen + ''' + def __init__(self, mod_opts, opts): self.mod_opts = mod_opts self.opts = opts @@ -430,9 +173,11 @@ class EventListener(object): if len(self.tag_map[tag]) == 0: del self.tag_map[tag] - def get_event(self, request, - tag='', - callback=None): + def get_event(self, + request, + tag='', + callback=None, + ): ''' Get an event (async of course) return a future that will get it later ''' @@ -992,108 +737,6 @@ class EventsSaltAPIHandler(SaltAPIHandler): self.finish() -class AllEventsHandler(tornado.websocket.WebSocketHandler): - ''' - Server side websocket handler. - ''' - def open(self, token): - ''' - Return a websocket connection to Salt - representing Salt's "real time" event stream. - ''' - logger.debug('In the websocket open method') - - self.token = token - # close the connection, if not authenticated - if not self.application.auth.get_tok(token): - logger.debug('Refusing websocket connection, bad token!') - self.close() - return - - self.connected = False - - @tornado.gen.coroutine - def on_message(self, message): - """Listens for a "websocket client ready" message. - Once that message is received an asynchronous job - is stated that yeilds messages to the client. - These messages make up salt's - "real time" event stream. - """ - logger.debug('Got websocket message {0}'.format(message)) - if message == 'websocket client ready': - if self.connected: - # TBD: Add ability to run commands in this branch - logger.debug('Websocket already connected, returning') - return - - self.connected = True - - while True: - try: - event = yield self.application.event_listener.get_event(self) - self.write_message(u'data: {0}\n\n'.format(json.dumps(event))) - except Exception as err: - logger.info('Error! Ending server side websocket connection. Reason = {0}'.format(str(err))) - break - - self.close() - else: - # TBD: Add logic to run salt commands here - pass - - def on_close(self, *args, **kwargs): - '''Cleanup. - - ''' - logger.debug('In the websocket close method') - self.close() - - -class FormattedEventsHandler(AllEventsHandler): - - @tornado.gen.coroutine - def on_message(self, message): - """Listens for a "websocket client ready" message. - Once that message is received an asynchronous job - is stated that yeilds messages to the client. - These messages make up salt's - "real time" event stream. - """ - logger.debug('Got websocket message {0}'.format(message)) - if message == 'websocket client ready': - if self.connected: - # TBD: Add ability to run commands in this branch - logger.debug('Websocket already connected, returning') - return - - self.connected = True - - evt_processor = event_processor.SaltInfo(self) - client = salt.netapi.NetapiClient(self.application.opts) - client.run({ - 'fun': 'grains.items', - 'tgt': '*', - 'token': self.token, - 'mode': 'client', - 'async': 'local_async', - 'client': 'local' - }) - while True: - try: - event = yield self.application.event_listener.get_event(self) - evt_processor.process(event, self.token, self.application.opts) - # self.write_message(u'data: {0}\n\n'.format(json.dumps(event))) - except Exception as err: - logger.debug('Error! Ending server side websocket connection. Reason = {0}'.format(str(err))) - break - - self.close() - else: - # TBD: Add logic to run salt commands here - pass - - class WebhookSaltAPIHandler(SaltAPIHandler): ''' Handler for /run requests diff --git a/salt/netapi/rest_tornado/saltnado_websockets.py b/salt/netapi/rest_tornado/saltnado_websockets.py new file mode 100644 index 0000000000..8a8923cf89 --- /dev/null +++ b/salt/netapi/rest_tornado/saltnado_websockets.py @@ -0,0 +1,400 @@ +# encoding: utf-8 +''' +A Websockets add-on to saltnado +=================== + +.. py:currentmodule:: salt.netapi.rest_tornado.saltnado + +:depends: - tornado Python module + +In order to enable saltnado_webosockets you must add websockets: True to your +saltnado config block. + +.. code-block:: yaml + + rest_tornado: + # can be any port + port: 8000 + ssl_crt: /etc/pki/api/certs/server.crt + # no need to specify ssl_key if cert and key + # are in one single file + ssl_key: /etc/pki/api/certs/server.key + debug: False + disable_ssl: False + websockets: True + +All Events +---------- + +Exposes ``all`` "real-time" events from Salt's event bus on a websocket connection. +It should be noted that "Real-time" here means these events are made available +to the server as soon as any salt related action (changes to minions, new jobs etc) happens. +Clients are however assumed to be able to tolerate any network transport related latencies. +Functionality provided by this endpoint is similar to the ``/events`` end point. + +The event bus on the Salt master exposes a large variety of things, notably +when executions are started on the master and also when minions ultimately +return their results. This URL provides a real-time window into a running +Salt infrastructure. Uses websocket as the transport mechanism. + +Exposes GET method to return websocket connections. +All requests should include an auth token. +A way to obtain obtain authentication tokens is shown below. + +.. code-block:: bash + + % curl -si localhost:8000/login \\ + -H "Accept: application/json" \\ + -d username='salt' \\ + -d password='salt' \\ + -d eauth='pam' + +Which results in the response + +.. code-block:: json + + { + "return": [{ + "perms": [".*", "@runner", "@wheel"], + "start": 1400556492.277421, + "token": "d0ce6c1a37e99dcc0374392f272fe19c0090cca7", + "expire": 1400599692.277422, + "user": "salt", + "eauth": "pam" + }] + } + +In this example the ``token`` returned is ``d0ce6c1a37e99dcc0374392f272fe19c0090cca7`` and can be included +in subsequent websocket requests (as part of the URL). + +The event stream can be easily consumed via JavaScript: + +.. code-block:: javascript + + // Note, you must be authenticated! + + // Get the Websocket connection to Salt + var source = new Websocket('wss://localhost:8000/all_events/d0ce6c1a37e99dcc0374392f272fe19c0090cca7'); + + // Get Salt's "real time" event stream. + source.onopen = function() { source.send('websocket client ready'); }; + + // Other handlers + source.onerror = function(e) { console.debug('error!', e); }; + + // e.data represents Salt's "real time" event data as serialized JSON. + source.onmessage = function(e) { console.debug(e.data); }; + + // Terminates websocket connection and Salt's "real time" event stream on the server. + source.close(); + +Or via Python, using the Python module +`websocket-client `_ for example. +Or the tornado +`client `_. + +.. code-block:: python + + # Note, you must be authenticated! + + from websocket import create_connection + + # Get the Websocket connection to Salt + ws = create_connection('wss://localhost:8000/all_events/d0ce6c1a37e99dcc0374392f272fe19c0090cca7') + + # Get Salt's "real time" event stream. + ws.send('websocket client ready') + + + # Simple listener to print results of Salt's "real time" event stream. + # Look at https://pypi.python.org/pypi/websocket-client/ for more examples. + while listening_to_events: + print ws.recv() # Salt's "real time" event data as serialized JSON. + + # Terminates websocket connection and Salt's "real time" event stream on the server. + ws.close() + + # Please refer to https://github.com/liris/websocket-client/issues/81 when using a self signed cert + +Above examples show how to establish a websocket connection to Salt and activating +real time updates from Salt's event stream by signaling ``websocket client ready``. + + +Formatted Events +----------------- + +Exposes ``formatted`` "real-time" events from Salt's event bus on a websocket connection. +It should be noted that "Real-time" here means these events are made available +to the server as soon as any salt related action (changes to minions, new jobs etc) happens. +Clients are however assumed to be able to tolerate any network transport related latencies. +Functionality provided by this endpoint is similar to the ``/events`` end point. + +The event bus on the Salt master exposes a large variety of things, notably +when executions are started on the master and also when minions ultimately +return their results. This URL provides a real-time window into a running +Salt infrastructure. Uses websocket as the transport mechanism. + +Formatted events parses the raw "real time" event stream and maintains +a current view of the following: + +- minions +- jobs + +A change to the minions (such as addition, removal of keys or connection drops) +or jobs is processed and clients are updated. +Since we use salt's presence events to track minions, +please enable ``presence_events`` +and set a small value for the ``loop_interval`` +in the salt master config file. + +Exposes GET method to return websocket connections. +All requests should include an auth token. +A way to obtain obtain authentication tokens is shown below. + +.. code-block:: bash + + % curl -si localhost:8000/login \\ + -H "Accept: application/json" \\ + -d username='salt' \\ + -d password='salt' \\ + -d eauth='pam' + +Which results in the response + +.. code-block:: json + + { + "return": [{ + "perms": [".*", "@runner", "@wheel"], + "start": 1400556492.277421, + "token": "d0ce6c1a37e99dcc0374392f272fe19c0090cca7", + "expire": 1400599692.277422, + "user": "salt", + "eauth": "pam" + }] + } + +In this example the ``token`` returned is ``d0ce6c1a37e99dcc0374392f272fe19c0090cca7`` and can be included +in subsequent websocket requests (as part of the URL). + +The event stream can be easily consumed via JavaScript: + +.. code-block:: javascript + + // Note, you must be authenticated! + + // Get the Websocket connection to Salt + var source = new Websocket('wss://localhost:8000/formatted_events/d0ce6c1a37e99dcc0374392f272fe19c0090cca7'); + + // Get Salt's "real time" event stream. + source.onopen = function() { source.send('websocket client ready'); }; + + // Other handlers + source.onerror = function(e) { console.debug('error!', e); }; + + // e.data represents Salt's "real time" event data as serialized JSON. + source.onmessage = function(e) { console.debug(e.data); }; + + // Terminates websocket connection and Salt's "real time" event stream on the server. + source.close(); + +Or via Python, using the Python module +`websocket-client `_ for example. +Or the tornado +`client `_. + +.. code-block:: python + + # Note, you must be authenticated! + + from websocket import create_connection + + # Get the Websocket connection to Salt + ws = create_connection('wss://localhost:8000/formatted_events/d0ce6c1a37e99dcc0374392f272fe19c0090cca7') + + # Get Salt's "real time" event stream. + ws.send('websocket client ready') + + + # Simple listener to print results of Salt's "real time" event stream. + # Look at https://pypi.python.org/pypi/websocket-client/ for more examples. + while listening_to_events: + print ws.recv() # Salt's "real time" event data as serialized JSON. + + # Terminates websocket connection and Salt's "real time" event stream on the server. + ws.close() + + # Please refer to https://github.com/liris/websocket-client/issues/81 when using a self signed cert + +Above examples show how to establish a websocket connection to Salt and activating +real time updates from Salt's event stream by signaling ``websocket client ready``. + +Example responses +----------------- + +``Minion information`` is a dictionary keyed by each connected minion's ``id`` (``mid``), +grains information for each minion is also included. + +Minion information is sent in response to the following minion events: + +- connection drops + - requires running ``manage.present`` periodically every ``loop_interval`` seconds +- minion addition +- minon removal + +.. code-block:: python + + # Not all grains are shown + data: { + "minions": { + "minion1": { + "id": "minion1", + "grains": { + "kernel": "Darwin", + "domain": "local", + "zmqversion": "4.0.3", + "kernelrelease": "13.2.0" + } + } + } + } + +``Job information`` is also tracked and delivered. + +Job information is also a dictionary +in which each job's information is keyed by salt's ``jid``. + +.. code-block:: python + + data: { + "jobs": { + "20140609153646699137": { + "tgt_type": "glob", + "jid": "20140609153646699137", + "tgt": "*", + "start_time": "2014-06-09T15:36:46.700315", + "state": "complete", + "fun": "test.ping", + "minions": { + "minion1": { + "return": true, + "retcode": 0, + "success": true + } + } + } + } + } + +Setup +===== +''' + +import tornado.websocket +from . import event_processor + +import tornado.gen + +import logging +logger = logging.getLogger() + +class AllEventsHandler(tornado.websocket.WebSocketHandler): + ''' + Server side websocket handler. + ''' + def open(self, token): + ''' + Return a websocket connection to Salt + representing Salt's "real time" event stream. + ''' + logger.debug('In the websocket open method') + + self.token = token + # close the connection, if not authenticated + if not self.application.auth.get_tok(token): + logger.debug('Refusing websocket connection, bad token!') + self.close() + return + + self.connected = False + + @tornado.gen.coroutine + def on_message(self, message): + """Listens for a "websocket client ready" message. + Once that message is received an asynchronous job + is stated that yeilds messages to the client. + These messages make up salt's + "real time" event stream. + """ + logger.debug('Got websocket message {0}'.format(message)) + if message == 'websocket client ready': + if self.connected: + # TBD: Add ability to run commands in this branch + logger.debug('Websocket already connected, returning') + return + + self.connected = True + + while True: + try: + event = yield self.application.event_listener.get_event(self) + self.write_message(u'data: {0}\n\n'.format(json.dumps(event))) + except Exception as err: + logger.info('Error! Ending server side websocket connection. Reason = {0}'.format(str(err))) + break + + self.close() + else: + # TBD: Add logic to run salt commands here + pass + + def on_close(self, *args, **kwargs): + '''Cleanup. + + ''' + logger.debug('In the websocket close method') + self.close() + + +class FormattedEventsHandler(AllEventsHandler): + + @tornado.gen.coroutine + def on_message(self, message): + """Listens for a "websocket client ready" message. + Once that message is received an asynchronous job + is stated that yeilds messages to the client. + These messages make up salt's + "real time" event stream. + """ + logger.debug('Got websocket message {0}'.format(message)) + if message == 'websocket client ready': + if self.connected: + # TBD: Add ability to run commands in this branch + logger.debug('Websocket already connected, returning') + return + + self.connected = True + + evt_processor = event_processor.SaltInfo(self) + client = salt.netapi.NetapiClient(self.application.opts) + client.run({ + 'fun': 'grains.items', + 'tgt': '*', + 'token': self.token, + 'mode': 'client', + 'async': 'local_async', + 'client': 'local' + }) + while True: + try: + event = yield self.application.event_listener.get_event(self) + evt_processor.process(event, self.token, self.application.opts) + # self.write_message(u'data: {0}\n\n'.format(json.dumps(event))) + except Exception as err: + logger.debug('Error! Ending server side websocket connection. Reason = {0}'.format(str(err))) + break + + self.close() + else: + # TBD: Add logic to run salt commands here + pass From 943476646a30839cbfa63758513be337f6294bb6 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 20 Jun 2014 15:16:13 -0700 Subject: [PATCH 05/55] pep8 cleanup --- salt/netapi/rest_tornado/saltnado.py | 22 +++++++++---------- .../rest_tornado/saltnado_websockets.py | 1 + 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index 62a16e68be..28e02cdf13 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -145,9 +145,9 @@ class EventListener(object): self.mod_opts = mod_opts self.opts = opts self.event = salt.utils.event.get_event( - 'master', - opts['sock_dir'], - opts['transport']) + 'master', + opts['sock_dir'], + opts['transport']) # tag -> list of futures self.tag_map = defaultdict(list) @@ -425,8 +425,8 @@ class SaltAuthHandler(BaseSaltAPIHandler): perms = self.application.opts['external_auth'][token['eauth']][token['name']] except (AttributeError, IndexError): logging.debug("Configuration for external_auth malformed for " - "eauth '{0}', and user '{1}'." - .format(token.get('eauth'), token.get('name')), exc_info=True) + "eauth '{0}', and user '{1}'." + .format(token.get('eauth'), token.get('name')), exc_info=True) # TODO better error -- 'Configuration for external_auth could not be read.' self.send_error(500) @@ -516,9 +516,9 @@ class SaltAPIHandler(BaseSaltAPIHandler, SaltClientsMixIn): # ping all the minions (to see who we have to talk to) # TODO: actually ping them all? this just gets the pub data minions = self.saltclients['local'](chunk['tgt'], - 'test.ping', - [], - expr_form=f_call['kwargs']['expr_form'])['minions'] + 'test.ping', + [], + expr_form=f_call['kwargs']['expr_form'])['minions'] chunk_ret = {} maxflight = get_batch_size(f_call['kwargs']['batch'], len(minions)) @@ -753,9 +753,9 @@ class WebhookSaltAPIHandler(SaltAPIHandler): # TODO: consolidate?? self.event = salt.utils.event.get_event( - 'master', - self.application.opts['sock_dir'], - self.application.opts['transport']) + 'master', + self.application.opts['sock_dir'], + self.application.opts['transport']) ret = self.event.fire_event({ 'post': self.raw_data, diff --git a/salt/netapi/rest_tornado/saltnado_websockets.py b/salt/netapi/rest_tornado/saltnado_websockets.py index 8a8923cf89..da829a1185 100644 --- a/salt/netapi/rest_tornado/saltnado_websockets.py +++ b/salt/netapi/rest_tornado/saltnado_websockets.py @@ -298,6 +298,7 @@ import tornado.gen import logging logger = logging.getLogger() + class AllEventsHandler(tornado.websocket.WebSocketHandler): ''' Server side websocket handler. From baacda228682a50acc5a4528d43f5d3a88c7c6ec Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 20 Jun 2014 15:57:11 -0700 Subject: [PATCH 06/55] Make sure to not leave hanging children processes if the parent is killed --- salt/client/netapi.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/salt/client/netapi.py b/salt/client/netapi.py index 0b3a718611..61c79fbc45 100644 --- a/salt/client/netapi.py +++ b/salt/client/netapi.py @@ -5,6 +5,7 @@ The main entry point for salt-api # Import python libs import logging import multiprocessing +import signal # Import salt-api libs import salt.loader @@ -18,6 +19,7 @@ class NetapiClient(object): ''' def __init__(self, opts): self.opts = opts + self.processes = [] def run(self): ''' @@ -27,4 +29,17 @@ class NetapiClient(object): for fun in netapi: if fun.endswith('.start'): logger.info("Starting '{0}' api module".format(fun)) - multiprocessing.Process(target=netapi[fun]).start() + p = multiprocessing.Process(target=netapi[fun]) + p.start() + self.processes.append(p) + + # make sure to kill the subprocesses if the parent is killed + signal.signal(signal.SIGTERM, self.kill_children) + + def kill_children(self, *args): + ''' + Kill all of the children + ''' + for p in self.processes: + p.terminate() + p.join() From 31f8c045069d670148dacf7f4099c8fee169624d Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 20 Jun 2014 16:44:20 -0700 Subject: [PATCH 07/55] Adding 2 missing imports --- salt/netapi/rest_tornado/saltnado_websockets.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/netapi/rest_tornado/saltnado_websockets.py b/salt/netapi/rest_tornado/saltnado_websockets.py index da829a1185..c93f25400e 100644 --- a/salt/netapi/rest_tornado/saltnado_websockets.py +++ b/salt/netapi/rest_tornado/saltnado_websockets.py @@ -295,6 +295,9 @@ from . import event_processor import tornado.gen +import json +import salt.netapi + import logging logger = logging.getLogger() From d654956c05d7e521afc73658408bbdb44fa27357 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 20 Jun 2014 16:49:28 -0700 Subject: [PATCH 08/55] misc pylint fixes (missing imports etc) Add utils function to import faster json libaries (if you have them) and switch couhcbase_return and saltnado to use them --- salt/netapi/rest_tornado/saltnado.py | 3 ++- salt/netapi/rest_tornado/saltnado_websockets.py | 4 +++- salt/returners/couchbase_return.py | 10 ++-------- salt/utils/__init__.py | 13 +++++++++++++ 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index 28e02cdf13..17ed7a5fe5 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -63,7 +63,6 @@ from collections import defaultdict import math import functools -import json import yaml import zmq import fnmatch @@ -78,6 +77,8 @@ import salt.runner import salt.auth from salt import syspaths + +json = salt.utils.import_json() logger = logging.getLogger() # The clients rest_cherrypi supports. We want to mimic the interface, but not diff --git a/salt/netapi/rest_tornado/saltnado_websockets.py b/salt/netapi/rest_tornado/saltnado_websockets.py index c93f25400e..0cf651175e 100644 --- a/salt/netapi/rest_tornado/saltnado_websockets.py +++ b/salt/netapi/rest_tornado/saltnado_websockets.py @@ -295,9 +295,11 @@ from . import event_processor import tornado.gen -import json +import salt.utils import salt.netapi +json = salt.utils.import_json() + import logging logger = logging.getLogger() diff --git a/salt/returners/couchbase_return.py b/salt/returners/couchbase_return.py index 48b89e7437..983915b868 100644 --- a/salt/returners/couchbase_return.py +++ b/salt/returners/couchbase_return.py @@ -55,14 +55,8 @@ def __virtual__(): return False # try to load some faster json libraries. In order of fastest to slowest - for fast_json in ('ujson', 'yajl'): - try: - mod = __import__(fast_json) - couchbase.set_json_converters(mod.dumps, mod.loads) - log.info('loaded {0} json lib'.format(fast_json)) - break - except ImportError: - continue + json = salt.utils.import_json() + couchbase.set_json_converters(json.dumps, json.loads) return __virtualname__ diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 95574f0a5b..8401c2e8c2 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -2333,3 +2333,16 @@ def total_seconds(td): method which does not exist in versions of Python < 2.7. ''' return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 + + +def import_json(): + ''' + Import a json module, starting with the quick ones and going down the list) + ''' + for fast_json in ('ujson', 'yajl', 'json'): + try: + mod = __import__(fast_json) + log.info('loaded {0} json lib'.format(fast_json)) + return mod + except ImportError: + continue From 16cfaad06146eb1e95ff43bd797000e0f0b2be3e Mon Sep 17 00:00:00 2001 From: Seth House Date: Fri, 20 Jun 2014 19:47:32 -0600 Subject: [PATCH 09/55] Quiet two pylint errors about missing __init__ method --- salt/netapi/rest_tornado/saltnado_websockets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/netapi/rest_tornado/saltnado_websockets.py b/salt/netapi/rest_tornado/saltnado_websockets.py index 0cf651175e..d90e6ab52f 100644 --- a/salt/netapi/rest_tornado/saltnado_websockets.py +++ b/salt/netapi/rest_tornado/saltnado_websockets.py @@ -304,7 +304,7 @@ import logging logger = logging.getLogger() -class AllEventsHandler(tornado.websocket.WebSocketHandler): +class AllEventsHandler(tornado.websocket.WebSocketHandler): # pylint: disable=W0232 ''' Server side websocket handler. ''' @@ -362,7 +362,7 @@ class AllEventsHandler(tornado.websocket.WebSocketHandler): self.close() -class FormattedEventsHandler(AllEventsHandler): +class FormattedEventsHandler(AllEventsHandler): # pylint: disable=W0232 @tornado.gen.coroutine def on_message(self, message): From 9d36f9f3019b9c86a69fcbaa47add914da7efc44 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 20 Jun 2014 19:00:10 -0700 Subject: [PATCH 10/55] Add some basic tests for some of the globals noted in #12292 This is by no means complete (a lot of modules get __low__ or __env__ from their callers), but this will help catch some of the problems where we change the loader and the magic stops. IMO we kinda need to overhaul the loader, and break out the module load, __virt__ running, packing of globals, and caching into seperate pieces --- tests/integration/loader/globals.py | 137 ++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 tests/integration/loader/globals.py diff --git a/tests/integration/loader/globals.py b/tests/integration/loader/globals.py new file mode 100644 index 0000000000..e3632fc8c7 --- /dev/null +++ b/tests/integration/loader/globals.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +''' + integration.loader.globals + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Test Salt's loader regarding globals that it should pack in +''' + +# Import Salt Testing libs +from salttesting import TestCase +from salttesting.helpers import ensure_in_syspath +ensure_in_syspath('../') + +# Import salt libs +import integration +import salt.loader +import inspect +import yaml + + +class LoaderGlobalsTest(integration.ModuleCase): + ''' + Test all of the globals that the loader is responsible for adding to modules + + This shouldn't be done here, but should rather be done per module type (in the cases where they are used) + so they can check ALL globals that they have (or should have) access to. + + This is intended as a shorter term way of testing these so we don't break the loader + ''' + def _verify_globals(self, mod_dict): + ''' + Verify that the globals listed in the doc string (from the test) are in these modules + ''' + # find the globals + global_vars = None + for key, val in mod_dict.iteritems(): + if hasattr(val, '__globals__'): + global_vars = val.__globals__ + break + # if we couldn't find any, then we have no modules-- so something is broken + if global_vars is None: + # TODO: log or something? Skip however we do that + return + + # get the names of the globals you should have + func_name = inspect.stack()[1][3] + names = yaml.load(getattr(self, func_name).__doc__).values()[0] + + for name in names: + assert name in global_vars + + def test_auth(self): + ''' + Test that auth mods have: + - __pillar__ + - __grains__ + - __salt__ + ''' + self._verify_globals(salt.loader.auth(self.master_opts)) + + def test_runners(self): + ''' + Test that runners have: + - __pillar__ + - __salt__ + - __opts__ + - __grains__ + ''' + self._verify_globals(salt.loader.runner(self.master_opts)) + + def test_returners(self): + ''' + Test that returners have: + - __salt__ + - __opts__ + - __pillar__ + - __grains__ + ''' + self._verify_globals(salt.loader.returners(self.master_opts, {})) + + def test_pillars(self): + ''' + Test that pillars have: + - __salt__ + - __opts__ + - __pillar__ + - __grains__ + ''' + self._verify_globals(salt.loader.pillars(self.master_opts, {})) + + def test_tops(self): + ''' + Test that tops have: [] + ''' + self._verify_globals(salt.loader.tops(self.master_opts)) + + def test_outputters(self): + ''' + Test that outputters have: + - __opts__ + - __pillar__ + - __grains__ + ''' + self._verify_globals(salt.loader.outputters(self.master_opts)) + + def test_states(self): + ''' + Test that states: + - __pillar__ + - __salt__ + - __opts__ + - __grains__ + ''' + self._verify_globals(salt.loader.states(self.master_opts, {})) + + def test_log_handlers(self): + ''' + Test that log_handlers have: + - __path__ + ''' + self._verify_globals(salt.loader.log_handlers(self.master_opts)) + + def test_renderers(self): + ''' + Test that renderers have: + - __salt__ # Execution functions (i.e. __salt__['test.echo']('foo')) + - __grains__ # Grains (i.e. __grains__['os']) + - __pillar__ # Pillar data (i.e. __pillar__['foo']) + - __opts__ # Minion configuration options + ''' + self._verify_globals(salt.loader.render(self.master_opts, {})) + + + +if __name__ == '__main__': + from integration import run_tests + run_tests(LoaderGlobalsTest, needs_daemon=False) From eed98ab4b01b306c49d238c808a5723444a8ae32 Mon Sep 17 00:00:00 2001 From: Seth House Date: Fri, 20 Jun 2014 20:05:42 -0600 Subject: [PATCH 11/55] Fixed a boolean check that was broken by overzealous lint fixing! --- salt/netapi/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/netapi/__init__.py b/salt/netapi/__init__.py index 1441bb95fc..e3f8bf9ec3 100644 --- a/salt/netapi/__init__.py +++ b/salt/netapi/__init__.py @@ -34,7 +34,7 @@ class NetapiClient(object): if 'client' not in low: raise SaltException('No client specified') - if 'token' not in low or 'eauth' not in low: + if not ('token' in low or 'eauth' in low): raise EauthAuthenticationError( 'No authentication credentials given') From 1881e17a8eb3a0aebab43e97c109eb7dd878857f Mon Sep 17 00:00:00 2001 From: nmadhok Date: Sat, 21 Jun 2014 00:41:49 -0400 Subject: [PATCH 12/55] Fixing syntax to valid JSON and fixing keyring and keyname field --- salt/modules/ddns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/ddns.py b/salt/modules/ddns.py index dd5488ba5f..7e4b93ac39 100644 --- a/salt/modules/ddns.py +++ b/salt/modules/ddns.py @@ -8,15 +8,15 @@ Support for RFC 2136 dynamic DNS updates. support this (the keyname is only needed if the keyring contains more than one key):: - ddns.keyring: keyring file (default=None) - ddns.keyname: key name in file (default=None) + keyring: keyring file (default=None) + keyname: key name in file (default=None) The keyring file needs to be in json format and the key name needs to end with an extra period in the file, similar to this: .. code-block:: bash - {'keyname.': 'keycontent'} + {"keyname.": "keycontent"} ''' # Import python libs import logging From dbb3401bb70e441cf58028570b65b14e94386b4e Mon Sep 17 00:00:00 2001 From: nmadhok Date: Sat, 21 Jun 2014 00:47:28 -0400 Subject: [PATCH 13/55] Fixing ddns.absent docs and fixing the invalid output that ddns.absent returns --- salt/states/ddns.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/states/ddns.py b/salt/states/ddns.py index c023918221..d82f778e72 100644 --- a/salt/states/ddns.py +++ b/salt/states/ddns.py @@ -90,7 +90,7 @@ def absent(name, zone, data=None, rdtype=None, **kwargs): DNS resource type. If omitted, all types will be purged. ``**kwargs`` - Additional arguments the ddns.update function may need (e.g. keyfile). + Additional arguments the ddns.delete function may need (e.g. keyfile). ''' ret = {'name': name, 'changes': {}, @@ -110,7 +110,10 @@ def absent(name, zone, data=None, rdtype=None, **kwargs): elif status: ret['result'] = True ret['comment'] = 'Deleted DNS record(s)' - ret['changes'] = True + ret['changes'] = {'Deleted': {'name': name, + 'zone': zone + } + } else: ret['result'] = False ret['comment'] = 'Failed to delete DNS record(s)' From f918e08c165229f57f797de8dcf2379c5fc4db6c Mon Sep 17 00:00:00 2001 From: nmadhok Date: Sat, 21 Jun 2014 01:22:31 -0400 Subject: [PATCH 14/55] Fixing pylint errors --- salt/daemons/flo/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/daemons/flo/core.py b/salt/daemons/flo/core.py index 6b34183f05..2ac29cab24 100644 --- a/salt/daemons/flo/core.py +++ b/salt/daemons/flo/core.py @@ -51,6 +51,7 @@ except ImportError: pass log = logging.getLogger(__name__) + class SaltRaetRoadStack(ioflo.base.deeding.Deed): ''' Initialize and run raet udp stack for Salt @@ -225,7 +226,8 @@ class SaltRaetRoadStackRejected(ioflo.base.deeding.Deed): if stack.remotes: rejected = (stack.remotes.values()[0].acceptance == raeting.acceptances.rejected) - else: #no remotes so assume rejected + else: + #no remotes so assume rejected rejected = True self.status.update(rejected=rejected) @@ -282,6 +284,7 @@ class SaltRaetRoadStackAllowed(ioflo.base.deeding.Deed): allowed = stack.remotes.values()[0].allowed self.status.update(allowed=allowed) + class SaltRaetRoadStackManager(ioflo.base.deeding.Deed): ''' Runs the manage method of RoadStack From 84e5faa7abc02972a8bd83dd31e2d08ca47a3cd3 Mon Sep 17 00:00:00 2001 From: nmadhok Date: Sat, 21 Jun 2014 01:30:47 -0400 Subject: [PATCH 15/55] Fixing pylint errors --- salt/daemons/flo/core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/daemons/flo/core.py b/salt/daemons/flo/core.py index 2ac29cab24..1f87ecfc4a 100644 --- a/salt/daemons/flo/core.py +++ b/salt/daemons/flo/core.py @@ -226,8 +226,7 @@ class SaltRaetRoadStackRejected(ioflo.base.deeding.Deed): if stack.remotes: rejected = (stack.remotes.values()[0].acceptance == raeting.acceptances.rejected) - else: - #no remotes so assume rejected + else: #no remotes so assume rejected rejected = True self.status.update(rejected=rejected) From d395e1fddf4e45661605239ef4b883dd388c3c34 Mon Sep 17 00:00:00 2001 From: nmadhok Date: Sat, 21 Jun 2014 01:45:04 -0400 Subject: [PATCH 16/55] Fixing left pylint error --- salt/daemons/flo/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/daemons/flo/core.py b/salt/daemons/flo/core.py index 1f87ecfc4a..b10f35a8df 100644 --- a/salt/daemons/flo/core.py +++ b/salt/daemons/flo/core.py @@ -226,7 +226,7 @@ class SaltRaetRoadStackRejected(ioflo.base.deeding.Deed): if stack.remotes: rejected = (stack.remotes.values()[0].acceptance == raeting.acceptances.rejected) - else: #no remotes so assume rejected + else: # no remotes so assume rejected rejected = True self.status.update(rejected=rejected) From 8ae2b9e93ab3dd830cf5437de0ac6ec6655cd08d Mon Sep 17 00:00:00 2001 From: nmadhok Date: Sat, 21 Jun 2014 02:00:24 -0400 Subject: [PATCH 17/55] Fixing pylint error --- salt/modules/postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index 8bc1000a54..9f35f20cd9 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -240,7 +240,7 @@ def psql_query(query, user=None, host=None, port=None, maintenance_db=None, Run an SQL-Query and return the results as a list. This command only supports SELECT statements. This limitation can be worked around with a query like this: - + WITH updated AS (UPDATE pg_authid SET rolconnlimit = 2000 WHERE rolname = 'rolename' RETURNING rolconnlimit) SELECT * FROM updated; From 63af29893a1eab74948d3ce6d4cba5011dfc8d3e Mon Sep 17 00:00:00 2001 From: nmadhok Date: Sat, 21 Jun 2014 02:16:29 -0400 Subject: [PATCH 18/55] Making the documentation more clear --- salt/states/ddns.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/states/ddns.py b/salt/states/ddns.py index d82f778e72..7e53945b3c 100644 --- a/salt/states/ddns.py +++ b/salt/states/ddns.py @@ -12,6 +12,9 @@ type dynamic updates. Requires dnspython module. ddns.present: - zone: example.com - ttl: 60 + - data: 111.222.333.444 + - nameserver: 123.234.345.456 + - keyfile: /srv/salt/tsig_key.txt ''' @@ -39,7 +42,7 @@ def present(name, zone, ttl, data, rdtype='A', **kwargs): DNS resource type. Default 'A'. ``**kwargs`` - Additional arguments the ddns.update function may need (e.g. keyfile). + Additional arguments the ddns.update function may need (e.g. nameserver, keyfile, keyname). ''' ret = {'name': name, 'changes': {}, @@ -90,7 +93,7 @@ def absent(name, zone, data=None, rdtype=None, **kwargs): DNS resource type. If omitted, all types will be purged. ``**kwargs`` - Additional arguments the ddns.delete function may need (e.g. keyfile). + Additional arguments the ddns.delete function may need (e.g. nameserver, keyfile, keyname). ''' ret = {'name': name, 'changes': {}, From 5dabfdb2ddaa863da2d99f95298d820d3ca2195a Mon Sep 17 00:00:00 2001 From: nmadhok Date: Sat, 21 Jun 2014 02:26:22 -0400 Subject: [PATCH 19/55] Fixing pylint error by removing hidden trailing spaces --- salt/modules/postgres.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index 9f35f20cd9..26c057ff04 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -240,8 +240,8 @@ def psql_query(query, user=None, host=None, port=None, maintenance_db=None, Run an SQL-Query and return the results as a list. This command only supports SELECT statements. This limitation can be worked around with a query like this: - - WITH updated AS (UPDATE pg_authid SET rolconnlimit = 2000 WHERE + + WITH updated AS (UPDATE pg_authid SET rolconnlimit = 2000 WHERE rolname = 'rolename' RETURNING rolconnlimit) SELECT * FROM updated; CLI Example: From ce4052a3b0c451b8a0c8a8fbbb3fcb2bf1e7c48f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 21 Jun 2014 09:55:41 +0100 Subject: [PATCH 20/55] Let's not forget the man pages. Refs #13554 --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 21948889c2..4813137a0e 100755 --- a/setup.py +++ b/setup.py @@ -570,11 +570,15 @@ if IS_WINDOWS_PLATFORM is False: 'doc/man/salt-master.1', 'doc/man/salt-key.1', 'doc/man/salt.1', + 'doc/man/salt-api.1', 'doc/man/salt-syndic.1', 'doc/man/salt-run.1', 'doc/man/salt-ssh.1', 'doc/man/salt-cloud.1' ]) + SETUP_KWARGS['data_files'][1][1].extend([ + 'doc/man/salt-api.7', + ]) # bbfreeze explicit includes From a5ae66c2670d9061d22f2b291d579b8c11a8ef04 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 21 Jun 2014 10:12:10 +0100 Subject: [PATCH 21/55] ZeroMQ dependencies should be pulled by default --- setup.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 21948889c2..e10c98755f 100755 --- a/setup.py +++ b/setup.py @@ -407,19 +407,13 @@ class Install(install): self.salt_transport ) ) - if self.salt_transport == 'none': + elif self.salt_transport == 'none': for requirement in _parse_requirements_file(SALT_ZEROMQ_REQS): if requirement not in self.distribution.install_requires: continue self.distribution.install_requires.remove(requirement) - return - if self.salt_transport in ('zeromq', 'both'): - self.distribution.install_requires.extend( - _parse_requirements_file(SALT_ZEROMQ_REQS) - ) - - if self.salt_transport in ('raet', 'both'): + elif self.salt_transport in ('raet', 'both'): self.distribution.install_requires.extend( _parse_requirements_file(SALT_RAET_REQS) ) @@ -550,7 +544,9 @@ SETUP_KWARGS = {'name': NAME, ], # Required for esky builds, ZeroMQ or RAET deps will be added # at install time - 'install_requires': _parse_requirements_file(SALT_REQS), + 'install_requires': + _parse_requirements_file(SALT_REQS) + + _parse_requirements_file(SALT_ZEROMQ_REQS), 'extras_require': { 'RAET': _parse_requirements_file(SALT_RAET_REQS), 'Cloud': _parse_requirements_file(SALT_CLOUD_REQS) From 99dfb00805ac38ad9764dd8c93798920ba1823ff Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 20 Jun 2014 13:52:48 +0000 Subject: [PATCH 22/55] Update 'doc/.tx/config' - Build #88 http://cookingwithsalt.org/job/salt/job/transifex/88/ Signed-off-by: Pedro Algarvio --- doc/.tx/config | 120 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/doc/.tx/config b/doc/.tx/config index 791fade073..259a037c5c 100644 --- a/doc/.tx/config +++ b/doc/.tx/config @@ -4424,3 +4424,123 @@ source_file = _build/locale/ref/states/all/salt.states.serverdensity_device.pot source_lang = en source_name = ref/states/all/salt.states.serverdensity_device.rst +[salt.ref--cli--salt-api] +file_filter = locale//LC_MESSAGES/ref/cli/salt-api.po +source_file = _build/locale/ref/cli/salt-api.pot +source_lang = en +source_name = ref/cli/salt-api.rst + +[salt.ref--netapi--all--index] +file_filter = locale//LC_MESSAGES/ref/netapi/all/index.po +source_file = _build/locale/ref/netapi/all/index.pot +source_lang = en +source_name = ref/netapi/all/index.rst + +[salt.ref--netapi--all--salt_netapi_rest_cherrypy] +file_filter = locale//LC_MESSAGES/ref/netapi/all/salt.netapi.rest_cherrypy.po +source_file = _build/locale/ref/netapi/all/salt.netapi.rest_cherrypy.pot +source_lang = en +source_name = ref/netapi/all/salt.netapi.rest_cherrypy.rst + +[salt.ref--netapi--all--salt_netapi_rest_tornado] +file_filter = locale//LC_MESSAGES/ref/netapi/all/salt.netapi.rest_tornado.po +source_file = _build/locale/ref/netapi/all/salt.netapi.rest_tornado.pot +source_lang = en +source_name = ref/netapi/all/salt.netapi.rest_tornado.rst + +[salt.ref--netapi--all--salt_netapi_rest_wsgi] +file_filter = locale//LC_MESSAGES/ref/netapi/all/salt.netapi.rest_wsgi.po +source_file = _build/locale/ref/netapi/all/salt.netapi.rest_wsgi.pot +source_lang = en +source_name = ref/netapi/all/salt.netapi.rest_wsgi.rst + +[salt.ref--pillar--all--salt_pillar_foreman] +file_filter = locale//LC_MESSAGES/ref/pillar/all/salt.pillar.foreman.po +source_file = _build/locale/ref/pillar/all/salt.pillar.foreman.pot +source_lang = en +source_name = ref/pillar/all/salt.pillar.foreman.rst + +[salt.topics--netapi--index] +file_filter = locale//LC_MESSAGES/topics/netapi/index.po +source_file = _build/locale/topics/netapi/index.pot +source_lang = en +source_name = topics/netapi/index.rst + +[salt.topics--netapi--writing] +file_filter = locale//LC_MESSAGES/topics/netapi/writing.po +source_file = _build/locale/topics/netapi/writing.pot +source_lang = en +source_name = topics/netapi/writing.rst + +[salt.topics--releases--2014_1_5] +file_filter = locale//LC_MESSAGES/topics/releases/2014.1.5.po +source_file = _build/locale/topics/releases/2014.1.5.pot +source_lang = en +source_name = topics/releases/2014.1.5.rst + +[salt.topics--releases--saltapi--0_5_0] +file_filter = locale//LC_MESSAGES/topics/releases/saltapi/0.5.0.po +source_file = _build/locale/topics/releases/saltapi/0.5.0.pot +source_lang = en +source_name = topics/releases/saltapi/0.5.0.rst + +[salt.topics--releases--saltapi--0_6_0] +file_filter = locale//LC_MESSAGES/topics/releases/saltapi/0.6.0.po +source_file = _build/locale/topics/releases/saltapi/0.6.0.pot +source_lang = en +source_name = topics/releases/saltapi/0.6.0.rst + +[salt.topics--releases--saltapi--0_7_0] +file_filter = locale//LC_MESSAGES/topics/releases/saltapi/0.7.0.po +source_file = _build/locale/topics/releases/saltapi/0.7.0.pot +source_lang = en +source_name = topics/releases/saltapi/0.7.0.rst + +[salt.topics--releases--saltapi--0_7_5] +file_filter = locale//LC_MESSAGES/topics/releases/saltapi/0.7.5.po +source_file = _build/locale/topics/releases/saltapi/0.7.5.pot +source_lang = en +source_name = topics/releases/saltapi/0.7.5.rst + +[salt.topics--releases--saltapi--0_8_0] +file_filter = locale//LC_MESSAGES/topics/releases/saltapi/0.8.0.po +source_file = _build/locale/topics/releases/saltapi/0.8.0.pot +source_lang = en +source_name = topics/releases/saltapi/0.8.0.rst + +[salt.topics--releases--saltapi--0_8_2] +file_filter = locale//LC_MESSAGES/topics/releases/saltapi/0.8.2.po +source_file = _build/locale/topics/releases/saltapi/0.8.2.pot +source_lang = en +source_name = topics/releases/saltapi/0.8.2.rst + +[salt.topics--releases--saltapi--0_8_3] +file_filter = locale//LC_MESSAGES/topics/releases/saltapi/0.8.3.po +source_file = _build/locale/topics/releases/saltapi/0.8.3.pot +source_lang = en +source_name = topics/releases/saltapi/0.8.3.rst + +[salt.topics--releases--saltapi--0_8_4] +file_filter = locale//LC_MESSAGES/topics/releases/saltapi/0.8.4.po +source_file = _build/locale/topics/releases/saltapi/0.8.4.pot +source_lang = en +source_name = topics/releases/saltapi/0.8.4.rst + +[salt.topics--releases--saltapi--index] +file_filter = locale//LC_MESSAGES/topics/releases/saltapi/index.po +source_file = _build/locale/topics/releases/saltapi/index.pot +source_lang = en +source_name = topics/releases/saltapi/index.rst + +[salt.topics--sdb--index] +file_filter = locale//LC_MESSAGES/topics/sdb/index.po +source_file = _build/locale/topics/sdb/index.pot +source_lang = en +source_name = topics/sdb/index.rst + +[salt.topics--targeting--ipcidr] +file_filter = locale//LC_MESSAGES/topics/targeting/ipcidr.po +source_file = _build/locale/topics/targeting/ipcidr.pot +source_lang = en +source_name = topics/targeting/ipcidr.rst + From d71128f98e37455c58e18eb4cc4a48783e5d9bad Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 21 Jun 2014 14:46:08 +0100 Subject: [PATCH 23/55] Update to latest bootstrap-script.sh, v2014.06.21 --- salt/cloud/deploy/bootstrap-salt.sh | 53 ++++++++++++++++++++++------- setup.py | 2 +- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/salt/cloud/deploy/bootstrap-salt.sh b/salt/cloud/deploy/bootstrap-salt.sh index c305db8665..9dd24657a0 100755 --- a/salt/cloud/deploy/bootstrap-salt.sh +++ b/salt/cloud/deploy/bootstrap-salt.sh @@ -17,7 +17,7 @@ # CREATED: 10/15/2012 09:49:37 PM WEST #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2014.06.19" +__ScriptVersion="2014.06.21" __ScriptName="bootstrap-salt.sh" #====================================================================================================================== @@ -418,6 +418,7 @@ fi # Export the http_proxy configuration to our current environment if [ "x${_HTTP_PROXY}" != "x" ]; then export http_proxy="$_HTTP_PROXY" + export https_proxy="$_HTTP_PROXY" fi # Let's discover how we're being called @@ -427,9 +428,8 @@ CALLER=$(ps -a -o pid,args | grep $$ | grep -v grep | tr -s ' ' | cut -d ' ' -f if [ "${CALLER}x" = "${0}x" ]; then CALLER="PIPED THROUGH" fi - echoinfo "${CALLER} ${0} -- Version ${__ScriptVersion}" -#echowarn "Running the unstable version of ${__ScriptName}" +echowarn "Running the unstable version of ${__ScriptName}" #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __exit_cleanup @@ -680,6 +680,9 @@ __gather_linux_system_info() { elif [ "${DISTRO_NAME}" = "EnterpriseEnterpriseServer" ]; then # This the Oracle Linux Enterprise ID before ORACLE LINUX 5 UPDATE 3 DISTRO_NAME="Oracle Linux" + elif [ "${DISTRO_NAME}" = "OracleServer" ]; then + # This the Oracle Linux Server 6.5 + DISTRO_NAME="Oracle Linux" fi rv=$(lsb_release -sr) [ "${rv}x" != "x" ] && DISTRO_VERSION=$(__parse_version_string "$rv") @@ -1055,7 +1058,7 @@ fi # Only RedHat based distros have testing support if [ ${ITYPE} = "testing" ]; then - if [ "$(echo ${DISTRO_NAME_L} | egrep '(centos|red_hat|amazon)')x" = "x" ]; then + if [ "$(echo ${DISTRO_NAME_L} | egrep '(centos|red_hat|amazon|oracle)')x" = "x" ]; then echoerror "${DISTRO_NAME} does not have testing packages support" exit 1 fi @@ -2380,7 +2383,14 @@ install_centos_stable_deps() { fi fi - yum -y install ${packages} --enablerepo=${_EPEL_REPO} || return 1 + if [ $DISTRO_NAME_L = "oracle_linux" ]; then + # We need to install one package at a time because --enablerepo=X disables ALL OTHER REPOS!!!! + for package in ${packages}; do + yum -y install ${package} || yum -y install ${package} --enablerepo=${_EPEL_REPO} || return 1 + done + else + yum -y install ${packages} --enablerepo=${_EPEL_REPO} || return 1 + fi if [ $_INSTALL_CLOUD -eq $BS_TRUE ]; then check_pip_allowed "You need to allow pip based installations (-P) in order to install apache-libcloud" @@ -2393,7 +2403,14 @@ install_centos_stable_deps() { if [ "x${_EXTRA_PACKAGES}" != "x" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" - yum install -y ${_EXTRA_PACKAGES} --enablerepo=${_EPEL_REPO} || return 1 + if [ $DISTRO_NAME_L = "oracle_linux" ]; then + # We need to install one package at a time because --enablerepo=X disables ALL OTHER REPOS!!!! + for package in ${_EXTRA_PACKAGES}; do + yum -y install ${package} || yum -y install ${package} --enablerepo=${_EPEL_REPO} || return 1 + done + else + yum install -y ${_EXTRA_PACKAGES} --enablerepo=${_EPEL_REPO} || return 1 + fi fi return 0 @@ -2407,7 +2424,14 @@ install_centos_stable() { if [ $_INSTALL_MASTER -eq $BS_TRUE ] || [ $_INSTALL_SYNDIC -eq $BS_TRUE ]; then packages="${packages} salt-master" fi - yum -y install ${packages} --enablerepo=${_EPEL_REPO} || return 1 + if [ $DISTRO_NAME_L = "oracle_linux" ]; then + # We need to install one package at a time because --enablerepo=X disables ALL OTHER REPOS!!!! + for package in ${packages}; do + yum -y install ${package} || yum -y install ${package} --enablerepo=${_EPEL_REPO} || return 1 + done + else + yum -y install ${packages} --enablerepo=${_EPEL_REPO} || return 1 + fi return 0 } @@ -2427,7 +2451,12 @@ install_centos_stable_post() { install_centos_git_deps() { install_centos_stable_deps || return 1 - yum -y install git --enablerepo=${_EPEL_REPO} || return 1 + if [ $DISTRO_NAME_L = "oracle_linux" ]; then + # try both ways --enablerepo=X disables ALL OTHER REPOS!!!! + yum -y install git || yum -y install git --enablerepo=${_EPEL_REPO} || return 1 + else + yum -y install git --enablerepo=${_EPEL_REPO} || return 1 + fi __git_clone_and_checkout || return 1 @@ -3409,11 +3438,11 @@ install_smartos_deps() { # Let's download, since they were not provided, the default configuration files if [ ! -f $_SALT_ETC_DIR/minion ] && [ ! -f $_TEMP_CONFIG_DIR/minion ]; then curl $_CURL_ARGS -s -o $_TEMP_CONFIG_DIR/minion -L \ - https://raw.github.com/saltstack/salt/develop/conf/minion || return 1 + https://raw.githubusercontent.com/saltstack/salt/develop/conf/minion || return 1 fi if [ ! -f $_SALT_ETC_DIR/master ] && [ ! -f $_TEMP_CONFIG_DIR/master ]; then curl $_CURL_ARGS -s -o $_TEMP_CONFIG_DIR/master -L \ - https://raw.github.com/saltstack/salt/develop/conf/master || return 1 + https://raw.githubusercontent.com/saltstack/salt/develop/conf/master || return 1 fi fi @@ -3465,7 +3494,7 @@ install_smartos_post() { if [ $? -eq 1 ]; then if [ ! -f $_TEMP_CONFIG_DIR/salt-$fname.xml ]; then curl $_CURL_ARGS -s -o $_TEMP_CONFIG_DIR/salt-$fname.xml -L \ - https://raw.github.com/saltstack/salt/develop/pkg/smartos/salt-$fname.xml + https://raw.githubusercontent.com/saltstack/salt/develop/pkg/smartos/salt-$fname.xml fi svccfg import $_TEMP_CONFIG_DIR/salt-$fname.xml if [ "${VIRTUAL_TYPE}" = "global" ]; then @@ -3761,7 +3790,7 @@ install_suse_11_stable_deps() { # Let's download, since they were not provided, the default configuration files if [ ! -f $_SALT_ETC_DIR/$fname ] && [ ! -f $_TEMP_CONFIG_DIR/$fname ]; then curl $_CURL_ARGS -s -o $_TEMP_CONFIG_DIR/$fname -L \ - https://raw.github.com/saltstack/salt/develop/conf/$fname || return 1 + https://raw.githubusercontent.com/saltstack/salt/develop/conf/$fname || return 1 fi done fi diff --git a/setup.py b/setup.py index 21948889c2..3495f08088 100755 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ BOOTSTRAP_SCRIPT_DISTRIBUTED_VERSION = os.environ.get( 'BOOTSTRAP_SCRIPT_VERSION', # If no bootstrap-script version was provided from the environment, let's # provide the one we define. - 'v2014.06.19' + 'v2014.06.21' ) # Store a reference to the executing platform From c952a4d61b81f5bfa19c744a21f25b914f04fabc Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 21 Jun 2014 15:33:11 +0100 Subject: [PATCH 24/55] Remove unused import. White-space lint fix --- tests/integration/loader/globals.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/loader/globals.py b/tests/integration/loader/globals.py index e3632fc8c7..8a5b2c1502 100644 --- a/tests/integration/loader/globals.py +++ b/tests/integration/loader/globals.py @@ -7,7 +7,6 @@ ''' # Import Salt Testing libs -from salttesting import TestCase from salttesting.helpers import ensure_in_syspath ensure_in_syspath('../') @@ -131,7 +130,6 @@ class LoaderGlobalsTest(integration.ModuleCase): self._verify_globals(salt.loader.render(self.master_opts, {})) - if __name__ == '__main__': from integration import run_tests run_tests(LoaderGlobalsTest, needs_daemon=False) From 55cd462fa7ae3b6046a07b756dcdd706fa5d7ab7 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 21 Jun 2014 15:34:33 +0100 Subject: [PATCH 25/55] No need to iterate through keys --- tests/integration/loader/globals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/loader/globals.py b/tests/integration/loader/globals.py index 8a5b2c1502..d3cea2fd3f 100644 --- a/tests/integration/loader/globals.py +++ b/tests/integration/loader/globals.py @@ -32,7 +32,7 @@ class LoaderGlobalsTest(integration.ModuleCase): ''' # find the globals global_vars = None - for key, val in mod_dict.iteritems(): + for val in mod_dict.itervalues(): if hasattr(val, '__globals__'): global_vars = val.__globals__ break From 94b1ab21ac5ed27ea08ea099a9f3ab64c89c142f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 21 Jun 2014 16:08:50 +0100 Subject: [PATCH 26/55] The stable version! --- salt/cloud/deploy/bootstrap-salt.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/cloud/deploy/bootstrap-salt.sh b/salt/cloud/deploy/bootstrap-salt.sh index 9dd24657a0..19a00e47c0 100755 --- a/salt/cloud/deploy/bootstrap-salt.sh +++ b/salt/cloud/deploy/bootstrap-salt.sh @@ -428,8 +428,9 @@ CALLER=$(ps -a -o pid,args | grep $$ | grep -v grep | tr -s ' ' | cut -d ' ' -f if [ "${CALLER}x" = "${0}x" ]; then CALLER="PIPED THROUGH" fi + echoinfo "${CALLER} ${0} -- Version ${__ScriptVersion}" -echowarn "Running the unstable version of ${__ScriptName}" +#echowarn "Running the unstable version of ${__ScriptName}" #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __exit_cleanup From 1f0e0d44652ece5eebbde15a7028937fc3a6d1dc Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 21 Jun 2014 19:00:55 +0100 Subject: [PATCH 27/55] Remove `doc/man/salt-api.7` --- doc/man/salt-api.7 | 2237 -------------------------------------------- 1 file changed, 2237 deletions(-) delete mode 100644 doc/man/salt-api.7 diff --git a/doc/man/salt-api.7 b/doc/man/salt-api.7 deleted file mode 100644 index 721e46993b..0000000000 --- a/doc/man/salt-api.7 +++ /dev/null @@ -1,2237 +0,0 @@ -.\" Man page generated from reStructuredText. -. -.TH "SALT-API" "7" "April 05, 2014" "0.8.3" "salt-api" -.SH NAME -salt-api \- salt-api Documentation -. -.nr rst2man-indent-level 0 -. -.de1 rstReportMargin -\\$1 \\n[an-margin] -level \\n[rst2man-indent-level] -level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] -- -\\n[rst2man-indent0] -\\n[rst2man-indent1] -\\n[rst2man-indent2] -.. -.de1 INDENT -.\" .rstReportMargin pre: -. RS \\$1 -. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] -. nr rst2man-indent-level +1 -.\" .rstReportMargin post: -.. -.de UNINDENT -. RE -.\" indent \\n[an-margin] -.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] -.nr rst2man-indent-level -1 -.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] -.in \\n[rst2man-indent\\n[rst2man-indent-level]]u -.. -.sp -\fBsalt\-api\fP is a modular interface on top of \fI\%Salt\fP that can provide a -variety of entry points into a running Salt system. It can start and manage -multiple interfaces allowing a REST API to coexist with XMLRPC or even a -Websocket API. -.SH GETTING STARTED -.INDENT 0.0 -.IP 1. 3 -Install \fBsalt\-api\fP on the same machine as your Salt master. -.IP 2. 3 -Edit your Salt master config file for all required options for each -\fBnetapi\fP module you wish to run. -.IP 3. 3 -Install any required additional libraries or software for each \fBnetapi\fP -module you wish to run. -.IP 4. 3 -Run \fBsalt\-api\fP which will then start all configured \fBnetapi\fP -modules. -.UNINDENT -.sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -Each \fBnetapi\fP module will have differing configuration requirements and -differing required software libraries. -.sp -Exactly like the various module types in Salt (\fIexecution modules\fP, -\fIrenderer modules\fP, \fIreturner modules\fP, etc.), \fInetapi -modules\fP in \fBsalt\-api\fP will \fInot\fP be loaded into memory or started -if all requirements are not met. -.UNINDENT -.UNINDENT -.SH INSTALLATION QUICKSTART -.SS salt\-api Quickstart -.sp -\fBsalt\-api\fP manages \fInetapi modules\fP which are modules that -(usually) bind to a port and start a service. Each netapi module will have -specific requirements for third\-party libraries and configuration (which goes -in the master config file). Read the documentation for each netapi module to -determine what is needed. -.sp -For example, the \fBrest_cherrypy\fP -netapi module requires that CherryPy be installed and that a \fBrest_cherrypy\fP -section be added to the master config that specifies which port to listen on. -.SS Installation -.SS PyPI -.sp -\fI\%https://pypi.python.org/pypi/salt\-api\fP -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -pip install salt\-api -.ft P -.fi -.UNINDENT -.UNINDENT -.SS RHEL, Fedora, CentOS -.sp -RPMs are available in the Fedora repositories and EPEL: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -yum install salt\-api -.ft P -.fi -.UNINDENT -.UNINDENT -.SS Ubuntu -.sp -PPA packages available for Ubuntu on LaunchPad: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -sudo add\-apt\-repository ppa:saltstack/salt -sudo apt\-get update -sudo apt\-get install salt\-api -.ft P -.fi -.UNINDENT -.UNINDENT -.SS openSUSE, SLES -.sp -RPMs are available via the OBS: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -zypper install salt\-api -.ft P -.fi -.UNINDENT -.UNINDENT -.SH NETAPI MODULES -.sp -The core functionality for \fBsalt\-api\fP lies in pluggable \fBnetapi\fP -modules that adhere to the simple interface of binding to a port and starting a -service. \fBsalt\-api\fP can manage one or many services concurrently. -.SS Full list of \fBnetapi\fP modules -.SS Full list of netapi modules -.SS rest_cherrypy -.SS A REST API for Salt -.INDENT 0.0 -.TP -.B depends -.INDENT 7.0 -.IP \(bu 2 -CherryPy Python module -.UNINDENT -.TP -.B configuration -All authentication is done through Salt\(aqs \fI\%external auth\fP system. Be sure that it is enabled and the user you are -authenticating as has permissions for all the functions you will be -running. -.sp -Example production configuration block; add to the Salt master config file: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -rest_cherrypy: - port: 8000 - ssl_crt: /etc/pki/tls/certs/localhost.crt - ssl_key: /etc/pki/tls/certs/localhost.key -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -The REST interface strongly recommends a secure HTTPS connection since Salt -authentication credentials will be sent over the wire. If you don\(aqt already -have a certificate and don\(aqt wish to buy one, you can generate a -self\-signed certificate using the -\fI\%create_self_signed_cert()\fP function in Salt (note -the dependencies for this module): -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% salt\-call tls.create_self_signed_cert -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -All available configuration options are detailed below. These settings -configure the CherryPy HTTP server and do not apply when using an external -server such as Apache or Nginx. -.INDENT 7.0 -.TP -.B port -\fBRequired\fP -.sp -The port for the webserver to listen on. -.TP -.B host -\fB0.0.0.0\fP -The socket interface for the HTTP server to listen on. -.sp -New in version 0.8.2. - -.TP -.B debug -\fBFalse\fP -Starts the web server in development mode. It will reload itself when -the underlying code is changed and will output more debugging info. -.TP -.B ssl_crt -The path to a SSL certificate. (See below) -.TP -.B ssl_key -The path to the private key for your SSL certificate. (See below) -.TP -.B disable_ssl -A flag to disable SSL. Warning: your Salt authentication credentials -will be sent in the clear! -.sp -New in version 0.8.3. - -.TP -.B thread_pool -\fB100\fP -The number of worker threads to start up in the pool. -.sp -Changed in version 0.8.4: Previous versions defaulted to a pool of \fB10\fP - -.TP -.B socket_queue_size -\fB30\fP -Specify the maximum number of HTTP connections to queue. -.sp -Changed in version 0.8.4: Previous versions defaulted to \fB5\fP connections. - -.TP -.B max_request_body_size -\fB1048576\fP -Changed in version 0.8.4: Previous versions defaulted to \fB104857600\fP for the size of the -request body - -.TP -.B collect_stats -False -Collect and report statistics about the CherryPy server -.sp -New in version 0.8.4. - -.sp -Reports are available via the \fBStats\fP URL. -.TP -.B static -A filesystem path to static HTML/JavaScript/CSS/image assets. -.TP -.B static_path -\fB/static\fP -The URL prefix to use when serving static assets out of the directory -specified in the \fBstatic\fP setting. -.sp -New in version 0.8.2. - -.TP -.B app -A filesystem path to an HTML file that will be served as a static file. -This is useful for bootstrapping a single\-page JavaScript app. -.sp -New in version 0.8.2. - -.TP -.B app_path -\fB/app\fP -The URL prefix to use for serving the HTML file specified in the \fBapp\fP -setting. This should be a simple name containing no slashes. -.sp -Any path information after the specified path is ignored; this is -useful for apps that utilize the HTML5 history API. -.sp -New in version 0.8.2. - -.TP -.B root_prefix -\fB/\fP -A URL path to the main entry point for the application. This is useful -for serving multiple applications from the same URL. -.sp -New in version 0.8.4. - -.UNINDENT -.UNINDENT -.SS Authentication -.sp -Authentication is performed by passing a session token with each request. The -token may be sent either via a custom header named \fIX\-Auth\-Token\fP -or sent inside a cookie. (The result is the same but browsers and some HTTP -clients handle cookies automatically and transparently so it is a convenience.) -.sp -Token are generated via the \fBLogin\fP URL. -.sp -\fBSEE ALSO:\fP -.INDENT 0.0 -.INDENT 3.5 -You can bypass the session handling via the \fBRun\fP URL. -.UNINDENT -.UNINDENT -.SS Usage -.sp -You access a running Salt master via this module by sending HTTP requests to -the URLs detailed below. -.INDENT 0.0 -.INDENT 3.5 -.IP "Content negotiation" -.sp -This REST interface is flexible in what data formats it will accept as well -as what formats it will return (e.g., JSON, YAML, x\-www\-form\-urlencoded). -.INDENT 0.0 -.IP \(bu 2 -Specify the format of data you are sending in a request by including the -\fIContent\-Type\fP header. -.IP \(bu 2 -Specify your desired output format for the response with the -\fIAccept\fP header. -.UNINDENT -.UNINDENT -.UNINDENT -.sp -This REST interface expects data sent in \fI\%POST\fP and -\fI\%PUT\fP requests to be in the format of a list of lowstate -dictionaries. This allows you to specify multiple commands in a single request. -.INDENT 0.0 -.TP -.B lowstate -A dictionary containing various keys that instruct Salt which command -to run, where that command lives, any parameters for that command, any -authentication credentials, what returner to use, etc. -.sp -Salt uses the lowstate data format internally in many places to pass -command data between functions. Salt also uses lowstate for the -\fI\%LocalClient()\fP Python API interface. -.UNINDENT -.sp -For example (in JSON format): -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -[{ - \(aqclient\(aq: \(aqlocal\(aq, - \(aqtgt\(aq: \(aq*\(aq, - \(aqfun\(aq: \(aqtest.fib\(aq, - \(aqarg\(aq: [\(aq10\(aq], -}] -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 0.0 -.INDENT 3.5 -.IP "x\-www\-form\-urlencoded" -.sp -This REST interface accepts data in the x\-www\-form\-urlencoded format. This -is the format used by HTML forms, the default format used by -\fBcurl\fP, the default format used by many JavaScript AJAX libraries -(such as jQuery), etc. This format will be converted to the -\fIlowstate\fP format as best as possible with the caveats below. It is -always preferable to format data in the lowstate format directly in a more -capable format such as JSON or YAML. -.INDENT 0.0 -.IP \(bu 2 -Only a single command may be sent in this format per HTTP request. -.IP \(bu 2 -Multiple \fBarg\fP params will be sent as a single list of params. -.sp -Note, some popular frameworks and languages (notably jQuery, PHP, and -Ruby on Rails) will automatically append empty brackets onto repeated -parameters. E.g., arg=one, arg=two will be sent as arg[]=one, arg[]=two. -Again, it is preferable to send lowstate via JSON or YAML directly by -specifying the \fIContent\-Type\fP header in the request. -.UNINDENT -.UNINDENT -.UNINDENT -.SS URL reference -.sp -The main entry point is the \fBroot URL (/)\fP and all -functionality is available at that URL. The other URLs are largely convenience -URLs that wrap that main entry point with shorthand or specialized -functionality. -.SS Deployment -.sp -The \fBrest_cherrypy\fP netapi module is a standard Python WSGI app. It can be -deployed one of two ways. -.SS \fBsalt\-api\fP using the CherryPy server -.sp -The default configuration is to run this module using \fBsalt\-api\fP to -start the Python\-based CherryPy server. This server is lightweight, -multi\-threaded, encrypted with SSL, and should be considered production\-ready. -.SS Using a WSGI\-compliant web server -.sp -This module may be deplayed on any WSGI\-compliant server such as Apache with -mod_wsgi or Nginx with FastCGI, to name just two (there are many). -.sp -Note, external WSGI servers handle URLs, paths, and SSL certs directly. The -\fBrest_cherrypy\fP configuration options are ignored and the \fBsalt\-api\fP daemon -does not need to be running at all. Remember Salt authentication credentials -are sent in the clear unless SSL is being enforced! -.sp -An example Apache virtual host configuration: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C - - ServerName example.com - ServerAlias *.example.com - - ServerAdmin webmaster@example.com - - LogLevel warn - ErrorLog /var/www/example.com/logs/error.log - CustomLog /var/www/example.com/logs/access.log combined - - DocumentRoot /var/www/example.com/htdocs - - WSGIScriptAlias / /path/to/saltapi/netapi/rest_cherrypy/wsgi.py - -.ft P -.fi -.UNINDENT -.UNINDENT -.SS REST URI Reference -.INDENT 0.0 -.TP -.B class saltapi.netapi.rest_cherrypy.app.LowDataAdapter -The primary entry point to the REST API. All functionality is available -through this URL. The other available URLs provide convenience wrappers -around this URL. -.INDENT 7.0 -.TP -.B GET() -.INDENT 7.0 -.TP -.B GET / -An explanation of the API with links of where to go next. -.sp -\fBExample request\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-i localhost:8000 -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -GET / HTTP/1.1 -Host: localhost:8000 -Accept: application/json -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample response\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -HTTP/1.1 200 OK -Content\-Type: application/json -.ft P -.fi -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B Status 200 -success -.TP -.B Status 401 -authentication required -.TP -.B Status 406 -requested Content\-Type not available -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 0.0 -.TP -.B class saltapi.netapi.rest_cherrypy.app.Login(*args, **kwargs) -All interactions with this REST API must be authenticated. Authentication -is performed through Salt\(aqs eauth system. You must set the eauth backend -and allowed users by editing the \fI\%external_auth\fP section in -your master config. -.sp -Authentication credentials are passed to the REST API via a session id in -one of two ways: -.sp -If the request is initiated from a browser it must pass a session id via a -cookie and that session must be valid and active. -.sp -If the request is initiated programmatically, the request must contain a -\fIX\-Auth\-Token\fP header with valid and active session id. -.INDENT 7.0 -.TP -.B GET() -Present the login interface -.INDENT 7.0 -.TP -.B GET /login -An explanation of how to log in. -.sp -\fBExample request\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-i localhost:8000/login -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -GET /login HTTP/1.1 -Host: localhost:8000 -Accept: text/html -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample response\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -HTTP/1.1 200 OK -Content\-Type: text/html -.ft P -.fi -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B Status 401 -authentication required -.TP -.B Status 406 -requested Content\-Type not available -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B POST(**kwargs) -Authenticate against Salt\(aqs eauth system -.sp -Changed in version 0.8.0: No longer returns a 302 redirect on success. - -.sp -Changed in version 0.8.1: Returns 401 on authentication failure - -.INDENT 7.0 -.TP -.B POST /login -\fBExample request\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-si localhost:8000/login \e - \-H "Accept: application/json" \e - \-d username=\(aqsaltuser\(aq \e - \-d password=\(aqsaltpass\(aq \e - \-d eauth=\(aqpam\(aq -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -POST / HTTP/1.1 -Host: localhost:8000 -Content\-Length: 42 -Content\-Type: application/x\-www\-form\-urlencoded -Accept: application/json - -username=saltuser&password=saltpass&eauth=pam -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample response\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -HTTP/1.1 200 OK -Content\-Type: application/json -Content\-Length: 206 -X\-Auth\-Token: 6d1b722e -Set\-Cookie: session_id=6d1b722e; expires=Sat, 17 Nov 2012 03:23:52 GMT; Path=/ - -{"return": { - "token": "6d1b722e", - "start": 1363805943.776223, - "expire": 1363849143.776224, - "user": "saltuser", - "eauth": "pam", - "perms": [ - "grains.*", - "status.*", - "sys.*", - "test.*" - ] -}} -.ft P -.fi -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B Form eauth -the eauth backend configured in your master config -.TP -.B Form username -username -.TP -.B Form password -password -.TP -.B Status 200 -success -.TP -.B Status 401 -could not authenticate using provided credentials -.TP -.B Status 406 -requested Content\-Type not available -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 0.0 -.TP -.B class saltapi.netapi.rest_cherrypy.app.Logout -.INDENT 7.0 -.TP -.B POST() -Destroy the currently active session and expire the session cookie -.sp -New in version 0.8.0. - -.UNINDENT -.UNINDENT -.INDENT 0.0 -.TP -.B class saltapi.netapi.rest_cherrypy.app.Minions -.INDENT 7.0 -.TP -.B GET(mid=None) -A convenience URL for getting lists of minions or getting minion -details -.INDENT 7.0 -.TP -.B GET /minions/(mid) -Get grains, modules, functions, and inline function documentation -for all minions or a single minion -.sp -\fBExample request\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-i localhost:8000/minions/ms\-3 -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -GET /minions/ms\-3 HTTP/1.1 -Host: localhost:8000 -Accept: application/x\-yaml -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample response\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -HTTP/1.1 200 OK -Content\-Length: 129005 -Content\-Type: application/x\-yaml - -return: -\- ms\-3: - grains.items: - ... -.ft P -.fi -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B Parameters -\fBmid\fP \-\- (optional) a minion id -.TP -.B Status 200 -success -.TP -.B Status 401 -authentication required -.TP -.B Status 406 -requested Content\-Type not available -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B POST(**kwargs) -Start an execution command and immediately return the job id -.INDENT 7.0 -.TP -.B POST /minions -You must pass low\-data in the request body either from an HTML form -or as JSON or YAML. The \fBclient\fP option is pre\-set to -\fBlocal_async\fP\&. -.sp -\fBExample request\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-sSi localhost:8000/minions \e - \-H "Accept: application/x\-yaml" \e - \-d tgt=\(aq*\(aq \e - \-d fun=\(aqstatus.diskusage\(aq -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -POST /minions HTTP/1.1 -Host: localhost:8000 -Accept: application/x\-yaml -Content\-Length: 26 -Content\-Type: application/x\-www\-form\-urlencoded - -tgt=*&fun=status.diskusage -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample response\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -HTTP/1.1 202 Accepted -Content\-Length: 86 -Content\-Type: application/x\-yaml - -return: -\- jid: \(aq20130603122505459265\(aq - minions: [ms\-4, ms\-3, ms\-2, ms\-1, ms\-0] -_links: - jobs: - \- href: /jobs/20130603122505459265 -.ft P -.fi -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B Form lowstate -lowstate data for the -\fBLocalClient\fP; the \fBclient\fP parameter will -be set to \fBlocal_async\fP -.sp -Lowstate may be supplied in any supported format by specifying the -\fIContent\-Type\fP header in the request. Supported formats -are listed in the \fIAlternates\fP response header. -.TP -.B Status 202 -success -.TP -.B Status 401 -authentication required -.TP -.B Status 406 -requested \fIContent\-Type\fP not available -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 0.0 -.TP -.B class saltapi.netapi.rest_cherrypy.app.Jobs -.INDENT 7.0 -.TP -.B GET(jid=None) -A convenience URL for getting lists of previously run jobs or getting -the return from a single job -.INDENT 7.0 -.TP -.B GET /jobs/(jid) -Get grains, modules, functions, and inline function documentation -for all minions or a single minion -.sp -\fBExample request\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-i localhost:8000/jobs -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -GET /jobs HTTP/1.1 -Host: localhost:8000 -Accept: application/x\-yaml -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample response\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -HTTP/1.1 200 OK -Content\-Length: 165 -Content\-Type: application/x\-yaml - -return: -\- \(aq20121130104633606931\(aq: - Arguments: - \- \(aq3\(aq - Function: test.fib - Start Time: 2012, Nov 30 10:46:33.606931 - Target: jerry - Target\-type: glob -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample request\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-i localhost:8000/jobs/20121130104633606931 -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -GET /jobs/20121130104633606931 HTTP/1.1 -Host: localhost:8000 -Accept: application/x\-yaml -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample response\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -HTTP/1.1 200 OK -Content\-Length: 73 -Content\-Type: application/x\-yaml - -info: -\- Arguments: - \- \(aq3\(aq - Function: test.fib - Minions: - \- jerry - Start Time: 2012, Nov 30 10:46:33.606931 - Target: \(aq*\(aq - Target\-type: glob - User: saltdev - jid: \(aq20121130104633606931\(aq -return: -\- jerry: - \- \- 0 - \- 1 - \- 1 - \- 2 - \- 6.9141387939453125e\-06 -.ft P -.fi -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B Parameters -\fBmid\fP \-\- (optional) a minion id -.TP -.B Status 200 -success -.TP -.B Status 401 -authentication required -.TP -.B Status 406 -requested Content\-Type not available -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 0.0 -.TP -.B class saltapi.netapi.rest_cherrypy.app.Run -.INDENT 7.0 -.TP -.B POST(**kwargs) -Run commands bypassing the normal session handling -.sp -New in version 0.8.0. - -.INDENT 7.0 -.TP -.B POST /run -This entry point is primarily for "one\-off" commands. Each request -must pass full Salt authentication credentials. Otherwise this URL -is identical to the root (\fB/\fP) execution URL. -.sp -\fBExample request\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-sS localhost:8000/run \e - \-H \(aqAccept: application/x\-yaml\(aq \e - \-d client=\(aqlocal\(aq \e - \-d tgt=\(aq*\(aq \e - \-d fun=\(aqtest.ping\(aq \e - \-d username=\(aqsaltdev\(aq \e - \-d password=\(aqsaltdev\(aq \e - \-d eauth=\(aqpam\(aq -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -POST /run HTTP/1.1 -Host: localhost:8000 -Accept: application/x\-yaml -Content\-Length: 75 -Content\-Type: application/x\-www\-form\-urlencoded - -client=local&tgt=*&fun=test.ping&username=saltdev&password=saltdev&eauth=pam -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample response\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -HTTP/1.1 200 OK -Content\-Length: 73 -Content\-Type: application/x\-yaml - -return: -\- ms\-0: true - ms\-1: true - ms\-2: true - ms\-3: true - ms\-4: true -.ft P -.fi -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B Form lowstate -A list of \fIlowstate\fP data appropriate for the -\fI\%client\fP specified client interface. Full -external authentication credentials must be included. -.TP -.B Status 200 -success -.TP -.B Status 401 -authentication failed -.TP -.B Status 406 -requested Content\-Type not available -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 0.0 -.TP -.B class saltapi.netapi.rest_cherrypy.app.Events -The event bus on the Salt master exposes a large variety of things, notably -when executions are started on the master and also when minions ultimately -return their results. This URL provides a real\-time window into a running -Salt infrastructure. -.INDENT 7.0 -.TP -.B GET(token=None) -Return an HTTP stream of the Salt master event bus; this stream is -formatted per the Server Sent Events (SSE) spec -.sp -New in version 0.8.3. - -.sp -Browser clients currently lack Cross\-origin resource sharing (CORS) -support for the \fBEventSource()\fP API. Cross\-domain requests from a -browser may instead pass the \fIX\-Auth\-Token\fP value as an URL -parameter: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-NsS localhost:8000/events/6d1b722e -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B GET /events -\fBExample request\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-NsS localhost:8000/events -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -GET /events HTTP/1.1 -Host: localhost:8000 -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample response\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -HTTP/1.1 200 OK -Connection: keep\-alive -Cache\-Control: no\-cache -Content\-Type: text/event\-stream;charset=utf\-8 - -retry: 400 -data: {\(aqtag\(aq: \(aq\(aq, \(aqdata\(aq: {\(aqminions\(aq: [\(aqms\-4\(aq, \(aqms\-3\(aq, \(aqms\-2\(aq, \(aqms\-1\(aq, \(aqms\-0\(aq]}} - -data: {\(aqtag\(aq: \(aq20130802115730568475\(aq, \(aqdata\(aq: {\(aqjid\(aq: \(aq20130802115730568475\(aq, \(aqreturn\(aq: True, \(aqretcode\(aq: 0, \(aqsuccess\(aq: True, \(aqcmd\(aq: \(aq_return\(aq, \(aqfun\(aq: \(aqtest.ping\(aq, \(aqid\(aq: \(aqms\-1\(aq}} -.ft P -.fi -.UNINDENT -.UNINDENT -.UNINDENT -.sp -The event stream can be easily consumed via JavaScript: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -# Note, you must be authenticated! -var source = new EventSource(\(aq/events\(aq); -source.onopen = function() { console.debug(\(aqopening\(aq) }; -source.onerror = function(e) { console.debug(\(aqerror!\(aq, e) }; -source.onmessage = function(e) { console.debug(e.data) }; -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -It is also possible to consume the stream via the shell. -.sp -Records are separated by blank lines; the \fBdata:\fP and \fBtag:\fP -prefixes will need to be removed manually before attempting to -unserialize the JSON. -.sp -curl\(aqs \fB\-N\fP flag turns off input buffering which is required to -process the stream incrementally. -.sp -Here is a basic example of printing each event as it comes in: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-NsS localhost:8000/events |\e - while IFS= read \-r line ; do - echo $line - done -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -Here is an example of using awk to filter events based on tag: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-NsS localhost:8000/events |\e - awk \(aq - BEGIN { RS=""; FS="\en" } - $1 ~ /^tag: salt\e/job\e/[0\-9]+\e/new$/ { print $0 } - \(aq -tag: salt/job/20140112010149808995/new -data: {"tag": "salt/job/20140112010149808995/new", "data": {"tgt_type": "glob", "jid": "20140112010149808995", "tgt": "jerry", "_stamp": "2014\-01\-12_01:01:49.809617", "user": "shouse", "arg": [], "fun": "test.ping", "minions": ["jerry"]}} -tag: 20140112010149808995 -data: {"tag": "20140112010149808995", "data": {"fun_args": [], "jid": "20140112010149808995", "return": true, "retcode": 0, "success": true, "cmd": "_return", "_stamp": "2014\-01\-12_01:01:49.819316", "fun": "test.ping", "id": "jerry"}} -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B Status 200 -success -.TP -.B Status 401 -could not authenticate using provided credentials -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 0.0 -.TP -.B class saltapi.netapi.rest_cherrypy.app.Webhook -A generic web hook entry point that fires an event on Salt\(aqs event bus -.sp -External services can POST data to this URL to trigger an event in Salt. -For example, Jenkins\-CI or Travis\-CI, or GitHub web hooks. -.sp -This entry point does not require authentication. The event data is taken -from the request body. -.sp -\fBNOTE:\fP -.INDENT 7.0 -.INDENT 3.5 -Be mindful of security -.sp -Salt\(aqs Reactor can run any code. If you write a Reactor SLS that -responds to a hook event be sure to validate that the event came from a -trusted source and contains valid data! Pass a secret key and use SSL. -.sp -This is a generic interface and securing it is up to you! -.UNINDENT -.UNINDENT -.sp -The event tag is prefixed with \fBsalt/netapi/hook\fP and the URL path is -appended to the end. For example, a \fBPOST\fP request sent to -\fB/hook/mycompany/myapp/mydata\fP will produce a Salt event with the tag -\fBsalt/netapi/hook/mycompany/myapp/mydata\fP\&. See the \fI\%Salt Reactor\fP documentation for how to react to events with various tags. -.sp -The following is an example \fB\&.travis.yml\fP file to send notifications to -Salt of successful test runs: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -language: python -script: python \-m unittest tests -after_success: - \- \(aqcurl \-sS http://saltapi\-url.example.com:8000/hook/travis/build/success \-d branch="${TRAVIS_BRANCH}" \-d commit="${TRAVIS_COMMIT}"\(aq -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B POST(*args, **kwargs) -Fire an event in Salt with a custom event tag and data -.sp -New in version 0.8.4. - -.INDENT 7.0 -.TP -.B POST /hook -\fBExample request\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-sS localhost:8000/hook \-d foo=\(aqFoo!\(aq \-d bar=\(aqBar!\(aq -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -POST /hook HTTP/1.1 -Host: localhost:8000 -Content\-Length: 16 -Content\-Type: application/x\-www\-form\-urlencoded - -foo=Foo&bar=Bar! -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample response\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -HTTP/1.1 200 OK -Content\-Length: 14 -Content\-Type: application/json - -{"success": true} -.ft P -.fi -.UNINDENT -.UNINDENT -.UNINDENT -.sp -As a practical example, an internal continuous\-integration build -server could send an HTTP POST request to the URL -\fBhttp://localhost:8000/hook/mycompany/build/success\fP which contains -the result of a build and the SHA of the version that was built as -JSON. That would then produce the following event in Salt that could be -used to kick off a deployment via Salt\(aqs Reactor: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -Event fired at Fri Feb 14 17:40:11 2014 -************************* -Tag: salt/netapi/hook/mycompany/build/success -Data: -{\(aq_stamp\(aq: \(aq2014\-02\-14_17:40:11.440996\(aq, - \(aqheaders\(aq: { - \(aqX\-My\-Secret\-Key\(aq: \(aqF0fAgoQjIT@W\(aq, - \(aqContent\-Length\(aq: \(aq37\(aq, - \(aqContent\-Type\(aq: \(aqapplication/json\(aq, - \(aqHost\(aq: \(aqlocalhost:8000\(aq, - \(aqRemote\-Addr\(aq: \(aq127.0.0.1\(aq}, - \(aqpost\(aq: {\(aqrevision\(aq: \(aqaa22a3c4b2e7\(aq, \(aqresult\(aq: True}} -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -Salt\(aqs Reactor could listen for the event: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -reactor: - \- \(aqsalt/netapi/hook/mycompany/build/*\(aq: - \- /srv/reactor/react_ci_builds.sls -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -And finally deploy the new build: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -{% set secret_key = data.get(\(aqheaders\(aq, {}).get(\(aqX\-My\-Secret\-Key\(aq) %} -{% set build = data.get(\(aqpost\(aq, {}) %} - -{% if secret_key == \(aqF0fAgoQjIT@W\(aq and build.result == True %} -deploy_my_app: - cmd.state.sls: - \- tgt: \(aqapplication*\(aq - \- arg: - \- myapp.deploy - \- \(aqpillar={revision: {{ revision }}}\(aq -{% endif %} -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B Status 200 -success -.TP -.B Status 406 -requested Content\-Type not available -.TP -.B Status 413 -request body is too large -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 0.0 -.TP -.B class saltapi.netapi.rest_cherrypy.app.Stats -Expose statistics on the running CherryPy server -.INDENT 7.0 -.TP -.B GET() -Return a dump of statistics collected from the CherryPy server -.INDENT 7.0 -.TP -.B Status 200 -success -.TP -.B Status 406 -requested Content\-Type not available -.UNINDENT -.UNINDENT -.UNINDENT -.SS rest_wsgi -.SS A minimalist REST API for Salt -.sp -This \fBrest_wsgi\fP module provides a no\-frills REST interface to a running Salt -master. There are no dependencies. -.sp -Please read this introductory section in entirety before deploying this module. -.INDENT 0.0 -.TP -.B configuration -All authentication is done through Salt\(aqs \fI\%external auth\fP system. Be sure that it is enabled and the user you are -authenticating as has permissions for all the functions you will be -running. -.sp -The configuration options for this module resides in the Salt master config -file. All available options are detailed below. -.INDENT 7.0 -.TP -.B port -\fBRequired\fP -.sp -The port for the webserver to listen on. -.UNINDENT -.sp -Example configuration: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -rest_wsgi: - port: 8001 -.ft P -.fi -.UNINDENT -.UNINDENT -.UNINDENT -.sp -This API is not very "RESTful"; please note the following: -.INDENT 0.0 -.IP \(bu 2 -All requests must be sent to the root URL (\fB/\fP). -.IP \(bu 2 -All requests must be sent as a POST request with JSON content in the request -body. -.IP \(bu 2 -All responses are in JSON. -.UNINDENT -.sp -\fBSEE ALSO:\fP -.INDENT 0.0 -.INDENT 3.5 -\fBrest_cherrypy\fP -.sp -The \fBrest_cherrypy\fP module is -more full\-featured, production\-ready, and has builtin security features. -.UNINDENT -.UNINDENT -.SS Deployment -.sp -The \fBrest_wsgi\fP netapi module is a standard Python WSGI app. It can be -deployed one of two ways. -.SS \fBsalt\-api\fP using a development\-only server -.sp -If run directly via salt\-api it uses the \fI\%wsgiref.simple_server()\fP that ships -in the Python standard library. This is a single\-threaded server that is -intended for testing and development. This server does \fBnot\fP use encryption; -please note that raw Salt authentication credentials must be sent with every -HTTP request. -.sp -\fBRunning this module via salt\-api is not recommended for most use!\fP -.SS Using a WSGI\-compliant web server -.sp -This module may be run via any WSGI\-compliant production server such as Apache -with mod_wsgi or Nginx with FastCGI. -.sp -It is highly recommended that this app be used with a server that supports -HTTPS encryption since raw Salt authentication credentials must be sent with -every request. Any apps that access Salt through this interface will need to -manually manage authentication credentials (either username and password or a -Salt token). Tread carefully. -.SS Usage examples -.INDENT 0.0 -.TP -.B POST / -\fBExample request\fP for a basic \fBtest.ping\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-sS \-i \e - \-H \(aqContent\-Type: application/json\(aq \e - \-d \(aq[{"eauth":"pam","username":"saltdev","password":"saltdev","client":"local","tgt":"*","fun":"test.ping"}]\(aq localhost:8001 -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample response\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -HTTP/1.0 200 OK -Content\-Length: 89 -Content\-Type: application/json - -{"return": [{"ms\-\-4": true, "ms\-\-3": true, "ms\-\-2": true, "ms\-\-1": true, "ms\-\-0": true}]} -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample request\fP for an asyncronous \fBtest.ping\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-sS \-i \e - \-H \(aqContent\-Type: application/json\(aq \e - \-d \(aq[{"eauth":"pam","username":"saltdev","password":"saltdev","client":"local_async","tgt":"*","fun":"test.ping"}]\(aq localhost:8001 -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample response\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -HTTP/1.0 200 OK -Content\-Length: 103 -Content\-Type: application/json - -{"return": [{"jid": "20130412192112593739", "minions": ["ms\-\-4", "ms\-\-3", "ms\-\-2", "ms\-\-1", "ms\-\-0"]}]} -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample request\fP for looking up a job ID: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-sS \-i \e - \-H \(aqContent\-Type: application/json\(aq \e - \-d \(aq[{"eauth":"pam","username":"saltdev","password":"saltdev","client":"runner","fun":"jobs.lookup_jid","jid":"20130412192112593739"}]\(aq localhost:8001 -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBExample response\fP: -.INDENT 7.0 -.INDENT 3.5 -.sp -.nf -.ft C -HTTP/1.0 200 OK -Content\-Length: 89 -Content\-Type: application/json - -{"return": [{"ms\-\-4": true, "ms\-\-3": true, "ms\-\-2": true, "ms\-\-1": true, "ms\-\-0": true}]} -.ft P -.fi -.UNINDENT -.UNINDENT -.UNINDENT -.INDENT 0.0 -.TP -.B form lowstate -A list of \fIlowstate\fP data appropriate for the -\fI\%client\fP interface you are calling. -.TP -.B status 200 -success -.TP -.B status 401 -authentication required -.UNINDENT -.SS \fBnetapi\fP developer reference -.SS Introduction to netapi modules -.sp -netapi modules generally bind to a port and start a service. They are -purposefully open\-ended. There could be multiple netapi modules that provide a -REST interface, a module that provides an XMPP interface, or Websockets, or -XMLRPC. -.sp -netapi modules are enabled by adding configuration to your master config file. -Check the docs for each module to see external requirements and configuration -settings. -.sp -Communication with Salt and Salt satellite projects is done by passing a list of -lowstate dictionaries to a client interface. A list of available client -interfaces is below. The lowstate dictionary items map to keyword arguments on -the client interface. -.sp -\fBSEE ALSO:\fP -.INDENT 0.0 -.INDENT 3.5 -\fI\%Python client API\fP -.UNINDENT -.UNINDENT -.SS Client interfaces -.INDENT 0.0 -.TP -.B class saltapi.APIClient(opts) -Provide a uniform method of accessing the various client interfaces in Salt -in the form of low\-data data structures. For example: -.sp -.nf -.ft C ->>> client = APIClient(__opts__) ->>> lowstate = {\(aqclient\(aq: \(aqlocal\(aq, \(aqtgt\(aq: \(aq*\(aq, \(aqfun\(aq: \(aqtest.ping\(aq, \(aqarg\(aq: \(aq\(aq} ->>> client.run(lowstate) -.ft P -.fi -.INDENT 7.0 -.TP -.B local(*args, **kwargs) -Run \fI\%execution modules\fP syncronously -.sp -Wraps \fI\%salt.client.LocalClient.cmd()\fP\&. -.INDENT 7.0 -.TP -.B Returns -Returns the result from the execution module -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B local_async(*args, **kwargs) -Run \fI\%execution modules\fP asyncronously -.sp -Wraps \fI\%salt.client.LocalClient.run_job()\fP\&. -.INDENT 7.0 -.TP -.B Returns -job ID -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B local_batch(*args, **kwargs) -Run \fI\%execution modules\fP against batches of minions -.sp -New in version 0.8.4. - -.sp -Wraps \fI\%salt.client.LocalClient.cmd_batch()\fP -.INDENT 7.0 -.TP -.B Returns -Returns the result from the exeuction module for each batch of -returns -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B runner(fun, **kwargs) -Run \fIrunner modules \fP -.sp -Wraps \fI\%salt.runner.RunnerClient.low()\fP\&. -.INDENT 7.0 -.TP -.B Returns -Returns the result from the runner module -.UNINDENT -.UNINDENT -.INDENT 7.0 -.TP -.B wheel(fun, **kwargs) -Run \fI\%wheel modules\fP -.sp -Wraps \fI\%salt.wheel.WheelClient.master_call()\fP\&. -.INDENT 7.0 -.TP -.B Returns -Returns the result from the wheel module -.UNINDENT -.UNINDENT -.UNINDENT -.SS Writing netapi modules -.sp -\fBnetapi\fP modules, put simply, bind a port and start a service. -They are purposefully open\-ended and can be used to present a variety of -external interfaces to Salt, and even present multiple interfaces at once. -.sp -\fBSEE ALSO:\fP -.INDENT 0.0 -.INDENT 3.5 -\fIThe full list of netapi modules\fP -.UNINDENT -.UNINDENT -.SS Configuration -.sp -All \fBnetapi\fP configuration is done in the \fI\%Salt master -config\fP and takes a form similar to the following: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -rest_cherrypy: - port: 8000 - debug: True - ssl_crt: /etc/pki/tls/certs/localhost.crt - ssl_key: /etc/pki/tls/certs/localhost.key -.ft P -.fi -.UNINDENT -.UNINDENT -.SS The \fB__virtual__\fP function -.sp -Like all module types in Salt, \fBnetapi\fP modules go through -Salt\(aqs loader interface to determine if they should be loaded into memory and -then executed. -.sp -The \fB__virtual__\fP function in the module makes this determination and should -return \fBFalse\fP or a string that will serve as the name of the module. If the -module raises an \fBImportError\fP or any other errors, it will not be loaded. -.SS The \fBstart\fP function -.sp -The \fBstart()\fP function will be called for each \fBnetapi\fP -module that is loaded. This function should contain the server loop that -actually starts the service. This is started in a multiprocess. -.SS Inline documentation -.sp -As with the rest of Salt, it is a best\-practice to include liberal inline -documentation in the form of a module docstring and docstrings on any classes, -methods, and functions in your \fBnetapi\fP module. -.SS Loader “magic” methods -.sp -The loader makes the \fB__opts__\fP data structure available to any function in -a \fBnetapi\fP module. -.SH RELEASES -.SS Release notes -.SS salt\-api 0.5.0 -.sp -\fBsalt\-api\fP is gearing up for the initial public release with 0.5.0. -Although this release ships with working basic functionality it is awaiting the -authentication backend that will be introduced in Salt 0.10.4 before it can be -considered ready for testing at large. -.SS REST API -.sp -This release presents the flagship netapi module which provides a RESTful -interface to a running Salt system. It allows for viewing minions, runners, and -jobs as well as running execution modules and runners of a running Salt system -through a REST API that returns JSON. -.SS Participation -.sp -\fBsalt\-api\fP is just getting off the ground so feedback, questions, and -ideas are critical as we solidify how this project fits into the overall Salt -infrastructure management stack. Please get involved by \fI\%filing issues\fP on -GitHub, \fI\%discussing on the mailing list\fP, and chatting in \fB#salt\fP on -Freenode. -.SS salt\-api 0.6.0 -.sp -\fBsalt\-api\fP inches closer to prime\-time with 0.6.0. This release adds -the beginnings of a universal interface for accessing Salt components via the -tried and true method of passing low\-data to functions (a core component of -Salt\(aqs remote execution and state management). -.SS Low\-data interface -.sp -A new view accepts :\fI\%http:post\fP: requests at the root URL that accepts raw -low\-data as :\fI\%http:post\fP: data and passes that low\-data along to a client -interface in Salt. Currently only LocalClient and RunnerClient interfaces have -been implemented in Salt with more coming in the next Salt release. -.SS External authentication -.sp -Raw low\-data can contain authentication credentials that make use of Salt\(aqs new -\fI\%external_auth\fP system. -.sp -The following is a proof\-of\-concept of a working eauth call. (It bears -repeating this is a pre\-alpha release and this should not be used by anyone for -anything real.) -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-si localhost:8000 \e - \-d client=local \e - \-d tgt=\(aq*\(aq \e - \-d fun=\(aqtest.ping\(aq \e - \-d arg \e - \-d eauth=pam \e - \-d username=saltdev \e - \-d password=saltdev -.ft P -.fi -.UNINDENT -.UNINDENT -.SS Participation -.sp -\fBsalt\-api\fP is just getting off the ground so feedback, questions, and -ideas are critical as we solidify how this project fits into the overall Salt -infrastructure management stack. Please get involved by \fI\%filing issues\fP on -GitHub, \fI\%discussing on the mailing list\fP, and chatting in \fB#salt\-devel\fP on -Freenode. -.SS salt\-api 0.7.0 -.sp -\fBsalt\-api\fP is ready for alpha\-testing in the real world. This release -solidifies how \fBsalt\-api\fP will communicate with the larger Salt -ecosystem. In addition authentication and encryption (via SSL) have been added. -.sp -The first netapi module was a proof of concept written in Flask. It was quite -useful to be able to quickly hammer out a URL structure and solidify on an -interface for programmatically calling out to Salt components. As of this -release that module has been deprecated and removed in favor of a netapi module -written in CherryPy. CherryPy affords tremendous flexibility when composing a -REST interface and will present a stable platform for building out a very -adaptable and featureful REST API—also we\(aqre using the excellent and fast -CherryPy webserver for securely serving the API. -.SS Low\-data interface -.sp -The last release introduced a proof\-of\-concept for how the various Salt -components will communicate with each other. This is done by passing a data -structure to a client interface. This release expands on that. There are -currently three client interfaces in Salt. -.sp -\fBSEE ALSO:\fP -.INDENT 0.0 -.INDENT 3.5 -\fInetapi\-introduction\fP -.UNINDENT -.UNINDENT -.SS Encryption and authentication -.sp -Encryption has been added via SSL. You can supply an existing certificate or -generate a self\-signed certificate through Salt\(aqs \fI\%tls\fP -module. -.sp -Authentication is performed through Salt\(aqs incredibly flexible \fI\%external -auth\fP system and is maintained when accessing the API via session -tokens. -.SS Participation -.sp -\fBsalt\-api\fP is just getting off the ground so feedback, questions, and -ideas are critical as we solidify how this project fits into the overall Salt -infrastructure management stack. Please get involved by \fI\%filing issues\fP on -GitHub, \fI\%discussing on the mailing list\fP, and chatting in \fB#salt\-devel\fP on -Freenode. -.SS salt\-api 0.7.5 -.sp -This release is a mostly a minor release to pave a better path for -\fBsalt\-ui\fP though there are some small feature additions and bugfixes. -.SS Changes -.INDENT 0.0 -.IP \(bu 2 -Convenience URLs \fB/minions\fP and \fB/jobs\fP have been added as well as a -async client wrapper. This starts a job and immediately returns the job ID, -allowing you to fetch the result of that job at a later time. -.IP \(bu 2 -The return format will now default to JSON if no specific format is -requested. -.IP \(bu 2 -A new setting \fBstatic\fP has been added that will serve any static media from -the directory specified. In addition if an \fBindex.html\fP file is found -in that directory and the \fBAccept\fP header in the request prefer HTML that -file will be served. -.IP \(bu 2 -All HTML, including the login form, has been removed from \fBsalt\-api\fP -and moved into the \fBsalt\-ui\fP project. -.IP \(bu 2 -Sessions now live as long as the Salt token. -.UNINDENT -.SS Participation -.sp -\fBsalt\-api\fP is just getting off the ground so feedback, questions, and -ideas are critical as we solidify how this project fits into the overall Salt -infrastructure management stack. Please get involved by \fI\%filing issues\fP on -GitHub, \fI\%discussing on the mailing list\fP, and chatting in \fB#salt\-devel\fP on -Freenode. -.SS salt\-api 0.8.0 -.sp -We are happy to announce the release of \fBsalt\-api\fP 0.8.0. -.sp -This release encompasses bugfixes and new features for the -\fBrest_cherrypy\fP netapi module that -provides a RESTful interface for a running Salt system. -.sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -Requires Salt 0.13 -.UNINDENT -.UNINDENT -.SS Changes -.sp -In addition to the usual documentation improvements and bug fixes this release -introduces the following changes and additions. -.sp -Please note the backward incompatible change detailed below. -.SS RPM packaging -.sp -Thanks to Andrew Niemantsvedriet (\fI\%@kaptk2\fP) \fBsalt\-api\fP is now -available in Fedora package repositories as well as RHEL compatible systems via -EPEL. -.INDENT 0.0 -.IP \(bu 2 -\fI\%http://dl.fedoraproject.org/pub/epel/5/i386/repoview/salt\-api.html\fP -.IP \(bu 2 -\fI\%http://dl.fedoraproject.org/pub/epel/5/x86_64/repoview/salt\-api.html\fP -.IP \(bu 2 -\fI\%http://dl.fedoraproject.org/pub/epel/6/i386/repoview/salt\-api.html\fP -.IP \(bu 2 -\fI\%http://dl.fedoraproject.org/pub/epel/6/x86_64/repoview/salt\-api.html\fP -.UNINDENT -.sp -Thanks also to Clint Savage (\fI\%@herlo\fP) and Thomas Spura (\fI\%@tomspur\fP) for -helping with that process. -.SS Ubuntu PPA packaging -.sp -Thanks to Sean Channel (\fI\%@seanchannel\fP, pentabular) \fBsalt\-api\fP is -available as a PPA on the SaltStack LaunchPad team. -.sp -\fI\%https://launchpad.net/~saltstack/+archive/salt\fP -.SS Authentication information on login -.sp -\fBWARNING:\fP -.INDENT 0.0 -.INDENT 3.5 -Backward incompatible change -.sp -The \fB/login\fP URL no -longer responds with a 302 redirect for success. -.sp -Although this is behavior is common in the browser world it is not useful -from an API so we have changed it to return a 200 response in this release. -.sp -We take backward compatibility very seriously and we apologize for the -inconvenience. In this case we felt the previous behavior was limiting. -Changes such as this will be rare. -.UNINDENT -.UNINDENT -.sp -New in this release is displaying information about the current session and the -current user. For example: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-sS localhost:8000/login \e - \-H \(aqAccept: application/x\-yaml\(aq - \-d username=\(aqsaltdev\(aq - \-d password=\(aqsaltdev\(aq - \-d eauth=\(aqpam\(aq - -return: -\- eauth: pam - expire: 1365508324.359403 - perms: - \- \(aq@wheel\(aq - \- grains.* - \- state.* - \- status.* - \- sys.* - \- test.* - start: 1365465124.359402 - token: caa7aa2b9dbc4a8adb6d2e19c3e52be68995ef4b - user: saltdev -.ft P -.fi -.UNINDENT -.UNINDENT -.SS Bypass session handling -.sp -A convenience URL has been added -(\fB/run\fP) to bypass the normal -session\-handling process. -.sp -The REST interface uses the concept of "lowstate" data to specify what function -should be executed in Salt (plus where that function is and any arguments to -the function). This is a thin wrapper around Salt\(aqs various "client" -interfaces, for example Salt\(aqs \fI\%LocalClient()\fP which can -accept authentication credentials directly. -.sp -Authentication with the REST API typically goes through the login URL and a -session is generated that is tied to a Salt external_auth token. That token is -then automatically added to the lowstate for subsequent requests that match the -current session. -.sp -It is sometimes useful to handle authentication or token management manually -from another program or script. For example: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -curl \-sS localhost:8000/run \e - \-d client=\(aqlocal\(aq \e - \-d tgt=\(aq*\(aq \e - \-d fun=\(aqtest.ping\(aq \e - \-d eauth=\(aqpam\(aq \e - \-d username=\(aqsaltdev\(aq \e - \-d password=\(aqsaltdev\(aq -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -It is a Bad Idea (TM) to do this unless you have a very good reason and a well -thought out security model. -.SS Logout -.sp -An URL has been added -(\fB/logout\fP) that will cause -the client\-side to expire the session cookie and the server\-side session to be -invalidated. -.SS Running the REST interface via any WSGI\-compliant server -.sp -The \fBrest_cherrypy\fP netapi module is -a regular WSGI application written using the CherryPy framework. It was written -with the intent of also running from any WSGI\-compliant server such as Apache -and mod_wsgi, Gunicorn, uWSGI, Nginx and FastCGI, etc. -.sp -The WSGI application entry point has been factored out into a stand\-alone file -in this release suitable for calling from an external server. -\fBsalt\-api\fP does not need to be running in this scenario. -.sp -For example, an Apache virtual host configuration: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C - - ServerName example.com - ServerAlias *.example.com - - ServerAdmin webmaster@example.com - - LogLevel warn - ErrorLog /var/www/example.com/logs/error.log - CustomLog /var/www/example.com/logs/access.log combined - - DocumentRoot /var/www/example.com/htdocs - - WSGIScriptAlias / /path/to/saltapi/netapi/rest_cherrypy/wsgi.py - -.ft P -.fi -.UNINDENT -.UNINDENT -.SS Participation -.sp -Please get involved by \fI\%filing issues\fP on GitHub, \fI\%discussing on the mailing -list\fP, and chatting in \fB#salt\-devel\fP on Freenode. -.SS salt\-api 0.8.2 -.sp -\fBsalt\-api\fP 0.8.2 is largely a bugfix release that fixes a -compatibility issue with changes in Salt 0.15.9. -.sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -Requires Salt 0.15.9 or greater -.UNINDENT -.UNINDENT -.sp -The following changes have been made to the \fBrest_cherrypy\fP netapi module that provides a RESTful -interface for a running Salt system: -.INDENT 0.0 -.IP \(bu 2 -Fixed issue #87 which caused the Salt master\(aqs PID file to be overwritten. -.IP \(bu 2 -Fixed an inconsistency with the return format for the \fB/minions\fP -convenience URL. -.sp -\fBWARNING:\fP -.INDENT 2.0 -.INDENT 3.5 -This is a backward incompatible change. -.UNINDENT -.UNINDENT -.IP \(bu 2 -Added a dedicated URL for serving an HTML app -.IP \(bu 2 -Added dedicated URL for serving static media -.UNINDENT -.SS salt\-api 0.8.3 -.sp -\fBsalt\-api\fP 0.8.3 is a small release largely concerning changes and -fixes to the \fBrest_cherrypy\fP netapi -module. -.sp -This release will likely be the final salt\-api release as a separate project. -The Salt team has begun the process of merging this project directly in to the -main Salt project. What this means for end users is only that there will be one -fewer package to install. Salt itself will ship with the current \fBnetapi\fP -modules and the API and configuration will remain otherwise unchanged. -.sp -The reasoning behind merging the two projects is simply to lower the barrier to -entry. Having a separate project was useful for experimentation and exploration -but there was no technical reason for the separation \-\- salt\-api uses the same -flexible module system that Salt uses and those modules will simply be moved -into Salt. -.sp -Going forward, Salt will ship with the same REST interface that salt\-api -currently provides. This will have the side benefit of not having to coordinate -incompatible Salt and salt\-api releases. -.SS \fBrest_cherrypy\fP changes -.sp -An HTTP stream of Salt\(aqs event bus has been added. This stream conforms to the -SSE (Server Sent Events) spec and is easily consumed via JavaScript clients. -This HTTP stream allows a real\-time window into a running Salt system. A client -watching the stream can see as soon as individual minions return data for a -job, authentication events, and any other events that go through the Salt -master. -.sp -A new configuration option to only allow access to whitelisted IP addresses. Of -course, IP addresses can be easily spoofed so this feature should be thought of -as a usability addition and not used for security purposes. -.sp -An option to disable SSL has been added. Previously SSL could only be disabled -while running the HTTP server with debugging options enabled. Now each item can -be enabled or disabled independently of the other. -.sp -In addition, there has been several bug fixes, packaging fixes, and minor code -simplification. -.SS salt\-api 0.8.4 -.sp -\fBsalt\-api\fP 0.8.4 sees a number of new features and feature -enhancements in the \fBrest_cherrypy\fP -netapi module. -.sp -Work to merge \fBsalt\-api\fP into the main Salt distribution continues and -it is likely to be included in Salt\(aqs Helium release. -.SS \fBrest_cherrypy\fP changes -.SS Web hooks -.sp -This release adds a \fBnew URL /hook\fP that allows salt\-api to serve as a -generic web hook interface for Salt. POST requests to the URL trigger events on -Salt\(aqs event bus. -.sp -External services like Amazon SNS, Travis CI, GitHub, etc can easily send -signals through Salt\(aqs Reactor. -.sp -The following HTTP call will trigger the following Salt event. -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -% curl \-sS http://localhost:8000/hook/some/tag \e - \-d some=\(aqData!\(aq -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -Event tag: \fBsalt/netapi/hook/some/tag\fP\&. Event data: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -{ - "_stamp": "2014\-04\-04T12:14:54.389614", - "post": { - "some": "Data!" - }, - "headers": { - "Content\-Type": "application/x\-www\-form\-urlencoded", - "Host": "localhost:8000", - "User\-Agent": "curl/7.32.0", - "Accept": "*/*", - "Content\-Length": "10", - "Remote\-Addr": "127.0.0.1" - } -} -.ft P -.fi -.UNINDENT -.UNINDENT -.SS Batch mode -.sp -The \fBlocal_batch()\fP client exposes Salt\(aqs batch mode -for executing commands on incremental subsets of minions. -.SS Tests! -.sp -We have added the necessary framework for testing the rest_cherrypy module and -this release includes a number of both unit and integration tests. The suite -can be run with the following command: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -python \-m unittest discover \-v -.ft P -.fi -.UNINDENT -.UNINDENT -.SS CherryPy server stats and configuration -.sp -A number of settings have been added to better configure the performance of the -CherryPy web server. In addition, a \fBnew URL /stats\fP has been added to expose metrics on -the health of the CherryPy web server. -.SS Improvements for running with external WSGI servers -.sp -Running the \fBrest_cherrypy\fP module via a WSGI\-capable server such as Apache -or Nginx can be tricky since the user the server is running as must have -permission to access the running Salt system. This release eases some of those -restrictions by accessing Salt\(aqs key interface through the external auth -system. Read access to the Salt configuration is required for the user the -server is running as and everything else should go through external auth. -.SS More information in the jobs URLs -.sp -The output for the \fB/jobs/\fP has been augmented with more -information about the job such as which minions are expected to return for that -job. This same output will be added to the other salt\-api URLs in the next -release. -.SS Improvements to the Server Sent Events stream -.sp -Event tags have been added to \fBthe HTTP event stream\fP as SSE tags which allows JavaScript -or other consumers to more easily match on certain tags without having to -inspect the whole event. -.SH REFERENCE -.INDENT 0.0 -.IP \(bu 2 -\fIgenindex\fP -.IP \(bu 2 -\fImodindex\fP -.IP \(bu 2 -\fIsearch\fP -.IP \(bu 2 -\fIglossary\fP -.UNINDENT -.SH AUTHOR -Thomas S. Hatch and many others, please see the Authors file -.SH COPYRIGHT -2012, Thomas S. Hatch -.\" Generated by docutils manpage writer. -. From f9084658e4a87fcdc88b706eb3f39ea3a2860c7a Mon Sep 17 00:00:00 2001 From: Andrew Burdo Date: Sat, 21 Jun 2014 22:03:11 +0300 Subject: [PATCH 28/55] Allow to append if the grain doesn't exist. --- salt/modules/grains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/grains.py b/salt/modules/grains.py index 086f9a4be8..abf7529f74 100644 --- a/salt/modules/grains.py +++ b/salt/modules/grains.py @@ -280,7 +280,7 @@ def append(key, val, convert=False): salt '*' grains.append key val ''' - grains = get(key) + grains = get(key, []) if not isinstance(grains, list) and convert is True: grains = [grains] if not isinstance(grains, list): From b67ac400703720b128e7157f6de32f9a924d1f02 Mon Sep 17 00:00:00 2001 From: Victor Lin Date: Sat, 21 Jun 2014 15:18:50 -0700 Subject: [PATCH 29/55] Try to add docker network_mode argument --- salt/modules/dockerio.py | 6 ++++-- salt/states/dockerio.py | 24 +++++++++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/salt/modules/dockerio.py b/salt/modules/dockerio.py index 5a9a039b63..466d5f0f7f 100644 --- a/salt/modules/dockerio.py +++ b/salt/modules/dockerio.py @@ -876,7 +876,8 @@ def start(container, links=None, privileged=False, dns=None, - volumes_from=None): + volumes_from=None, + network_mode=None): ''' Restart the specified container @@ -923,7 +924,8 @@ def start(container, links=links, privileged=privileged, dns=dns, - volumes_from=volumes_from) + volumes_from=volumes_from, + network_mode=network_mode) except TypeError: # maybe older version of docker-py <= 0.3.1 dns and # volumes_from are not accepted diff --git a/salt/states/dockerio.py b/salt/states/dockerio.py index 9833fd1e86..6ff261fd4d 100644 --- a/salt/states/dockerio.py +++ b/salt/states/dockerio.py @@ -556,7 +556,7 @@ def script(*args, **kw): def running(name, container=None, port_bindings=None, binds=None, publish_all_ports=False, links=None, lxc_conf=None, privileged=False, dns=None, volumes_from=None, - check_is_running=True): + network_mode=None, check_is_running=True): ''' Ensure that a container is running. (`docker inspect`) @@ -623,6 +623,16 @@ def running(name, container=None, port_bindings=None, binds=None, - dns: - name_other_container + network_mode + - 'bridge': creates a new network stack for the container on the docker bridge + - 'none': no networking for this container + - 'container:[name|id]': reuses another container network stack) + - 'host': use the host network stack inside the container + + .. code-block:: yaml + + - network_mode: host + check_is_running Enable checking if a container should run or not. Useful for data-only containers that must be linked to another one. @@ -639,7 +649,7 @@ def running(name, container=None, port_bindings=None, binds=None, container, binds=binds, port_bindings=port_bindings, lxc_conf=lxc_conf, publish_all_ports=publish_all_ports, links=links, privileged=privileged, - dns=dns, volumes_from=volumes_from, + dns=dns, volumes_from=volumes_from, network_mode=network_mode, ) if check_is_running: is_running = __salt__['docker.is_running'](container) @@ -651,9 +661,13 @@ def running(name, container=None, port_bindings=None, binds=None, changes={name: True}) else: return _invalid( - comment=('Container {0!r}' - ' cannot be started\n{0!s}').format(container, - started['out'])) + comment=( + 'Container {0!r} cannot be started\n{0!s}' + .format( + container, + started['out'] + ) + ) else: return _valid( comment='Container {0!r} started.\n'.format(container), From 962fb0c2118e1aa738d8f7f3e2802dffe25e0b49 Mon Sep 17 00:00:00 2001 From: Victor Lin Date: Sat, 21 Jun 2014 15:30:24 -0700 Subject: [PATCH 30/55] Fix syntax error --- salt/states/dockerio.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/states/dockerio.py b/salt/states/dockerio.py index 6ff261fd4d..51637dac13 100644 --- a/salt/states/dockerio.py +++ b/salt/states/dockerio.py @@ -665,9 +665,10 @@ def running(name, container=None, port_bindings=None, binds=None, 'Container {0!r} cannot be started\n{0!s}' .format( container, - started['out'] + started['out'], ) ) + ) else: return _valid( comment='Container {0!r} started.\n'.format(container), From 823076a82403e94ff649e9341eff6725c78149e9 Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Sun, 22 Jun 2014 11:35:04 +0200 Subject: [PATCH 31/55] cmd: Forward the working dir to VT --- salt/modules/cmdmod.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index e15b292764..f0515206c8 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -474,6 +474,7 @@ def _run(cmd, shell=True, log_stdout=True, log_stderr=True, + cwd=cwd, env=env, log_stdin_level=output_loglevel, log_stdout_level=output_loglevel, From 2e153844e8940fce149939218c812ef05cf81f03 Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Sat, 21 Jun 2014 17:08:17 +0200 Subject: [PATCH 32/55] buildout: fix bootstrap file perms --- salt/modules/zcbuildout.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/modules/zcbuildout.py b/salt/modules/zcbuildout.py index 3bae2c8daf..28e2e80c2a 100644 --- a/salt/modules/zcbuildout.py +++ b/salt/modules/zcbuildout.py @@ -721,6 +721,15 @@ def bootstrap(directory='.', bootstrap_args += ' --accept-buildout-test-releases' if config and '"-c"' in content: bootstrap_args += ' -c {0}'.format(config) + # be sure that the bootstrap belongs to the running user + try: + if runas: + uid = __salt__['user.info'](runas)['uid'] + gid = __salt__['user.info'](runas)['gid'] + os.chown('bootstrap.py', uid, gid) + except (IOError, OSError): + # dont block here, try to execute it if can pass + pass cmd = '{0} bootstrap.py {1}'.format(python, bootstrap_args) ret = _Popen(cmd, directory=directory, runas=runas, env=env) output = ret['output'] From ad692588f3bcc10b1011e294b2abccf744c120af Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Sat, 21 Jun 2014 17:01:21 +0200 Subject: [PATCH 33/55] VT support for buildout mods --- salt/modules/zcbuildout.py | 52 ++++++++++++++++++++++++++++++-------- salt/states/zcbuildout.py | 11 +++++++- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/salt/modules/zcbuildout.py b/salt/modules/zcbuildout.py index 3bae2c8daf..3152a1e7e2 100644 --- a/salt/modules/zcbuildout.py +++ b/salt/modules/zcbuildout.py @@ -252,7 +252,9 @@ def _Popen(command, directory='.', runas=None, env=(), - exitcode=0): + exitcode=0, + use_vt=False, + loglevel=None): ''' Run a command. @@ -272,13 +274,20 @@ def _Popen(command, fails if cmd does not return this exit code (set to None to disable check) + use_vt + Use the new salt VT to stream output [experimental] + ''' ret = None directory = os.path.abspath(directory) if isinstance(command, list): command = ' '.join(command) LOG.debug(u'Running {0}'.format(command)) - ret = __salt__['cmd.run_all'](command, cwd=directory, runas=runas, env=env) + if not loglevel: + loglevel = 'debug' + ret = __salt__['cmd.run_all']( + command, cwd=directory, output_loglevel=loglevel, + runas=runas, env=env, use_vt=use_vt) out = ret['stdout'] + '\n\n' + ret['stderr'] if (exitcode is not None) and (ret['retcode'] != exitcode): raise _BuildoutError(out) @@ -543,7 +552,9 @@ def bootstrap(directory='.', buildout_ver=None, test_release=False, offline=False, - new_st=None): + new_st=None, + use_vt=False, + loglevel=None): ''' Run the buildout bootstrap dance (python bootstrap.py). @@ -583,6 +594,9 @@ def bootstrap(directory='.', unless Do not execute cmd if statement on the host return 0 + use_vt + Use the new salt VT to stream output [experimental] + CLI Example: .. code-block:: bash @@ -722,7 +736,8 @@ def bootstrap(directory='.', if config and '"-c"' in content: bootstrap_args += ' -c {0}'.format(config) cmd = '{0} bootstrap.py {1}'.format(python, bootstrap_args) - ret = _Popen(cmd, directory=directory, runas=runas, env=env) + ret = _Popen(cmd, directory=directory, runas=runas, loglevel=loglevel, + env=env, use_vt=use_vt) output = ret['output'] return {'comment': cmd, 'out': output} @@ -738,7 +753,9 @@ def run_buildout(directory='.', runas=None, env=(), verbose=False, - debug=False): + debug=False, + use_vt=False, + loglevel=None): ''' Run a buildout in a directory. @@ -772,6 +789,9 @@ def run_buildout(directory='.', verbose run buildout in verbose mode (-vvvvv) + use_vt + Use the new salt VT to stream output [experimental] + CLI Example: .. code-block:: bash @@ -809,7 +829,9 @@ def run_buildout(directory='.', cmd, directory=directory, runas=runas, env=env, - output=True) + output=True, + loglevel=loglevel, + use_vt=use_vt) ) else: LOG.info(u'Installing all buildout parts') @@ -817,7 +839,9 @@ def run_buildout(directory='.', bcmd, config, ' '.join(argv)) cmds.append(cmd) outputs.append( - _Popen(cmd, directory=directory, runas=runas, env=env, output=True) + _Popen( + cmd, directory=directory, runas=runas, loglevel=loglevel, + env=env, output=True, use_vt=use_vt) ) return {'comment': '\n'.join(cmds), @@ -888,7 +912,9 @@ def buildout(directory='.', debug=False, verbose=False, onlyif=None, - unless=None): + unless=None, + use_vt=False, + loglevel=None): ''' Run buildout in a directory. @@ -939,6 +965,8 @@ def buildout(directory='.', verbose run buildout in verbose mode (-vvvvv) + use_vt + Use the new salt VT to stream output [experimental] CLI Example: @@ -956,7 +984,9 @@ def buildout(directory='.', env=env, runas=runas, distribute=distribute, - python=python) + python=python, + use_vt=use_vt, + loglevel=loglevel) buildout_ret = run_buildout(directory=directory, config=config, parts=parts, @@ -965,7 +995,9 @@ def buildout(directory='.', runas=runas, env=env, verbose=verbose, - debug=debug) + debug=debug, + use_vt=use_vt, + loglevel=loglevel) # signal the decorator or our return return _merge_statuses([boot_ret, buildout_ret]) diff --git a/salt/states/zcbuildout.py b/salt/states/zcbuildout.py index 3befb11506..f080785f67 100644 --- a/salt/states/zcbuildout.py +++ b/salt/states/zcbuildout.py @@ -133,7 +133,9 @@ def installed(name, debug=False, verbose=False, unless=None, - onlyif=None): + onlyif=None, + use_vt=False, + loglevel='debug'): ''' Install buildout in a specific directory @@ -198,6 +200,12 @@ def installed(name, verbose run buildout in verbose mode (-vvvvv) + use_vt + Use the new salt VT to stream output [experimental] + + loglevel + loglevel for buildout commands + ''' ret = {} @@ -248,6 +256,7 @@ def installed(name, verbose=verbose, onlyif=onlyif, unless=unless, + use_vt=use_vt ) ret.update(_ret_status(func(**kwargs), name, quiet=quiet)) return ret From 49419f94c380ff2aa69552adfbe26f5695cbd126 Mon Sep 17 00:00:00 2001 From: Matthieu Guegan Date: Sun, 22 Jun 2014 11:38:35 +0200 Subject: [PATCH 34/55] Add a method context environment PATH Some modules are referring to PATH env variable which do not contains /opt/local/.. paths by default. This fix provides correct PATH handling --- pkg/smartos/salt-minion.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/smartos/salt-minion.xml b/pkg/smartos/salt-minion.xml index 1373ffcff0..619276dc1e 100644 --- a/pkg/smartos/salt-minion.xml +++ b/pkg/smartos/salt-minion.xml @@ -31,7 +31,12 @@ - + + + + + Date: Sun, 22 Jun 2014 13:43:11 +0200 Subject: [PATCH 35/55] review for cmdmod fix --- salt/modules/zcbuildout.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/salt/modules/zcbuildout.py b/salt/modules/zcbuildout.py index 28e2e80c2a..ecd4b35978 100644 --- a/salt/modules/zcbuildout.py +++ b/salt/modules/zcbuildout.py @@ -39,8 +39,10 @@ def __virtual__(): # Import python libs import os import re +import logging import sys import traceback +import copy import urllib2 # Import salt libs @@ -67,12 +69,18 @@ _URL_VERSIONS = { 2: u'http://downloads.buildout.org/2/bootstrap.py', } DEFAULT_VER = 2 +_logger = logging.getLogger(__name__) -def _salt_callback(func): +def _salt_callback(func, **kwargs): LOG.clear() def _call_callback(*a, **kw): + # cleanup the module kwargs before calling it from the + # decorator + kw = copy.deepcopy(kw) + for k in [ar for ar in kw if '__pub' in ar]: + kw.pop(k, None) st = BASE_STATUS.copy() directory = kw.get('directory', '.') onlyif = kw.get('onlyif', None) @@ -727,9 +735,11 @@ def bootstrap(directory='.', uid = __salt__['user.info'](runas)['uid'] gid = __salt__['user.info'](runas)['gid'] os.chown('bootstrap.py', uid, gid) - except (IOError, OSError): + except (IOError, OSError) as exc: # dont block here, try to execute it if can pass - pass + _logger.error('BUILDOUT bootstrap permissions error:' + ' {0}'.format(exc), + exc_info=_logger.isEnabledFor(logging.DEBUG)) cmd = '{0} bootstrap.py {1}'.format(python, bootstrap_args) ret = _Popen(cmd, directory=directory, runas=runas, env=env) output = ret['output'] From 6e4ac89be01887e2a9491246e7c8b817797eb453 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 22 Jun 2014 16:16:49 -0500 Subject: [PATCH 36/55] Clarify that templates are used on managed files --- salt/states/file.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index d36eaa363f..8d58644118 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -6,11 +6,12 @@ Operations on regular files, special files, directories, and symlinks Salt States can aggressively manipulate files on a system. There are a number of ways in which files can be managed. -Regular files can be enforced with the ``managed`` function. This function -downloads files from the salt master and places them on the target system. -The downloaded files can be rendered as a jinja, mako, or wempy template, -adding a dynamic component to file management. An example of ``file.managed`` -which makes use of the jinja templating system would look like this: +Regular files can be enforced with the :mod:`file.managed +` state. This state downloads files from the salt +master and places them on the target system. Managed files can be rendered as a +jinja, mako, or wempy template, adding a dynamic component to file management. +An example of :mod:`file.managed ` which makes use of +the jinja templating system would look like this: .. code-block:: yaml From 910ea9ca6dd887627cca038793b9e8382f50d56e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 23 Jun 2014 00:06:28 +0100 Subject: [PATCH 37/55] Convert cloud related relative paths to absolute paths(if they exist) The cloud configuration file can point to a relative master file: ``` master_config: another/dir ``` We just convert those into `/another/dir` if the path exists. This also works for `providers_config` and `profiles_config` --- salt/config.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/salt/config.py b/salt/config.py index a51247bfee..19dd0f63d9 100644 --- a/salt/config.py +++ b/salt/config.py @@ -34,7 +34,7 @@ import salt.utils.xdg from salt._compat import string_types import sys -#can't use salt.utils.is_windows, because config.py is included from salt.utils +# can't use salt.utils.is_windows, because config.py is included from salt.utils if not sys.platform.lower().startswith('win'): import salt.cloud.exceptions @@ -651,6 +651,27 @@ def _read_conf_file(path): return conf_opts +def _absolute_path(path, relative_to=None): + ''' + Return an absolute path. In case ``relative_to`` is passed and ``path`` is + not an absolute path, we try to prepend ``relative_to`` to ``path``and if + that path exists, return that one + ''' + + if path and os.path.isabs(path): + return path + if path and relative_to is not None: + _abspath = os.path.join(relative_to, path) + if os.path.isfile(_abspath): + log.debug( + 'Relative path {0!r} converted to existing absolute path {1!r}'.format( + path, _abspath + ) + ) + return _abspath + return path + + def load_config(path, env_var, default_path=None): ''' Returns configuration dict from parsing either the file described by @@ -1017,6 +1038,9 @@ def cloud_config(path, env_var='SALT_CLOUD_CONFIG', defaults=None, # configuration file, and master_config_path = os.path.join(config_dir, 'master') + # Convert relative to absolute paths if necessary + master_config_path = _absolute_path(master_config_path, config_dir) + if 'providers_config' in overrides and providers_config_path is None: # The configuration setting is being specified in the main cloud # configuration file @@ -1025,6 +1049,9 @@ def cloud_config(path, env_var='SALT_CLOUD_CONFIG', defaults=None, and not providers_config_path: providers_config_path = os.path.join(config_dir, 'cloud.providers') + # Convert relative to absolute paths if necessary + providers_config_path = _absolute_path(providers_config_path, config_dir) + if 'profiles_config' in overrides and profiles_config_path is None: # The configuration setting is being specified in the main cloud # configuration file @@ -1033,6 +1060,9 @@ def cloud_config(path, env_var='SALT_CLOUD_CONFIG', defaults=None, and not profiles_config_path: profiles_config_path = os.path.join(config_dir, 'cloud.profiles') + # Convert relative to absolute paths if necessary + profiles_config_path = _absolute_path(profiles_config_path, config_dir) + # Prepare the deploy scripts search path deploy_scripts_search_path = overrides.get( 'deploy_scripts_search_path', From 992cdc07b149e10ce94e0e2a494c3b26c5ca6aec Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 23 Jun 2014 00:58:39 +0100 Subject: [PATCH 38/55] Reorder imports and don't even run setUp is expensive tests are not enabled --- tests/integration/cloud/providers/linode.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/cloud/providers/linode.py b/tests/integration/cloud/providers/linode.py index 43f51da372..5e71044d8d 100644 --- a/tests/integration/cloud/providers/linode.py +++ b/tests/integration/cloud/providers/linode.py @@ -6,16 +6,16 @@ # Import Python Libs import os -# Import Salt Libs -import integration -from salt.config import cloud_providers_config - # Import Salt Testing Libs from salttesting import skipIf from salttesting.helpers import ensure_in_syspath, expensiveTest ensure_in_syspath('../../../') +# Import Salt Libs +import integration +from salt.config import cloud_providers_config + # Import Third-Party Libs try: import libcloud # pylint: disable=W0611 @@ -30,6 +30,7 @@ class LinodeTest(integration.ShellCase): Integration tests for the Linode cloud provider in Salt-Cloud ''' + @expensiveTest def setUp(self): ''' Sets up the test requirements @@ -63,7 +64,6 @@ class LinodeTest(integration.ShellCase): ) ) - @expensiveTest def test_instance(self): ''' Test creating an instance on Linode From 99a8e05aa982ecefb04f14f1568df80fe62ddeba Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 23 Jun 2014 00:59:27 +0100 Subject: [PATCH 39/55] Reorder imports and don't even run setUp is expensive tests are not enabled --- tests/integration/cloud/providers/digital_ocean.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/integration/cloud/providers/digital_ocean.py b/tests/integration/cloud/providers/digital_ocean.py index 56274668f6..fd5422e0cc 100644 --- a/tests/integration/cloud/providers/digital_ocean.py +++ b/tests/integration/cloud/providers/digital_ocean.py @@ -6,15 +6,14 @@ # Import Python Libs import os -# Import Salt Libs -import integration -from salt.config import cloud_providers_config - # Import Salt Testing Libs from salttesting import skipIf from salttesting.helpers import ensure_in_syspath, expensiveTest ensure_in_syspath('../../../') +# Import Salt Libs +import integration +from salt.config import cloud_providers_config # Import Third-Party Libs try: @@ -38,6 +37,7 @@ class DigitalOceanTest(integration.ShellCase): Integration tests for the Digital Ocean cloud provider in Salt-Cloud ''' + @expensiveTest def setUp(self): ''' Sets up the test requirements @@ -70,7 +70,6 @@ class DigitalOceanTest(integration.ShellCase): .format(provider) ) - @expensiveTest def test_instance(self): ''' Test creating an instance on Digital Ocean From a283d52df123ebfbe6742313f602855186b59cfd Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 23 Jun 2014 01:00:04 +0100 Subject: [PATCH 40/55] Reorder imports and don't even run setUp is expensive tests are not enabled --- tests/integration/cloud/providers/gogrid.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/cloud/providers/gogrid.py b/tests/integration/cloud/providers/gogrid.py index 0a0f4730ac..d5de728d73 100644 --- a/tests/integration/cloud/providers/gogrid.py +++ b/tests/integration/cloud/providers/gogrid.py @@ -6,16 +6,16 @@ # Import Python Libs import os -# Import Salt Libs -import integration -from salt.config import cloud_providers_config - # Import Salt Testing Libs from salttesting import skipIf from salttesting.helpers import ensure_in_syspath, expensiveTest ensure_in_syspath('../../../') +# Import Salt Libs +import integration +from salt.config import cloud_providers_config + # Import Third-Party Libs try: import libcloud # pylint: disable=W0611 @@ -31,6 +31,7 @@ class GoGridTest(integration.ShellCase): Integration tests for the GoGrid cloud provider in Salt-Cloud ''' + @expensiveTest def setUp(self): ''' Sets up the test requirements @@ -63,7 +64,6 @@ class GoGridTest(integration.ShellCase): .format(provider) ) - @expensiveTest def test_instance(self): ''' Test creating an instance on GoGrid From de4892b7eecb26147535ff1925919b528377613e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 23 Jun 2014 01:00:42 +0100 Subject: [PATCH 41/55] Reorder imports and don't even run setUp is expensive tests are not enabled --- tests/integration/cloud/providers/rackspace.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/cloud/providers/rackspace.py b/tests/integration/cloud/providers/rackspace.py index c7f78f9115..85558dbc36 100644 --- a/tests/integration/cloud/providers/rackspace.py +++ b/tests/integration/cloud/providers/rackspace.py @@ -6,16 +6,16 @@ # Import Python Libs import os -# Import Salt Libs -import integration -from salt.config import cloud_providers_config - # Import Salt Testing Libs from salttesting import skipIf from salttesting.helpers import ensure_in_syspath, expensiveTest ensure_in_syspath('../../../') +# Import Salt Libs +import integration +from salt.config import cloud_providers_config + # Import Third-Party Libs try: import libcloud # pylint: disable=W0611 @@ -30,6 +30,7 @@ class RackspaceTest(integration.ShellCase): Integration tests for the Rackspace cloud provider using the Openstack driver ''' + @expensiveTest def setUp(self): ''' Sets up the test requirements @@ -63,7 +64,6 @@ class RackspaceTest(integration.ShellCase): .format(provider) ) - @expensiveTest def test_instance(self): ''' Test creating an instance on rackspace with the openstack driver From 9e76bb4ec37146a22b4ea71c7f8f70cc217d4728 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 23 Jun 2014 01:16:23 +0100 Subject: [PATCH 42/55] Also include the cloud requirements file --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 8bf096fa16..b124c45569 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,7 @@ include LICENSE include README.rst include _requirements.txt include raet-requirements.txt +include cloud-requirements.txt include zeromq-requirements.txt include tests/*.py recursive-include tests * From c5cb5010fce16ddf42eadd1515daf703767b8c72 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 22 Jun 2014 21:42:39 -0500 Subject: [PATCH 43/55] Add example of using py renderer for file.managed --- salt/states/file.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/states/file.py b/salt/states/file.py index 8d58644118..1a75995e5e 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -30,6 +30,17 @@ the jinja templating system would look like this: custom_var: "override" {% endif %} +It is also possible to use the :mod:`py renderer ` as a +templating option. The template would be a python script which would need to +contain a function called ``run()``, which returns a string. The returned +string will be the contents of the managed file. For example: + +.. code-block:: python + + def run(): + lines = ('foo', 'bar', 'baz') + return '\n\n'.join(lines) + .. note:: When using both the ``defaults`` and ``context`` arguments, note the extra From d8efa82b154d71c6d9b1bd61620fb1fe048689ae Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 22 Jun 2014 22:40:40 -0600 Subject: [PATCH 44/55] initial commit for minion swarm to work with raet This skips the single keygen that zeromq transport uses because 1. Key gen is not exposed to salt-key yet for raet 2. raet keys are super fast, so I don't think this will be an issue, but testing on a grander scale will tell us for sure --- tests/minionswarm.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/minionswarm.py b/tests/minionswarm.py index d0ca813ed6..541c3bbec4 100644 --- a/tests/minionswarm.py +++ b/tests/minionswarm.py @@ -66,6 +66,10 @@ def parse(): dest='root_dir', default=None, help='Override the minion root_dir config') + parser.add_option('--transport', + dest='transport', + default='zeromq', + help='Declare which transport to use, default is zeromq') parser.add_option( '-c', '--config-dir', default='/etc/salt', help=('Pass in an alternative configuration directory. Default: ' @@ -99,7 +103,8 @@ class Swarm(object): self.swarm_root = tempfile.mkdtemp(prefix='mswarm-root', suffix='.d', dir=tmpdir) - self.pki = self._pki_dir() + if self.opts['transport'] == 'zeromq': + self.pki = self._pki_dir() self.__zfill = len(str(self.opts['minions'])) self.confs = set() @@ -133,22 +138,25 @@ class Swarm(object): dpath = os.path.join(self.swarm_root, minion_id) os.makedirs(dpath) - minion_pkidir = os.path.join(dpath, 'pki') - os.makedirs(minion_pkidir) - minion_pem = os.path.join(self.pki, 'minion.pem') - minion_pub = os.path.join(self.pki, 'minion.pub') - shutil.copy(minion_pem, minion_pkidir) - shutil.copy(minion_pub, minion_pkidir) - data = { 'id': minion_id, 'user': self.opts['user'], - 'pki_dir': minion_pkidir, 'cachedir': os.path.join(dpath, 'cache'), 'master': self.opts['master'], 'log_file': os.path.join(dpath, 'minion.log') } + if self.opts['transport'] == 'zeromq': + minion_pkidir = os.path.join(dpath, 'pki') + os.makedirs(minion_pkidir) + minion_pem = os.path.join(self.pki, 'minion.pem') + minion_pub = os.path.join(self.pki, 'minion.pub') + shutil.copy(minion_pem, minion_pkidir) + shutil.copy(minion_pub, minion_pkidir) + data['pki_dir'] = minion_pkidir + elif self.opts['transport'] == 'raet': + data['transport'] = 'raet' + if self.opts['root_dir']: data['root_dir'] = self.opts['root_dir'] From 317eef5fa0643c1386d0afe5a5f4b54356f2798c Mon Sep 17 00:00:00 2001 From: lkoenigsberger Date: Mon, 23 Jun 2014 08:49:49 +0200 Subject: [PATCH 45/55] Update msazure.py Added new Sizes (A8, A9) for virtual machines. --- salt/cloud/clouds/msazure.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/cloud/clouds/msazure.py b/salt/cloud/clouds/msazure.py index 502adb9e64..f4682f6c85 100644 --- a/salt/cloud/clouds/msazure.py +++ b/salt/cloud/clouds/msazure.py @@ -216,6 +216,14 @@ def avail_sizes(call=None): 'name': 'A7', 'description': '8 cores, 56GB RAM', }, + 'A8': { + 'name': 'A8', + 'description': '8 cores, 56GB RAM, 40 Gbit/s InfiniBand', + }, + 'A9': { + 'name': 'A9', + 'description': '16 cores, 112GB RAM, 40 Gbit/s InfiniBand', + }, } From 78b6550f9b9f59772a7a4b6f0b2a53c84e476e9f Mon Sep 17 00:00:00 2001 From: Nicolas Limage Date: Mon, 23 Jun 2014 18:24:33 +0200 Subject: [PATCH 46/55] fixes potential ext_pillar line mismatches --- salt/pillar/git_pillar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/pillar/git_pillar.py b/salt/pillar/git_pillar.py index fa69116887..9fcf392ae6 100644 --- a/salt/pillar/git_pillar.py +++ b/salt/pillar/git_pillar.py @@ -102,9 +102,9 @@ class GitPillar(object): self.working_dir = '' self.repo = None - needle = '{0} {1}'.format(self.branch, self.rp_location) for idx, opts_dict in enumerate(self.opts['ext_pillar']): - if opts_dict.get('git', '').startswith(needle): + lopts = opts_dict.get('git', '').split() + if len(lopts) >= 2 and lopts[:2] == [self.branch, self.rp_location]: rp_ = os.path.join(self.opts['cachedir'], 'pillar_gitfs', str(idx)) From 77836a173a7d796230fabe2758ec8a32b108b76a Mon Sep 17 00:00:00 2001 From: Chad Heuschober Date: Mon, 23 Jun 2014 13:55:26 -0400 Subject: [PATCH 47/55] Fixes exception that fires when config_get() is called and user has no .gitconfig --- salt/states/git.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/states/git.py b/salt/states/git.py index 6b3d57a580..d0029eeae8 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -484,7 +484,7 @@ def config(name, cwd=repo, user=user) except CommandExecutionError: - oval = 'None' + oval = None if value == oval: ret['comment'] = 'No changes made' @@ -504,6 +504,9 @@ def config(name, cwd=repo, user=user) + if oval is None: + oval = 'None' + ret['changes'][name] = '{0} => {1}'.format(oval, nval) return ret From e0a6a5a13ec28c4042550d749fd5969b21a8b38a Mon Sep 17 00:00:00 2001 From: Henrik Holmboe Date: Mon, 23 Jun 2014 19:56:39 +0200 Subject: [PATCH 48/55] Reflect the latest released version I did this exact PR in #12695 the last time around. Luckily #12720 seems to take a stab at automating this. :) --- doc/topics/releases/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/releases/index.rst b/doc/topics/releases/index.rst index 48b411a1a0..2d58d89776 100644 --- a/doc/topics/releases/index.rst +++ b/doc/topics/releases/index.rst @@ -5,7 +5,7 @@ Release notes .. releasestree:: :maxdepth: 1 - 2014.1.4 + 2014.1.5 Archive ======= From 4260fab3be21038138810c6531051e120ad344e2 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Mon, 23 Jun 2014 20:09:01 +0200 Subject: [PATCH 49/55] Corrected tiny typo --- salt/states/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/cmd.py b/salt/states/cmd.py index e1ec147912..70d58fddac 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -158,7 +158,7 @@ executed when the state it is watching changes. Example: - require: - file: /usr/local/bin/postinstall.sh -How do I create a environment from a pillar map? +How do I create an environment from a pillar map? ------------------------------------------------------------------------------- The map that comes from a pillar cannot be directly consumed by the env option. From d9f53a37f9f3de40117e7ebada0f76ae5a7be748 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 23 Jun 2014 12:17:26 -0600 Subject: [PATCH 50/55] One more cloud config unit test --- tests/unit/config_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 29c2c9ea32..d47a6f749e 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -550,6 +550,16 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn): self.assertRaises(SaltCloudConfigError, sconfig.cloud_config, PATH, providers_config_path='bar') + # apply_vm_profiles_config tests + + def test_apply_vm_profiles_config_bad_profile_format(self): + ''' + Tests passing in a bad profile format in overrides + ''' + overrides = {'foo': 'bar', 'conf_file': PATH} + self.assertRaises(SaltCloudConfigError, sconfig.apply_vm_profiles_config, + PATH, overrides, defaults=DEFAULT) + # apply_cloud_providers_config tests def test_apply_cloud_providers_config_same_providers(self): From 221b2706d1a183d1e7a845cd2b0b11e3ea247418 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 16 Jun 2014 15:05:09 -0700 Subject: [PATCH 51/55] Changes to schedule.modify to show a diff of what has changes when a scheduled job is updated. Also some cleanup of duplicated code between add and modify, now moved into a common build_schedule_item function. Some additional cleanup and the initial work on a state to manage scheduled jobs. --- salt/modules/schedule.py | 177 +++++++++++++++++++----------- salt/states/schedule.py | 226 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 339 insertions(+), 64 deletions(-) create mode 100644 salt/states/schedule.py diff --git a/salt/modules/schedule.py b/salt/modules/schedule.py index 3e95e9f2e0..215b66985e 100644 --- a/salt/modules/schedule.py +++ b/salt/modules/schedule.py @@ -7,10 +7,12 @@ Module for manging the Salt schedule on a minion ''' # Import Python libs +import difflib import os import yaml import salt.utils +import salt.utils.odict __proxyenabled__ = ['*'] @@ -23,6 +25,8 @@ __func_alias__ = { } SCHEDULE_CONF = [ + 'name', + 'maxrunning', 'function', 'splay', 'range', @@ -40,7 +44,7 @@ SCHEDULE_CONF = [ ] -def list_(show_all=False): +def list_(show_all=False, return_yaml=True): ''' List the jobs currently scheduled on the minion @@ -81,9 +85,12 @@ def list_(show_all=False): del schedule[job]['_seconds'] if schedule: - tmp = {'schedule': schedule} - yaml_out = yaml.safe_dump(tmp, default_flow_style=False) - return yaml_out + if return_yaml: + tmp = {'schedule': schedule} + yaml_out = yaml.safe_dump(tmp, default_flow_style=False) + return yaml_out + else: + return schedule else: return None @@ -147,7 +154,6 @@ def delete(name): ret['comment'] = 'Failed to delete job {0} from schedule.'.format(name) ret['result'] = False elif 'schedule' in __pillar__ and name in __pillar__['schedule']: - log.debug('found job in pillar') out = __salt__['event.fire']({'name': name, 'where': 'pillar', 'func': 'delete'}, 'manage_schedule') if out: ret['comment'] = 'Deleted Job {0} from schedule.'.format(name) @@ -160,6 +166,75 @@ def delete(name): return ret +def build_schedule_item(name, **kwargs): + ''' + Build a schedule job + + CLI Example: + + .. code-block:: bash + + salt '*' schedule.build_schedule_item job1 function='test.ping' seconds=3600 + ''' + + ret = {'comment': [], + 'result': True} + + if not name: + ret['comment'] = 'Job name is required.' + ret['result'] = False + + schedule = {} + schedule[name] = salt.utils.odict.OrderedDict() + schedule[name]['function'] = kwargs['function'] + + time_conflict = False + for item in ['seconds', 'minutes', 'hours', 'days']: + if item in kwargs and 'when' in kwargs: + time_conflict = True + + if time_conflict: + return 'Error: Unable to use "seconds", "minutes", "hours", or "days" with "when" option.' + + for item in ['seconds', 'minutes', 'hours', 'days']: + if item in kwargs: + schedule[name][item] = kwargs[item] + + if 'job_args' in kwargs: + schedule[name]['args'] = kwargs['job_args'] + + if 'job_kwargs' in kwargs: + schedule[name]['kwargs'] = kwargs['job_kwargs'] + + if 'maxrunning' in kwargs: + schedule[name]['maxrunning'] = kwargs['maxrunning'] + else: + schedule[name]['maxrunning'] = 1 + + if 'name' in kwargs: + schedule[name]['name'] = kwargs['name'] + else: + schedule[name]['name'] = name + + if 'jid_include' not in kwargs or kwargs['jid_include']: + schedule[name]['jid_include'] = True + + if 'splay' in kwargs: + if isinstance(kwargs['splay'], dict): + # Ensure ordering of start and end arguments + schedule[name]['splay'] = salt.utils.odict.OrderedDict() + schedule[name]['splay']['start'] = kwargs['splay']['start'] + schedule[name]['splay']['end'] = kwargs['splay']['end'] + else: + schedule[name]['splay'] = kwargs['splay'] + + for item in ['range', 'when', 'returner']: + if item in kwargs: + schedule[name][item] = kwargs[item] + + return schedule[name] + + def add(name, **kwargs): ''' Add a job to the schedule @@ -187,30 +262,19 @@ def add(name, **kwargs): ret['comment'] = 'Job name is required.' ret['result'] = False - schedule = {} - schedule[name] = {'function': kwargs['function']} - time_conflict = False for item in ['seconds', 'minutes', 'hours', 'days']: if item in kwargs and 'when' in kwargs: time_conflict = True if time_conflict: - return 'Error: Unable to use "seconds", "minutes", "hours", or "days" with "when" option.' + ret['result'] = False + ret['comment'] = 'Error: Unable to use "seconds", "minutes", "hours", or "days" with "when" option.' - for item in ['seconds', 'minutes', 'hours', 'days']: - if item in kwargs: - schedule[name][item] = kwargs[item] + _new = build_schedule_item(name, **kwargs) - if 'job_args' in kwargs: - schedule[name]['args'] = kwargs['job_args'] - - if 'job_kwargs' in kwargs: - schedule[name]['kwargs'] = kwargs['job_kwargs'] - - for item in ['splay', 'range', 'when', 'returner', 'jid_include']: - if item in kwargs: - schedule[name][item] = kwargs[item] + schedule = {} + schedule[name] = _new out = __salt__['event.fire']({'name': name, 'schedule': schedule, 'func': 'add'}, 'manage_schedule') if out: @@ -232,9 +296,19 @@ def modify(name, **kwargs): salt '*' schedule.modify job1 function='test.ping' seconds=3600 ''' - ret = {'comment': [], + ret = {'comment': '', + 'changes': {}, 'result': True} + time_conflict = False + for item in ['seconds', 'minutes', 'hours', 'days']: + if item in kwargs and 'when' in kwargs: + time_conflict = True + + if time_conflict: + ret['result'] = False + ret['comment'] = 'Error: Unable to use "seconds", "minutes", "hours", or "days" with "when" option.' + current_schedule = __opts__['schedule'].copy() if 'schedule' in __pillar__: current_schedule.update(__pillar__['schedule']) @@ -244,39 +318,33 @@ def modify(name, **kwargs): ret['result'] = False return ret - schedule = {'function': kwargs['function']} + _current = current_schedule[name] + if '_seconds' in _current: + _current['seconds'] = _current['_seconds'] + del _current['_seconds'] - time_conflict = False - for item in ['seconds', 'minutes', 'hours', 'days']: - if item in kwargs and 'when' in kwargs: - time_conflict = True + _new = build_schedule_item(name, **kwargs) + if _new == _current: + ret['comment'] = 'Job {0} in correct state'.format(name) + return ret - if time_conflict: - return 'Error: Unable to use "seconds", "minutes", "hours", or "days" with "when" option.' + _current_lines = ['%s:%s\n' % (key, value) + for (key, value) in sorted(_current.items())] + _new_lines = ['%s:%s\n' % (key, value) + for (key, value) in sorted(_new.items())] + _diff = difflib.unified_diff(_current_lines, _new_lines) - for item in ['seconds', 'minutes', 'hours', 'days']: - if item in kwargs: - schedule[item] = kwargs[item] - - if 'job_args' in kwargs: - schedule['args'] = kwargs['job_args'] - - if 'job_kwargs' in kwargs: - schedule['kwargs'] = kwargs['job_kwargs'] - - for item in ['splay', 'range', 'when', 'returner', 'jid_include']: - if item in kwargs: - schedule[item] = kwargs[item] + ret['changes']['diff'] = ''.join(_diff) if name in __opts__['schedule']: - out = __salt__['event.fire']({'name': name, 'schedule': schedule, 'func': 'modify'}, 'manage_schedule') + out = __salt__['event.fire']({'name': name, 'schedule': _new, 'func': 'modify'}, 'manage_schedule') if out: ret['comment'] = 'Modified job: {0} in schedule.'.format(name) else: ret['comment'] = 'Failed to modify job {0} in schedule.'.format(name) ret['result'] = False elif 'schedule' in __pillar__ and name in __pillar__['schedule']: - out = __salt__['event.fire']({'name': name, 'schedule': schedule, 'where': 'pillar', 'func': 'modify'}, 'manage_schedule') + out = __salt__['event.fire']({'name': name, 'schedule': _new, 'where': 'pillar', 'func': 'modify'}, 'manage_schedule') if out: ret['comment'] = 'Modified job: {0} in schedule.'.format(name) else: @@ -424,26 +492,7 @@ def save(): ret = {'comment': [], 'result': True} - schedule = __opts__['schedule'] - for job in schedule.keys(): - if job == 'enabled': - continue - if job.startswith('_'): - del schedule[job] - continue - - for item in schedule[job].keys(): - if item not in SCHEDULE_CONF: - del schedule[job][item] - continue - if schedule[job][item] == 'true': - schedule[job][item] = True - if schedule[job][item] == 'false': - schedule[job][item] = False - - if '_seconds' in schedule[job].keys(): - schedule[job]['seconds'] = schedule[job]['_seconds'] - del schedule[job]['_seconds'] + schedule = list_(return_yaml=False) # move this file into an configurable opt sfn = '{0}/{1}/schedule.conf'.format(__opts__['config_dir'], os.path.dirname(__opts__['default_include'])) diff --git a/salt/states/schedule.py b/salt/states/schedule.py new file mode 100644 index 0000000000..ad24d0defa --- /dev/null +++ b/salt/states/schedule.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +''' +Management of cron, the Unix command scheduler +============================================== + +Cron declarations require a number of parameters. The following are the +parameters used by Salt to define the various timing values for a cron job: + +* ``minute`` +* ``hour`` +* ``daymonth`` +* ``month`` +* ``dayweek`` (0 to 6 are Sunday through Saturday, 7 can also be used for + Sunday) + +.. warning:: + + Any timing arguments not specified take a value of ``*``. This means that + setting ``hour`` to ``5``, while not defining the ``minute`` param, will + result in Salt adding a job that will execute every minute between 5 and 6 + A.M.! + + Additionally, the default user for these states is ``root``. Therefore, if + the cron job is for another user, it is necessary to specify that user with + the ``user`` parameter. + +A long time ago (before 2014.2), when making changes to an existing cron job, +the name declaration is the parameter used to uniquely identify the job, +so if an existing cron that looks like this: + +.. code-block:: yaml + + date > /tmp/crontest: + cron.present: + - user: root + - minute: 5 + +Is changed to this: + +.. code-block:: yaml + + date > /tmp/crontest: + cron.present: + - user: root + - minute: 7 + - hour: 2 + +Then the existing cron will be updated, but if the cron command is changed, +then a new cron job will be added to the user's crontab. + +The current behavior is still relying on that mechanism, but you can also +specify an identifier to identify your crontabs: +.. versionadded:: 2014.2 +.. code-block:: yaml + + date > /tmp/crontest: + cron.present: + - identifier: SUPERCRON + - user: root + - minute: 7 + - hour: 2 + +And, some months later, you modify it: +.. versionadded:: 2014.2 +.. code-block:: yaml + + superscript > /tmp/crontest: + cron.present: + - identifier: SUPERCRON + - user: root + - minute: 3 + - hour: 4 + +The old **date > /tmp/crontest** will be replaced by +**superscript > /tmp/crontest**. + +Additionally, Salt also supports running a cron every ``x minutes`` very similarly to the Unix +convention of using ``*/5`` to have a job run every five minutes. In Salt, this +looks like: + +.. code-block:: yaml + + date > /tmp/crontest: + cron.present: + - user: root + - minute: '*/5' + +The job will now run every 5 minutes. + +Additionally, the temporal parameters (minute, hour, etc.) can be randomized by +using ``random`` instead of using a specific value. For example, by using the +``random`` keyword in the ``minute`` parameter of a cron state, the same cron +job can be pushed to hundreds or thousands of hosts, and they would each use a +randomly-generated minute. This can be helpful when the cron job accesses a +network resource, and it is not desirable for all hosts to run the job +concurrently. + +.. code-block:: yaml + + /path/to/cron/script: + cron.present: + - user: root + - minute: random + - hour: 2 + +.. versionadded:: 0.16.0 + +Since Salt assumes a value of ``*`` for unspecified temporal parameters, adding +a parameter to the state and setting it to ``random`` will change that value +from ``*`` to a randomized numeric value. However, if that field in the cron +entry on the minion already contains a numeric value, then using the ``random`` +keyword will not modify it. +''' + +import logging +log = logging.getLogger(__name__) + + +def present(name, + **kwargs): + ''' + Verifies that the specified cron job is present for the specified user. + For more advanced information about what exactly can be set in the cron + timing parameters, check your cron system's documentation. Most Unix-like + systems' cron documentation can be found via the crontab man page: + ``man 5 crontab``. + + name + The command that should be executed by the cron job. + + minute + The information to be set into the minute section, this can be any + string supported by your cron system's the minute field. Default is + ``*`` + + hour + The information to be set in the hour section. Default is ``*`` + + daymonth + The information to be set in the day of month section. Default is ``*`` + + month + The information to be set in the month section. Default is ``*`` + + dayweek + The information to be set in the day of week section. Default is ``*`` + + comment + User comment to be added on line previous the cron job + + identifier + Custom-defined identifier for tracking the cron line for future crontab + edits. This defaults to the state id + ''' + + ret = {'name': name, + 'result': True, + 'changes': {}, + 'comment': []} + + current_schedule = __salt__['schedule.list'](show_all=True, return_yaml=False) + + if name in current_schedule: + new_item = __salt__['schedule.build_schedule_item'](name, **kwargs) + if new_item == current_schedule[name]: + ret['comment'].append('Job {0} in correct state'.format(name)) + else: + result = __salt__['schedule.modify'](name, **kwargs) + if not result['result']: + ret['result'] = result['result'] + ret['comment'].append(result['comment']) + return ret + else: + ret['comment'].append('Modifying job {0} in schedule'.format(name)) + ret['changes'] = result['changes'] + else: + result = __salt__['schedule.add'](name, **kwargs) + if not result['result']: + ret['result'] = result['result'] + ret['comment'].append(result['comment']) + return ret + else: + ret['comment'].append('Adding new job {0} to schedule'.format(name)) + + ret['comment'] = '\n'.join(ret['comment']) + return ret + + +def absent(name, **kwargs): + ''' + Verifies that the specified cron job is absent for the specified user; only + the name is matched when removing a cron job. + + name + The command that should be absent in the user crontab. + + user + The name of the user whose crontab needs to be modified, defaults to + the root user + + identifier + Custom-defined identifier for tracking the cron line for future crontab + edits. This defaults to the state id + ''' + ### NOTE: The keyword arguments in **kwargs are ignored in this state, but + ### cannot be removed from the function definition, otherwise the use + ### of unsupported arguments will result in a traceback. + + ret = {'name': name, + 'result': True, + 'changes': {}, + 'comment': []} + + current_schedule = __salt__['schedule.list'](show_all=True, return_yaml=False) + if name in current_schedule: + result = __salt__['schedule.delete'](name) + if not result['result']: + ret['result'] = result['result'] + ret['comment'].append(result['comment']) + else: + ret['comment'].append('Removed job {0} from schedule'.format(name)) + else: + ret['comment'].append('Job {0} not present in schedule'.format(name)) + + ret['comment'] = '\n'.join(ret['comment']) + return ret From 60fdd152337a5c2fabbeb723881fb52ea1577e17 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Sun, 22 Jun 2014 19:49:14 -0700 Subject: [PATCH 52/55] lint fixes --- salt/states/schedule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/schedule.py b/salt/states/schedule.py index ad24d0defa..3101ac49c5 100644 --- a/salt/states/schedule.py +++ b/salt/states/schedule.py @@ -220,7 +220,7 @@ def absent(name, **kwargs): else: ret['comment'].append('Removed job {0} from schedule'.format(name)) else: - ret['comment'].append('Job {0} not present in schedule'.format(name)) + ret['comment'].append('Job {0} not present in schedule'.format(name)) ret['comment'] = '\n'.join(ret['comment']) return ret From c95452ba2fb0ef87308c3870b310b7d140d3529a Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 23 Jun 2014 13:42:22 -0700 Subject: [PATCH 53/55] fixing docstrings. --- salt/states/schedule.py | 209 +++++++++++++++------------------------- salt/utils/schedule.py | 5 +- 2 files changed, 78 insertions(+), 136 deletions(-) diff --git a/salt/states/schedule.py b/salt/states/schedule.py index 3101ac49c5..2018d4757d 100644 --- a/salt/states/schedule.py +++ b/salt/states/schedule.py @@ -1,115 +1,47 @@ # -*- coding: utf-8 -*- ''' -Management of cron, the Unix command scheduler +Management of the Salt scheduler ============================================== -Cron declarations require a number of parameters. The following are the -parameters used by Salt to define the various timing values for a cron job: - -* ``minute`` -* ``hour`` -* ``daymonth`` -* ``month`` -* ``dayweek`` (0 to 6 are Sunday through Saturday, 7 can also be used for - Sunday) - -.. warning:: - - Any timing arguments not specified take a value of ``*``. This means that - setting ``hour`` to ``5``, while not defining the ``minute`` param, will - result in Salt adding a job that will execute every minute between 5 and 6 - A.M.! - - Additionally, the default user for these states is ``root``. Therefore, if - the cron job is for another user, it is necessary to specify that user with - the ``user`` parameter. - -A long time ago (before 2014.2), when making changes to an existing cron job, -the name declaration is the parameter used to uniquely identify the job, -so if an existing cron that looks like this: - .. code-block:: yaml - date > /tmp/crontest: - cron.present: - - user: root - - minute: 5 + job3: + schedule.present: + - function: test.ping + - seconds: 3600 + - splay: 10 -Is changed to this: + This will schedule the command: test.ping every 3600 seconds + (every hour) splaying the time between 0 and 10 seconds -.. code-block:: yaml + job2: + schedule.present: + - function: test.ping + - seconds: 15 + - splay: + - start: 10 + - end: 20 - date > /tmp/crontest: - cron.present: - - user: root - - minute: 7 - - hour: 2 + This will schedule the command: test.ping every 3600 seconds + (every hour) splaying the time between 10 and 20 seconds -Then the existing cron will be updated, but if the cron command is changed, -then a new cron job will be added to the user's crontab. + job1: + schedule.present: + - function: state.sls + - args: + - httpd + - kwargs: + test: True + - when: + - Monday 5:00pm + - Tuesday 3:00pm + - Wednesday 5:00pm + - Thursday 3:00pm + - Friday 5:00pm -The current behavior is still relying on that mechanism, but you can also -specify an identifier to identify your crontabs: -.. versionadded:: 2014.2 -.. code-block:: yaml + This will schedule the command: state.sls httpd test=True at 5pm on Monday, + Wednesday and Friday, and 3pm on Tuesday and Thursday. - date > /tmp/crontest: - cron.present: - - identifier: SUPERCRON - - user: root - - minute: 7 - - hour: 2 - -And, some months later, you modify it: -.. versionadded:: 2014.2 -.. code-block:: yaml - - superscript > /tmp/crontest: - cron.present: - - identifier: SUPERCRON - - user: root - - minute: 3 - - hour: 4 - -The old **date > /tmp/crontest** will be replaced by -**superscript > /tmp/crontest**. - -Additionally, Salt also supports running a cron every ``x minutes`` very similarly to the Unix -convention of using ``*/5`` to have a job run every five minutes. In Salt, this -looks like: - -.. code-block:: yaml - - date > /tmp/crontest: - cron.present: - - user: root - - minute: '*/5' - -The job will now run every 5 minutes. - -Additionally, the temporal parameters (minute, hour, etc.) can be randomized by -using ``random`` instead of using a specific value. For example, by using the -``random`` keyword in the ``minute`` parameter of a cron state, the same cron -job can be pushed to hundreds or thousands of hosts, and they would each use a -randomly-generated minute. This can be helpful when the cron job accesses a -network resource, and it is not desirable for all hosts to run the job -concurrently. - -.. code-block:: yaml - - /path/to/cron/script: - cron.present: - - user: root - - minute: random - - hour: 2 - -.. versionadded:: 0.16.0 - -Since Salt assumes a value of ``*`` for unspecified temporal parameters, adding -a parameter to the state and setting it to ``random`` will change that value -from ``*`` to a randomized numeric value. However, if that field in the cron -entry on the minion already contains a numeric value, then using the ``random`` -keyword will not modify it. ''' import logging @@ -119,38 +51,57 @@ log = logging.getLogger(__name__) def present(name, **kwargs): ''' - Verifies that the specified cron job is present for the specified user. - For more advanced information about what exactly can be set in the cron - timing parameters, check your cron system's documentation. Most Unix-like - systems' cron documentation can be found via the crontab man page: - ``man 5 crontab``. + Ensure a job is present in the schedule name - The command that should be executed by the cron job. + The unique name that is given to the scheduled job. - minute - The information to be set into the minute section, this can be any - string supported by your cron system's the minute field. Default is - ``*`` + seconds + The scheduled job will be executed after the specified + number of seconds have passed. - hour - The information to be set in the hour section. Default is ``*`` + minutes + The scheduled job will be executed after the specified + number of minutes have passed. - daymonth - The information to be set in the day of month section. Default is ``*`` + hours + The scheduled job will be executed after the specified + number of hours have passed. - month - The information to be set in the month section. Default is ``*`` + days + The scheduled job will be executed after the specified + number of days have passed. - dayweek - The information to be set in the day of week section. Default is ``*`` + when + This will schedule the job at the specified time(s). + The when parameter must be a single value or a dictionary + with the date string(s) using the dateutil format. - comment - User comment to be added on line previous the cron job + function + The function that should be executed by the scheduled job. + + job_args + The arguments that will be used by the scheduled job. + + job_kwargs + The keyword arguments that will be used by the scheduled job. + + maxrunning + Ensure that there are no more than N copies of a particular job running. + + jid_include + Include the job into the job cache. + + splay + The amount of time in seconds to splay a scheduled job. + Can be specified as a single value in seconds or as a dictionary + range with 'start' and 'end' values. + + range + This will schedule the command within the range specified. + The range parameter must be a dictionary with the date strings + using the dateutil format. - identifier - Custom-defined identifier for tracking the cron line for future crontab - edits. This defaults to the state id ''' ret = {'name': name, @@ -188,19 +139,11 @@ def present(name, def absent(name, **kwargs): ''' - Verifies that the specified cron job is absent for the specified user; only - the name is matched when removing a cron job. + Ensure a job is absent from the schedule name - The command that should be absent in the user crontab. + The unique name that is given to the scheduled job. - user - The name of the user whose crontab needs to be modified, defaults to - the root user - - identifier - Custom-defined identifier for tracking the cron line for future crontab - edits. This defaults to the state id ''' ### NOTE: The keyword arguments in **kwargs are ignored in this state, but ### cannot be removed from the function definition, otherwise the use diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 8f1d8bbc6f..eb33ae4317 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -77,8 +77,8 @@ localtime. - Thursday 3:00pm - Friday 5:00pm -This will schedule the command: state.sls httpd test=True at 5pm on Monday, Wednesday -and Friday, and 3pm on Tuesday and Thursday. +This will schedule the command: state.sls httpd test=True at 5pm on Monday, +Wednesday and Friday, and 3pm on Tuesday and Thursday. schedule: job1: @@ -92,7 +92,6 @@ and Friday, and 3pm on Tuesday and Thursday. start: 8:00am end: 5:00pm -w This will schedule the command: state.sls httpd test=True every 3600 seconds (every hour) between the hours of 8am and 5pm. The range parameter must be a dictionary with the date strings using the dateutil format. From aa9ae49129d69e41001237316c3fcf17a2cce044 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 23 Jun 2014 16:00:19 -0500 Subject: [PATCH 54/55] Add additional information to the comments for the shar script --- pkg/shar/build_shar.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/shar/build_shar.sh b/pkg/shar/build_shar.sh index 083911cc2e..ac2ac860d9 100755 --- a/pkg/shar/build_shar.sh +++ b/pkg/shar/build_shar.sh @@ -13,6 +13,10 @@ # # It will fetch libzmq and build it as a pyzmq extension. # +# IMPORTANT: Unpacking the shar requires uudecode, which is distributed along +# with sharutils. Thus, you should have sharutils installed on any host which +# will need to unpack the shar archive. +# # The script is capable of building a shar archive using several methods: # # 1. Using a custom pip requirements file @@ -27,8 +31,10 @@ # option can be used to specify directory from which dependencies will be # sourced. Any missing dependencies will be retrieved with pip. # -# It is recommended to run this script on a machine which does not have any of -# the Salt dependencies already installed. +# It is strongly recommended to run this script on a machine which does not +# have any of the Salt dependencies already installed, because if the script +# detects that ZeroMQ is already installed, then pyzmq's setup.py will not +# build a bundled ZeroMQ. # # Run the script with -h for usage details. # From f821ccaa880317a9899dec4f4485773085923a6b Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 23 Jun 2014 15:19:11 -0600 Subject: [PATCH 55/55] More cloud profile config tests --- tests/unit/config_test.py | 48 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index d47a6f749e..416f0e6b53 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -560,6 +560,54 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn): self.assertRaises(SaltCloudConfigError, sconfig.apply_vm_profiles_config, PATH, overrides, defaults=DEFAULT) + def test_apply_vm_profiles_config_success(self): + ''' + Tests passing in valid provider and profile config files successfully + ''' + providers = {'test-provider': + {'digital_ocean': + {'provider': 'digital_ocean', 'profiles': {}}}} + overrides = {'test-profile': + {'provider': 'test-provider', + 'image': 'Ubuntu 12.10 x64', + 'size': '512MB'}, + 'conf_file': PATH} + ret = {'test-profile': + {'profile': 'test-profile', + 'provider': 'test-provider:digital_ocean', + 'image': 'Ubuntu 12.10 x64', + 'size': '512MB'}} + self.assertEqual(sconfig.apply_vm_profiles_config(providers, + overrides, + defaults=DEFAULT), ret) + + def test_apply_vm_profiles_config_extend_success(self): + ''' + Tests profile extends functionality with valid provider and profile configs + ''' + providers = {'test-config': {'ec2': {'profiles': {}, 'provider': 'ec2'}}} + overrides = {'Amazon': {'image': 'test-image-1', + 'extends': 'dev-instances'}, + 'Fedora': {'image': 'test-image-2', + 'extends': 'dev-instances'}, + 'conf_file': PATH, + 'dev-instances': {'ssh_username': 'test_user', + 'provider': 'test-config'}} + ret = {'Amazon': {'profile': 'Amazon', + 'ssh_username': 'test_user', + 'image': 'test-image-1', + 'provider': 'test-config:ec2'}, + 'Fedora': {'profile': 'Fedora', + 'ssh_username': 'test_user', + 'image': 'test-image-2', + 'provider': 'test-config:ec2'}, + 'dev-instances': {'profile': 'dev-instances', + 'ssh_username': 'test_user', + 'provider': 'test-config:ec2'}} + self.assertEqual(sconfig.apply_vm_profiles_config(providers, + overrides, + defaults=DEFAULT), ret) + # apply_cloud_providers_config tests def test_apply_cloud_providers_config_same_providers(self):