From 182e988fd19cbd3f10b29251223144e96525c72f Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Tue, 12 Jul 2016 13:02:11 -0600 Subject: [PATCH 01/95] Properly gate pythoncom --- salt/utils/winapi.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/salt/utils/winapi.py b/salt/utils/winapi.py index b1d63375d2..26d89d8d49 100644 --- a/salt/utils/winapi.py +++ b/salt/utils/winapi.py @@ -4,11 +4,32 @@ from __future__ import absolute_import # Import python libs import logging -import pythoncom import threading +from salt.loader import minion_mods + +try: + import pythoncom + HAS_LIBS = True +except ImportError: + HAS_LIBS = False log = logging.getLogger(__name__) +__salt__ = None + + +def __virtual__(): + ''' + Only load if required libraries exist + ''' + if not HAS_LIBS: + return False + else: + global __salt__ + if not __salt__: + __salt__ = minion_mods(__opts__) + return True + class Com(object): def __init__(self): From dbd8482eb5f0dc1ef271d0e61b3a7e64981c2636 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 22 Jun 2016 16:56:50 -0700 Subject: [PATCH 02/95] Fix integration tests on windows --- tests/integration/__init__.py | 19 ++++++++++--------- tests/integration/files/conf/master | 7 ++----- tests/integration/files/conf/minion | 9 +++++---- tests/integration/files/conf/sub_minion | 12 +++++------- tests/integration/files/conf/syndic_master | 6 +++--- tests/integration/states/pkg.py | 1 + 6 files changed, 26 insertions(+), 28 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 92b1a4aba0..6c0d64f957 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -717,13 +717,14 @@ class TestDaemon(object): self.master_process.display_name = 'salt-master' self.minion_process = SaltMinion(self.minion_opts, TMP_CONF_DIR, SCRIPT_DIR) self.minion_process.display_name = 'salt-minion' - self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) - self.sub_minion_process.display_name = 'sub salt-minion' + #self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) + #self.sub_minion_process.display_name = 'sub salt-minion' self.smaster_process = SaltMaster(self.syndic_master_opts, TMP_SYNDIC_MASTER_CONF_DIR, SCRIPT_DIR) self.smaster_process.display_name = 'syndic salt-master' self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR) self.syndic_process.display_name = 'salt-syndic' - for process in (self.master_process, self.minion_process, self.sub_minion_process, + #for process in (self.master_process, self.minion_process, self.sub_minion_process, + for process in (self.master_process, self.minion_process, self.smaster_process, self.syndic_process): sys.stdout.write( ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format( @@ -978,19 +979,19 @@ class TestDaemon(object): running_tests_user = win32api.GetUserName() else: running_tests_user = pwd.getpwuid(os.getuid()).pw_name - master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'master')) - master_opts['user'] = running_tests_user + tests_known_hosts_file = os.path.join(TMP_CONF_DIR, 'salt_ssh_known_hosts') with salt.utils.fopen(tests_known_hosts_file, 'w') as known_hosts: known_hosts.write('') + + master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'master')) + master_opts['user'] = running_tests_user master_opts['known_hosts_file'] = tests_known_hosts_file master_opts['conf_dir'] = TMP_CONF_DIR - minion_config_path = os.path.join(CONF_DIR, 'minion') - minion_opts = salt.config._read_conf_file(minion_config_path) + minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'minion')) minion_opts['user'] = running_tests_user minion_opts['conf_dir'] = TMP_CONF_DIR - minion_opts['root_dir'] = master_opts['root_dir'] = os.path.join(TMP, 'rootdir') sub_minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'sub_minion')) @@ -1209,7 +1210,7 @@ class TestDaemon(object): ''' Kill the minion and master processes ''' - self.sub_minion_process.terminate() + #self.sub_minion_process.terminate() self.minion_process.terminate() self.master_process.terminate() try: diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index dd5f30e5bc..7ac7cb2bc4 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -5,11 +5,8 @@ publish_port: 64505 ret_port: 64506 worker_threads: 3 root_dir: /tmp/salttest -pidfile: masterpid -pki_dir: pki -cachedir: cache +pidfile: master.pid timeout: 3 -sock_dir: .salt-unix open_mode: True syndic_master: localhost fileserver_list_cache_time: 0 @@ -17,7 +14,7 @@ pillar_opts: True peer: '.*': - 'test.*' -log_file: master +log_file: master.log log_level_logfile: debug key_logfile: key token_file: /tmp/ksfjhdgiuebfgnkefvsikhfjdgvkjahcsidk diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index 0ccfe90ec0..630887efaf 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -1,17 +1,18 @@ # basic config master: localhost master_port: 64506 +tcp_pub_port: 64512 +tcp_pull_port: 64513 +interface: 127.0.0.1 root_dir: /tmp/salttest -pki_dir: pki id: minion -cachedir: cachedir -sock_dir: minion_sock #acceptance_wait_time: = 1 open_mode: True -log_file: minion +log_file: minion.log log_level_logfile: debug #loop_interval: 0.05 config_dir: /tmp/salt-tests-tmpdir +pidfile: minion.pid # module extension test.foo: baz diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index c21e4d9785..623b741599 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -1,15 +1,16 @@ # basic config master: localhost master_port: 64506 +tcp_pub_port: 64510 +tcp_pull_port: 64511 root_dir: /tmp/subsalttest -pki_dir: pki -id: sub_minion -cachedir: cachedir sock_dir: sub_minion_sock +id: sub_minion #acceptance_wait_time: 1 open_mode: True -log_file: sub_minion +log_file: sub_minion.log log_level_logfile: debug +pidfile: sub_minion.pid # module extension test.foo: baz @@ -33,6 +34,3 @@ grains: - jamie - zoe -ipc_mode: tcp -tcp_pub_port: 64510 -tcp_pull_port: 64511 diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index a88cb36b43..dbd6ed0da7 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -4,16 +4,16 @@ publish_port: 54505 ret_port: 54506 worker_threads: 3 root_dir: /tmp/saltsyndictest -pidfile: syndicmasterpid +pidfile: syndic_master.pid pki_dir: pki cachedir: cache timeout: 1 -sock_dir: .salt-unix-syndic +sock_dir: syndic_master_sock open_mode: True order_masters: True fileserver_list_cache_time: 0 pillar_opts: True tcp_master_publish_pull: 33305 tcp_master_workers: 33306 -log_file: syndic_master +log_file: syndic_master.log log_level_logfile: debug diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index 846cee0b16..39a6279652 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -37,6 +37,7 @@ _PKG_TARGETS = { 'FreeBSD': ['aalib', 'pth'], 'SUSE': ['aalib', 'python-pssh'], 'MacOS': ['libpng', 'jpeg'], + 'Windows': ['firefox', '7zip'], } _PKG_TARGETS_32 = { From a28e0782e5b294269aec594b7d8ad61cce5edaad Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 24 Jun 2016 08:10:00 -0700 Subject: [PATCH 03/95] Fix subminion --- tests/integration/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 6c0d64f957..3b692bf676 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -717,14 +717,14 @@ class TestDaemon(object): self.master_process.display_name = 'salt-master' self.minion_process = SaltMinion(self.minion_opts, TMP_CONF_DIR, SCRIPT_DIR) self.minion_process.display_name = 'salt-minion' - #self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) - #self.sub_minion_process.display_name = 'sub salt-minion' + self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) + self.sub_minion_process.display_name = 'sub salt-minion' self.smaster_process = SaltMaster(self.syndic_master_opts, TMP_SYNDIC_MASTER_CONF_DIR, SCRIPT_DIR) self.smaster_process.display_name = 'syndic salt-master' self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR) self.syndic_process.display_name = 'salt-syndic' - #for process in (self.master_process, self.minion_process, self.sub_minion_process, - for process in (self.master_process, self.minion_process, + for process in (self.master_process, self.minion_process, self.sub_minion_process, + #for process in (self.master_process, self.minion_process, self.smaster_process, self.syndic_process): sys.stdout.write( ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format( @@ -987,22 +987,22 @@ class TestDaemon(object): master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'master')) master_opts['user'] = running_tests_user master_opts['known_hosts_file'] = tests_known_hosts_file - master_opts['conf_dir'] = TMP_CONF_DIR + master_opts['config_dir'] = TMP_CONF_DIR minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'minion')) minion_opts['user'] = running_tests_user - minion_opts['conf_dir'] = TMP_CONF_DIR + minion_opts['config_dir'] = TMP_CONF_DIR minion_opts['root_dir'] = master_opts['root_dir'] = os.path.join(TMP, 'rootdir') sub_minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'sub_minion')) sub_minion_opts['user'] = running_tests_user - sub_minion_opts['conf_dir'] = TMP_SUB_MINION_CONF_DIR + sub_minion_opts['config_dir'] = TMP_SUB_MINION_CONF_DIR sub_minion_opts['root_dir'] = os.path.join(TMP, 'rootdir-sub-minion') syndic_master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'syndic_master')) syndic_master_opts['user'] = running_tests_user syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master') - syndic_master_opts['conf_dir'] = TMP_SYNDIC_MASTER_CONF_DIR + syndic_master_opts['config_dir'] = TMP_SYNDIC_MASTER_CONF_DIR # The syndic config file has an include setting to include the master configuration # Let's start with a copy of the syndic master configuration @@ -1012,7 +1012,7 @@ class TestDaemon(object): # Lets remove the include setting syndic_opts.pop('include') syndic_opts['user'] = running_tests_user - syndic_opts['conf_dir'] = TMP_SYNDIC_MINION_CONF_DIR + syndic_opts['config_dir'] = TMP_SYNDIC_MINION_CONF_DIR if transport == 'raet': master_opts['transport'] = 'raet' @@ -1210,7 +1210,7 @@ class TestDaemon(object): ''' Kill the minion and master processes ''' - #self.sub_minion_process.terminate() + self.sub_minion_process.terminate() self.minion_process.terminate() self.master_process.terminate() try: From c9d435734f8bb42fdfe13696afe52dab25855adf Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 27 Jun 2016 09:10:52 -0600 Subject: [PATCH 04/95] Add Pedro's suggestions --- tests/integration/__init__.py | 1 - tests/integration/files/conf/master | 1 + tests/integration/files/conf/sub_minion | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 3b692bf676..b122a89a42 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -724,7 +724,6 @@ class TestDaemon(object): self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR) self.syndic_process.display_name = 'salt-syndic' for process in (self.master_process, self.minion_process, self.sub_minion_process, - #for process in (self.master_process, self.minion_process, self.smaster_process, self.syndic_process): sys.stdout.write( ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format( diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 7ac7cb2bc4..eaba4fcf1c 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -5,6 +5,7 @@ publish_port: 64505 ret_port: 64506 worker_threads: 3 root_dir: /tmp/salttest +pki_dir: pki pidfile: master.pid timeout: 3 open_mode: True diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 623b741599..0ebb244cb5 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -11,6 +11,7 @@ open_mode: True log_file: sub_minion.log log_level_logfile: debug pidfile: sub_minion.pid +ipc_mode: tcp # module extension test.foo: baz From 2636014ccf8a0c0d95c05da94510df15f7f631eb Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 22 Jun 2016 16:56:50 -0700 Subject: [PATCH 05/95] Fix integration tests on windows --- tests/integration/__init__.py | 9 +++++---- tests/integration/files/conf/master | 1 - tests/integration/files/conf/sub_minion | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index b122a89a42..36aff9f127 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -717,13 +717,14 @@ class TestDaemon(object): self.master_process.display_name = 'salt-master' self.minion_process = SaltMinion(self.minion_opts, TMP_CONF_DIR, SCRIPT_DIR) self.minion_process.display_name = 'salt-minion' - self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) - self.sub_minion_process.display_name = 'sub salt-minion' + #self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) + #self.sub_minion_process.display_name = 'sub salt-minion' self.smaster_process = SaltMaster(self.syndic_master_opts, TMP_SYNDIC_MASTER_CONF_DIR, SCRIPT_DIR) self.smaster_process.display_name = 'syndic salt-master' self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR) self.syndic_process.display_name = 'salt-syndic' - for process in (self.master_process, self.minion_process, self.sub_minion_process, + #for process in (self.master_process, self.minion_process, self.sub_minion_process, + for process in (self.master_process, self.minion_process, self.smaster_process, self.syndic_process): sys.stdout.write( ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format( @@ -1209,7 +1210,7 @@ class TestDaemon(object): ''' Kill the minion and master processes ''' - self.sub_minion_process.terminate() + #self.sub_minion_process.terminate() self.minion_process.terminate() self.master_process.terminate() try: diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index eaba4fcf1c..7ac7cb2bc4 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -5,7 +5,6 @@ publish_port: 64505 ret_port: 64506 worker_threads: 3 root_dir: /tmp/salttest -pki_dir: pki pidfile: master.pid timeout: 3 open_mode: True diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 0ebb244cb5..623b741599 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -11,7 +11,6 @@ open_mode: True log_file: sub_minion.log log_level_logfile: debug pidfile: sub_minion.pid -ipc_mode: tcp # module extension test.foo: baz From 528d0d2bb5d66d92608415951b7a53ad54597ce4 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 24 Jun 2016 08:10:00 -0700 Subject: [PATCH 06/95] Fix subminion --- tests/integration/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 36aff9f127..3b692bf676 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -717,14 +717,14 @@ class TestDaemon(object): self.master_process.display_name = 'salt-master' self.minion_process = SaltMinion(self.minion_opts, TMP_CONF_DIR, SCRIPT_DIR) self.minion_process.display_name = 'salt-minion' - #self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) - #self.sub_minion_process.display_name = 'sub salt-minion' + self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) + self.sub_minion_process.display_name = 'sub salt-minion' self.smaster_process = SaltMaster(self.syndic_master_opts, TMP_SYNDIC_MASTER_CONF_DIR, SCRIPT_DIR) self.smaster_process.display_name = 'syndic salt-master' self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR) self.syndic_process.display_name = 'salt-syndic' - #for process in (self.master_process, self.minion_process, self.sub_minion_process, - for process in (self.master_process, self.minion_process, + for process in (self.master_process, self.minion_process, self.sub_minion_process, + #for process in (self.master_process, self.minion_process, self.smaster_process, self.syndic_process): sys.stdout.write( ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format( @@ -1210,7 +1210,7 @@ class TestDaemon(object): ''' Kill the minion and master processes ''' - #self.sub_minion_process.terminate() + self.sub_minion_process.terminate() self.minion_process.terminate() self.master_process.terminate() try: From f4b3c969d11f3919e89ce7f7ee0a38fc28807eee Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 27 Jun 2016 09:10:52 -0600 Subject: [PATCH 07/95] Add Pedro's suggestions --- tests/integration/__init__.py | 1 - tests/integration/files/conf/master | 1 + tests/integration/files/conf/sub_minion | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 3b692bf676..b122a89a42 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -724,7 +724,6 @@ class TestDaemon(object): self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR) self.syndic_process.display_name = 'salt-syndic' for process in (self.master_process, self.minion_process, self.sub_minion_process, - #for process in (self.master_process, self.minion_process, self.smaster_process, self.syndic_process): sys.stdout.write( ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format( diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 7ac7cb2bc4..eaba4fcf1c 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -5,6 +5,7 @@ publish_port: 64505 ret_port: 64506 worker_threads: 3 root_dir: /tmp/salttest +pki_dir: pki pidfile: master.pid timeout: 3 open_mode: True diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 623b741599..0ebb244cb5 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -11,6 +11,7 @@ open_mode: True log_file: sub_minion.log log_level_logfile: debug pidfile: sub_minion.pid +ipc_mode: tcp # module extension test.foo: baz From 074fd4a8beb01f0d2dde777428920f0db39a9d85 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 27 Jun 2016 19:26:46 -0700 Subject: [PATCH 08/95] Fix configs, remove default values --- tests/integration/files/conf/master | 4 +--- tests/integration/files/conf/minion | 9 +++------ tests/integration/files/conf/sub_minion | 8 +++----- tests/integration/files/conf/syndic | 4 ++-- tests/integration/files/conf/syndic_master | 2 -- 5 files changed, 9 insertions(+), 18 deletions(-) diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index eaba4fcf1c..f86a6e59aa 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -4,8 +4,6 @@ interface: 127.0.0.1 publish_port: 64505 ret_port: 64506 worker_threads: 3 -root_dir: /tmp/salttest -pki_dir: pki pidfile: master.pid timeout: 3 open_mode: True @@ -17,7 +15,7 @@ peer: - 'test.*' log_file: master.log log_level_logfile: debug -key_logfile: key +key_logfile: key.log token_file: /tmp/ksfjhdgiuebfgnkefvsikhfjdgvkjahcsidk file_buffer_size: 8192 diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index 630887efaf..6a49043630 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -1,18 +1,15 @@ # basic config master: localhost master_port: 64506 -tcp_pub_port: 64512 -tcp_pull_port: 64513 interface: 127.0.0.1 -root_dir: /tmp/salttest +tcp_pub_port: 64510 +tcp_pull_port: 64511 id: minion -#acceptance_wait_time: = 1 open_mode: True log_file: minion.log log_level_logfile: debug -#loop_interval: 0.05 -config_dir: /tmp/salt-tests-tmpdir pidfile: minion.pid +sock_dir: minion_sock # module extension test.foo: baz diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 0ebb244cb5..56d5e2f3e8 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -1,17 +1,15 @@ # basic config master: localhost master_port: 64506 -tcp_pub_port: 64510 -tcp_pull_port: 64511 -root_dir: /tmp/subsalttest +interface: 127.0.0.1 +tcp_pub_port: 64520 +tcp_pull_port: 64521 sock_dir: sub_minion_sock id: sub_minion -#acceptance_wait_time: 1 open_mode: True log_file: sub_minion.log log_level_logfile: debug pidfile: sub_minion.pid -ipc_mode: tcp # module extension test.foo: baz diff --git a/tests/integration/files/conf/syndic b/tests/integration/files/conf/syndic index 35b88d053d..3c2201a28b 100644 --- a/tests/integration/files/conf/syndic +++ b/tests/integration/files/conf/syndic @@ -5,9 +5,9 @@ include: master # to avoid duplication order_masters: True acceptance_wait_time: 1 -syndic_log_file: osyndic.log +syndic_log_file: syndic.log log_level_logfile: debug -syndic_pidfile: osyndic.pid +syndic_pidfile: syndic.pid syndic_master: localhost syndic_master_port: 54506 id: syndic diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index dbd6ed0da7..542430b262 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -3,9 +3,7 @@ interface: 127.0.0.1 publish_port: 54505 ret_port: 54506 worker_threads: 3 -root_dir: /tmp/saltsyndictest pidfile: syndic_master.pid -pki_dir: pki cachedir: cache timeout: 1 sock_dir: syndic_master_sock From 05629c8efb73b90ed3f1fd3942b15251903acb99 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 27 Jun 2016 20:54:26 -0600 Subject: [PATCH 09/95] Fix master and sub_minion configs --- tests/integration/files/conf/master | 11 ++++++----- tests/integration/files/conf/sub_minion | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index f86a6e59aa..a41f2ad98a 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -1,24 +1,25 @@ id: master -user: ubuntu interface: 127.0.0.1 publish_port: 64505 ret_port: 64506 worker_threads: 3 pidfile: master.pid +user: ubuntu timeout: 3 +sock_dir: master_sock open_mode: True syndic_master: localhost fileserver_list_cache_time: 0 +file_buffer_size: 8192 pillar_opts: True -peer: - '.*': - - 'test.*' log_file: master.log log_level_logfile: debug key_logfile: key.log token_file: /tmp/ksfjhdgiuebfgnkefvsikhfjdgvkjahcsidk -file_buffer_size: 8192 +peer: + '.*': + - 'test.*' ext_pillar: - git: master https://github.com/saltstack/pillar1.git diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 56d5e2f3e8..3190618a36 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -10,6 +10,7 @@ open_mode: True log_file: sub_minion.log log_level_logfile: debug pidfile: sub_minion.pid +ipc_mode: tcp # module extension test.foo: baz From 7030c925cc401291c502ff6679d3f7c267d32ca6 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 28 Jun 2016 08:54:07 -0600 Subject: [PATCH 10/95] Make additional changes to master configs --- tests/integration/files/conf/master | 5 +++-- tests/integration/files/conf/syndic_master | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index a41f2ad98a..269a2a967e 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -4,9 +4,10 @@ publish_port: 64505 ret_port: 64506 worker_threads: 3 pidfile: master.pid -user: ubuntu -timeout: 3 +cachedir: cache sock_dir: master_sock +pki_dir: pki +timeout: 3 open_mode: True syndic_master: localhost fileserver_list_cache_time: 0 diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index 542430b262..b8acf56a54 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -5,8 +5,9 @@ ret_port: 54506 worker_threads: 3 pidfile: syndic_master.pid cachedir: cache -timeout: 1 sock_dir: syndic_master_sock +pki_dir: pki +timeout: 1 open_mode: True order_masters: True fileserver_list_cache_time: 0 From 291f6369a76d6bca82a7d0b62a1de7993bbf2975 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 28 Jun 2016 09:08:45 -0700 Subject: [PATCH 11/95] Fix mem leak caused by pki_dir --- tests/integration/__init__.py | 4 ++++ tests/integration/files/conf/master | 2 -- tests/integration/files/conf/syndic_master | 2 -- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index b122a89a42..44b4411892 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -987,6 +987,8 @@ class TestDaemon(object): master_opts['user'] = running_tests_user master_opts['known_hosts_file'] = tests_known_hosts_file master_opts['config_dir'] = TMP_CONF_DIR + master_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') + #master_opts['pki_dir'] = os.path.join(TMP, 'rootdir','pki') minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'minion')) minion_opts['user'] = running_tests_user @@ -1002,6 +1004,8 @@ class TestDaemon(object): syndic_master_opts['user'] = running_tests_user syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master') syndic_master_opts['config_dir'] = TMP_SYNDIC_MASTER_CONF_DIR + syndic_master_opts['cachedir'] = os.path.join(TMP, 'rootdir-syndic-master', 'cache') + #syndic_master_opts['pki_dir'] = os.path.join(TMP, 'rootdir-syndic-master', 'pki') # The syndic config file has an include setting to include the master configuration # Let's start with a copy of the syndic master configuration diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 269a2a967e..0b5b09e2e4 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -4,9 +4,7 @@ publish_port: 64505 ret_port: 64506 worker_threads: 3 pidfile: master.pid -cachedir: cache sock_dir: master_sock -pki_dir: pki timeout: 3 open_mode: True syndic_master: localhost diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index b8acf56a54..817c5ba0fd 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -4,9 +4,7 @@ publish_port: 54505 ret_port: 54506 worker_threads: 3 pidfile: syndic_master.pid -cachedir: cache sock_dir: syndic_master_sock -pki_dir: pki timeout: 1 open_mode: True order_masters: True From 66c58ddaf12dc581ec4401e5dd1d3a3fd71fb553 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 28 Jun 2016 10:08:58 -0700 Subject: [PATCH 12/95] Fix lint --- tests/integration/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 44b4411892..d3e9db8b7f 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -30,6 +30,11 @@ try: except ImportError: pass +try: + import salt.ext.six.moves.socketserver as socketserver +except ImportError: + import socketserver + STATE_FUNCTION_RUNNING_RE = re.compile( r'''The function (?:"|')(?P.*)(?:"|') is running as PID ''' r'(?P[\d]+) and was started at (?P.*) with jid (?P[\d]+)' From 4cbb679550b53581156b59db321fa2d433f79854 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 7 Jul 2016 12:27:48 -0600 Subject: [PATCH 13/95] Add more descriptive report for interactive mode --- tests/integration/__init__.py | 21 ++++++++++++--------- tests/integration/files/conf/minion | 3 ++- tests/runtests.py | 26 ++++++++++++++++++++------ 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index d3e9db8b7f..6d3e93f293 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -170,7 +170,7 @@ def get_unused_localhost_port(): usock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) usock.bind(('127.0.0.1', 0)) port = usock.getsockname()[1] - if port in (54505, 54506, 64505, 64506, 64510, 64511): + if port in (54505, 54506, 64505, 64506, 64510, 64511, 64520, 64521): # These ports are hardcoded in the test configuration port = get_unused_localhost_port() usock.close() @@ -989,28 +989,31 @@ class TestDaemon(object): known_hosts.write('') master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'master')) - master_opts['user'] = running_tests_user master_opts['known_hosts_file'] = tests_known_hosts_file - master_opts['config_dir'] = TMP_CONF_DIR master_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') - #master_opts['pki_dir'] = os.path.join(TMP, 'rootdir','pki') + master_opts['user'] = running_tests_user + master_opts['config_dir'] = TMP_CONF_DIR + master_opts['root_dir'] = os.path.join(TMP, 'rootdir') + master_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'master') minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'minion')) minion_opts['user'] = running_tests_user minion_opts['config_dir'] = TMP_CONF_DIR - minion_opts['root_dir'] = master_opts['root_dir'] = os.path.join(TMP, 'rootdir') + minion_opts['root_dir'] = os.path.join(TMP, 'rootdir') + minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'minion') sub_minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'sub_minion')) sub_minion_opts['user'] = running_tests_user sub_minion_opts['config_dir'] = TMP_SUB_MINION_CONF_DIR sub_minion_opts['root_dir'] = os.path.join(TMP, 'rootdir-sub-minion') + sub_minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir-sub-minion', 'pki', 'minion') syndic_master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'syndic_master')) - syndic_master_opts['user'] = running_tests_user - syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master') - syndic_master_opts['config_dir'] = TMP_SYNDIC_MASTER_CONF_DIR syndic_master_opts['cachedir'] = os.path.join(TMP, 'rootdir-syndic-master', 'cache') - #syndic_master_opts['pki_dir'] = os.path.join(TMP, 'rootdir-syndic-master', 'pki') + syndic_master_opts['user'] = running_tests_user + syndic_master_opts['config_dir'] = TMP_SYNDIC_MASTER_CONF_DIR + syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master') + syndic_master_opts['pki_dir'] = os.path.join(TMP, 'rootdir-syndic-master', 'pki', 'master') # The syndic config file has an include setting to include the master configuration # Let's start with a copy of the syndic master configuration diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index 6a49043630..d4465f36ce 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -4,12 +4,13 @@ master_port: 64506 interface: 127.0.0.1 tcp_pub_port: 64510 tcp_pull_port: 64511 +sock_dir: minion_sock id: minion open_mode: True log_file: minion.log log_level_logfile: debug pidfile: minion.pid -sock_dir: minion_sock +ipc_mode: tcp # module extension test.foo: baz diff --git a/tests/runtests.py b/tests/runtests.py index edd3477352..1790fc86a6 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -389,15 +389,10 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print_header(' * Salt daemons started') master_conf = TestDaemon.config('master') minion_conf = TestDaemon.config('minion') + sub_minion_conf = TestDaemon.config('sub_minion') syndic_conf = TestDaemon.config('syndic') syndic_master_conf = TestDaemon.config('syndic_master') - print_header(' * Syndic master configuration values', top=False) - print('interface: {0}'.format(syndic_master_conf['interface'])) - print('publish port: {0}'.format(syndic_master_conf['publish_port'])) - print('return port: {0}'.format(syndic_master_conf['ret_port'])) - print('\n') - print_header(' * Master configuration values', top=True) print('interface: {0}'.format(master_conf['interface'])) print('publish port: {0}'.format(master_conf['publish_port'])) @@ -406,6 +401,25 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print_header(' * Minion configuration values', top=True) print('interface: {0}'.format(minion_conf['interface'])) + print('master port: {0}'.format(minion_conf['master_port'])) + print('ipc mode: {0}'.format(minion_conf['ipc_mode'])) + print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) + print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port'])) + print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) + print('\n') + + print_header(' * Sub Minion configuration values', top=True) + print('interface: {0}'.format(sub_minion_conf['interface'])) + print('master port: {0}'.format(sub_minion_conf['master_port'])) + print('ipc mode: {0}'.format(sub_minion_conf['ipc_mode'])) + print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port'])) + print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port'])) + print('\n') + + print_header(' * Syndic master configuration values', top=False) + print('interface: {0}'.format(syndic_master_conf['interface'])) + print('publish port: {0}'.format(syndic_master_conf['publish_port'])) + print('return port: {0}'.format(syndic_master_conf['ret_port'])) print('\n') print_header(' * Syndic configuration values', top=True) From e1a42174d87371b39bf624f969cad8e356e7131a Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 7 Jul 2016 12:45:14 -0600 Subject: [PATCH 14/95] Remove redundant tcp_pull_port entry for minion --- tests/runtests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/runtests.py b/tests/runtests.py index 1790fc86a6..deef73503d 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -403,7 +403,6 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print('interface: {0}'.format(minion_conf['interface'])) print('master port: {0}'.format(minion_conf['master_port'])) print('ipc mode: {0}'.format(minion_conf['ipc_mode'])) - print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port'])) print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) print('\n') From 23193b08e1ed10933cfb875eb321aceb8845e1a7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 7 Jul 2016 15:47:16 -0700 Subject: [PATCH 15/95] Check actual minion ports For some reason this fixed the PYTHONPATH issue --- tests/integration/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 6d3e93f293..504b411f92 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -585,7 +585,8 @@ class SaltMinion(SaltDaemonScriptBase): return script_args def get_check_ports(self): - return set([self.config['id']]) + return set([self.config['tcp_pub_port'], + self.config['tcp_pull_port']]) class SaltMaster(SaltDaemonScriptBase): From 131df72a6fa720b0b5f8d7b46bcff7c34c86f2b5 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 7 Jul 2016 17:27:47 -0600 Subject: [PATCH 16/95] Remove configuration from configs --- tests/integration/files/conf/master | 1 - tests/integration/files/conf/minion | 2 -- tests/integration/files/conf/sub_minion | 1 - tests/integration/files/conf/syndic_master | 1 - tests/runtests.py | 13 ++++++++----- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 0b5b09e2e4..fcae3c2a0c 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -1,5 +1,4 @@ id: master -interface: 127.0.0.1 publish_port: 64505 ret_port: 64506 worker_threads: 3 diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index d4465f36ce..bf7429da23 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -1,7 +1,6 @@ # basic config master: localhost master_port: 64506 -interface: 127.0.0.1 tcp_pub_port: 64510 tcp_pull_port: 64511 sock_dir: minion_sock @@ -10,7 +9,6 @@ open_mode: True log_file: minion.log log_level_logfile: debug pidfile: minion.pid -ipc_mode: tcp # module extension test.foo: baz diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 3190618a36..12e6bf0a32 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -1,7 +1,6 @@ # basic config master: localhost master_port: 64506 -interface: 127.0.0.1 tcp_pub_port: 64520 tcp_pull_port: 64521 sock_dir: sub_minion_sock diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index 817c5ba0fd..146384c9f8 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -1,5 +1,4 @@ id: syndic_master -interface: 127.0.0.1 publish_port: 54505 ret_port: 54506 worker_threads: 3 diff --git a/tests/runtests.py b/tests/runtests.py index deef73503d..d8c493d8a3 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -401,6 +401,7 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print_header(' * Minion configuration values', top=True) print('interface: {0}'.format(minion_conf['interface'])) + print('master: {0}'.format(minion_conf['master'])) print('master port: {0}'.format(minion_conf['master_port'])) print('ipc mode: {0}'.format(minion_conf['ipc_mode'])) print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port'])) @@ -409,23 +410,25 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print_header(' * Sub Minion configuration values', top=True) print('interface: {0}'.format(sub_minion_conf['interface'])) + print('master: {0}'.format(sub_minion_conf['master'])) print('master port: {0}'.format(sub_minion_conf['master_port'])) print('ipc mode: {0}'.format(sub_minion_conf['ipc_mode'])) print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port'])) print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port'])) print('\n') + print_header(' * Syndic configuration values', top=True) + print('interface: {0}'.format(syndic_conf['interface'])) + print('syndic master: {0}'.format(syndic_conf['syndic_master'])) + print('syndic master port: {0}'.format(syndic_conf['syndic_master_port'])) + print('\n') + print_header(' * Syndic master configuration values', top=False) print('interface: {0}'.format(syndic_master_conf['interface'])) print('publish port: {0}'.format(syndic_master_conf['publish_port'])) print('return port: {0}'.format(syndic_master_conf['ret_port'])) print('\n') - print_header(' * Syndic configuration values', top=True) - print('interface: {0}'.format(syndic_conf['interface'])) - print('syndic master port: {0}'.format(syndic_conf['syndic_master'])) - print('\n') - print_header(' Your client configuration is at {0}'.format(TestDaemon.config_location())) print('To access the minion: `salt -c {0} minion test.ping'.format(TestDaemon.config_location())) From 1cb52a7abbb91ab1ae55332dee591cf0c37e56e1 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 8 Jul 2016 10:52:04 -0600 Subject: [PATCH 17/95] Remove backtick --- tests/runtests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/runtests.py b/tests/runtests.py index d8c493d8a3..1af46f602a 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -430,7 +430,7 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print('\n') print_header(' Your client configuration is at {0}'.format(TestDaemon.config_location())) - print('To access the minion: `salt -c {0} minion test.ping'.format(TestDaemon.config_location())) + print('To access the minion: salt -c {0} minion test.ping'.format(TestDaemon.config_location())) while True: time.sleep(1) From 2dbd5d4e92fd24363d91fcca760eb0e34e1deeb3 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 8 Jul 2016 15:59:08 -0600 Subject: [PATCH 18/95] Add cachedir for minions and syndic --- tests/integration/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 504b411f92..8cf2ef6b35 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -998,12 +998,14 @@ class TestDaemon(object): master_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'master') minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'minion')) + minion_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') minion_opts['user'] = running_tests_user minion_opts['config_dir'] = TMP_CONF_DIR minion_opts['root_dir'] = os.path.join(TMP, 'rootdir') minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'minion') sub_minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'sub_minion')) + sub_minion_opts['cachedir'] = os.path.join(TMP, 'rootdir-sub-minion', 'cache') sub_minion_opts['user'] = running_tests_user sub_minion_opts['config_dir'] = TMP_SUB_MINION_CONF_DIR sub_minion_opts['root_dir'] = os.path.join(TMP, 'rootdir-sub-minion') @@ -1023,6 +1025,7 @@ class TestDaemon(object): syndic_opts.update(salt.config._read_conf_file(os.path.join(CONF_DIR, 'syndic'))) # Lets remove the include setting syndic_opts.pop('include') + syndic_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') syndic_opts['user'] = running_tests_user syndic_opts['config_dir'] = TMP_SYNDIC_MINION_CONF_DIR From 82bc69896e76370076e15ae8144ee4b38b297af2 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 8 Jul 2016 16:14:48 -0600 Subject: [PATCH 19/95] Cleans up config parameters --- tests/integration/__init__.py | 22 ++++++++++++---------- tests/integration/files/conf/master | 3 ++- tests/integration/files/conf/minion | 1 + tests/integration/files/conf/sub_minion | 1 + tests/integration/files/conf/syndic | 17 ++++++----------- tests/integration/files/conf/syndic_master | 6 +++++- 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 8cf2ef6b35..b20dfb83e6 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -989,6 +989,7 @@ class TestDaemon(object): with salt.utils.fopen(tests_known_hosts_file, 'w') as known_hosts: known_hosts.write('') + # This master connects to syndic_master via a syndic master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'master')) master_opts['known_hosts_file'] = tests_known_hosts_file master_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') @@ -997,6 +998,15 @@ class TestDaemon(object): master_opts['root_dir'] = os.path.join(TMP, 'rootdir') master_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'master') + # This is the syndic for master + # Let's start with a copy of the syndic master configuration + syndic_opts = copy.deepcopy(master_opts) + # Let's update with the syndic configuration + syndic_opts.update(salt.config._read_conf_file(os.path.join(CONF_DIR, 'syndic'))) + syndic_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') + syndic_opts['config_dir'] = TMP_SYNDIC_MINION_CONF_DIR + + # This minion connects to master minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'minion')) minion_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') minion_opts['user'] = running_tests_user @@ -1004,6 +1014,7 @@ class TestDaemon(object): minion_opts['root_dir'] = os.path.join(TMP, 'rootdir') minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'minion') + # This sub_minion also connects to master sub_minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'sub_minion')) sub_minion_opts['cachedir'] = os.path.join(TMP, 'rootdir-sub-minion', 'cache') sub_minion_opts['user'] = running_tests_user @@ -1011,6 +1022,7 @@ class TestDaemon(object): sub_minion_opts['root_dir'] = os.path.join(TMP, 'rootdir-sub-minion') sub_minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir-sub-minion', 'pki', 'minion') + # This is the master of masters syndic_master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'syndic_master')) syndic_master_opts['cachedir'] = os.path.join(TMP, 'rootdir-syndic-master', 'cache') syndic_master_opts['user'] = running_tests_user @@ -1018,16 +1030,6 @@ class TestDaemon(object): syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master') syndic_master_opts['pki_dir'] = os.path.join(TMP, 'rootdir-syndic-master', 'pki', 'master') - # The syndic config file has an include setting to include the master configuration - # Let's start with a copy of the syndic master configuration - syndic_opts = copy.deepcopy(master_opts) - # Let's update with the syndic configuration - syndic_opts.update(salt.config._read_conf_file(os.path.join(CONF_DIR, 'syndic'))) - # Lets remove the include setting - syndic_opts.pop('include') - syndic_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') - syndic_opts['user'] = running_tests_user - syndic_opts['config_dir'] = TMP_SYNDIC_MINION_CONF_DIR if transport == 'raet': master_opts['transport'] = 'raet' diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index fcae3c2a0c..989d152fd3 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -1,3 +1,5 @@ +# Master Settings +# Connects to syndic_master via syndic id: master publish_port: 64505 ret_port: 64506 @@ -6,7 +8,6 @@ pidfile: master.pid sock_dir: master_sock timeout: 3 open_mode: True -syndic_master: localhost fileserver_list_cache_time: 0 file_buffer_size: 8192 pillar_opts: True diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index bf7429da23..0996078f51 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -1,4 +1,5 @@ # basic config +# Connects to master master: localhost master_port: 64506 tcp_pub_port: 64510 diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 12e6bf0a32..b9d860a48f 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -1,4 +1,5 @@ # basic config +# Connects to master master: localhost master_port: 64506 tcp_pub_port: 64520 diff --git a/tests/integration/files/conf/syndic b/tests/integration/files/conf/syndic index 3c2201a28b..059899c45c 100644 --- a/tests/integration/files/conf/syndic +++ b/tests/integration/files/conf/syndic @@ -1,13 +1,8 @@ -include: master -# syndic basic config -# same config as master ./except the syndic bits -# in the TestCase we add at the top of the configfile the content of ./master -# to avoid duplication -order_masters: True -acceptance_wait_time: 1 -syndic_log_file: syndic.log -log_level_logfile: debug -syndic_pidfile: syndic.pid +# Syndic Settings +id: syndic syndic_master: localhost syndic_master_port: 54506 -id: syndic +syndic_log_file: syndic.log +syndic_pidfile: syndic.pid +tcp_pub_port: 64530 +tcp_pull_port: 64531 diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index 146384c9f8..676fbcf5b9 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -1,3 +1,5 @@ +# Master Settings +# This is the Master of Masters id: syndic_master publish_port: 54505 ret_port: 54506 @@ -6,10 +8,12 @@ pidfile: syndic_master.pid sock_dir: syndic_master_sock timeout: 1 open_mode: True -order_masters: True fileserver_list_cache_time: 0 pillar_opts: True tcp_master_publish_pull: 33305 tcp_master_workers: 33306 log_file: syndic_master.log log_level_logfile: debug + +# Syndic Settings +order_masters: True From 6dc0eb842c21ccaed6f77319068a7bd4276ec4fb Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 8 Jul 2016 16:21:27 -0600 Subject: [PATCH 20/95] Add more info to --interactive report --- tests/runtests.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/runtests.py b/tests/runtests.py index 1af46f602a..b0445f987d 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -393,16 +393,33 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): syndic_conf = TestDaemon.config('syndic') syndic_master_conf = TestDaemon.config('syndic_master') + print_header(' * Syndic master configuration values (MoM)', top=False) + print('interface: {0}'.format(syndic_master_conf['interface'])) + print('publish port: {0}'.format(syndic_master_conf['publish_port'])) + print('return port: {0}'.format(syndic_master_conf['ret_port'])) + print('transport: {0}'.format(syndic_master_conf['transport'])) + print('\n') + + print_header(' * Syndic configuration values', top=True) + print('interface: {0}'.format(syndic_conf['interface'])) + print('syndic master: {0}'.format(syndic_conf['syndic_master'])) + print('syndic master port: {0}'.format(syndic_conf['syndic_master_port'])) + print('tcp pub port: {0}'.format(syndic_conf['tcp_pub_port'])) + print('tcp pull port: {0}'.format(syndic_conf['tcp_pull_port'])) + print('\n') + print_header(' * Master configuration values', top=True) print('interface: {0}'.format(master_conf['interface'])) print('publish port: {0}'.format(master_conf['publish_port'])) print('return port: {0}'.format(master_conf['ret_port'])) + print('transport: {0}'.format(master_conf['transport'])) print('\n') print_header(' * Minion configuration values', top=True) print('interface: {0}'.format(minion_conf['interface'])) print('master: {0}'.format(minion_conf['master'])) print('master port: {0}'.format(minion_conf['master_port'])) + print('transport: {0}'.format(minion_conf['transport'])) print('ipc mode: {0}'.format(minion_conf['ipc_mode'])) print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port'])) print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) @@ -412,23 +429,12 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print('interface: {0}'.format(sub_minion_conf['interface'])) print('master: {0}'.format(sub_minion_conf['master'])) print('master port: {0}'.format(sub_minion_conf['master_port'])) + print('transport: {0}'.format(sub_minion_conf['transport'])) print('ipc mode: {0}'.format(sub_minion_conf['ipc_mode'])) print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port'])) print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port'])) print('\n') - print_header(' * Syndic configuration values', top=True) - print('interface: {0}'.format(syndic_conf['interface'])) - print('syndic master: {0}'.format(syndic_conf['syndic_master'])) - print('syndic master port: {0}'.format(syndic_conf['syndic_master_port'])) - print('\n') - - print_header(' * Syndic master configuration values', top=False) - print('interface: {0}'.format(syndic_master_conf['interface'])) - print('publish port: {0}'.format(syndic_master_conf['publish_port'])) - print('return port: {0}'.format(syndic_master_conf['ret_port'])) - print('\n') - print_header(' Your client configuration is at {0}'.format(TestDaemon.config_location())) print('To access the minion: salt -c {0} minion test.ping'.format(TestDaemon.config_location())) From 1f6f4b29f08c6ffbd0005fb8aee11c1ba1d96837 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 11 Jul 2016 15:15:28 -0700 Subject: [PATCH 21/95] Add tcp_master settings to both masters --- tests/integration/__init__.py | 7 +++++-- tests/integration/files/conf/master | 7 +++++++ tests/integration/files/conf/sub_minion | 1 - tests/integration/files/conf/syndic | 4 ++-- tests/integration/files/conf/syndic_master | 9 +++++++-- tests/runtests.py | 18 ++++++------------ 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index b20dfb83e6..17adac8fbb 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -585,8 +585,11 @@ class SaltMinion(SaltDaemonScriptBase): return script_args def get_check_ports(self): - return set([self.config['tcp_pub_port'], - self.config['tcp_pull_port']]) + if salt.utils.is_windows(): + return set([self.config['tcp_pub_port'], + self.config['tcp_pull_port']]) + else: + return set([self.config['id']]) class SaltMaster(SaltDaemonScriptBase): diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 989d152fd3..4c908068b3 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -16,6 +16,13 @@ log_level_logfile: debug key_logfile: key.log token_file: /tmp/ksfjhdgiuebfgnkefvsikhfjdgvkjahcsidk +# These settings needed for tests on Windows which defaults +# to ipc_mode: tcp +tcp_master_pub_port: 64512 +tcp_master_pull_port: 64513 +tcp_master_publish_pull: 64514 +tcp_master_workers: 64515 + peer: '.*': - 'test.*' diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index b9d860a48f..b1f105fa1b 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -10,7 +10,6 @@ open_mode: True log_file: sub_minion.log log_level_logfile: debug pidfile: sub_minion.pid -ipc_mode: tcp # module extension test.foo: baz diff --git a/tests/integration/files/conf/syndic b/tests/integration/files/conf/syndic index 059899c45c..788828d3c1 100644 --- a/tests/integration/files/conf/syndic +++ b/tests/integration/files/conf/syndic @@ -4,5 +4,5 @@ syndic_master: localhost syndic_master_port: 54506 syndic_log_file: syndic.log syndic_pidfile: syndic.pid -tcp_pub_port: 64530 -tcp_pull_port: 64531 +tcp_pub_port: 64510 +tcp_pull_port: 64511 diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index 676fbcf5b9..eae676dbfa 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -10,10 +10,15 @@ timeout: 1 open_mode: True fileserver_list_cache_time: 0 pillar_opts: True -tcp_master_publish_pull: 33305 -tcp_master_workers: 33306 log_file: syndic_master.log log_level_logfile: debug +# These settings needed for tests on Windows which defaults +# to ipc_mode: tcp +tcp_master_pub_port: 54512 +tcp_master_pull_port: 54513 +tcp_master_publish_pull: 54514 +tcp_master_workers: 54515 + # Syndic Settings order_masters: True diff --git a/tests/runtests.py b/tests/runtests.py index b0445f987d..0fe7966ace 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -397,42 +397,36 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print('interface: {0}'.format(syndic_master_conf['interface'])) print('publish port: {0}'.format(syndic_master_conf['publish_port'])) print('return port: {0}'.format(syndic_master_conf['ret_port'])) - print('transport: {0}'.format(syndic_master_conf['transport'])) print('\n') print_header(' * Syndic configuration values', top=True) print('interface: {0}'.format(syndic_conf['interface'])) print('syndic master: {0}'.format(syndic_conf['syndic_master'])) print('syndic master port: {0}'.format(syndic_conf['syndic_master_port'])) - print('tcp pub port: {0}'.format(syndic_conf['tcp_pub_port'])) - print('tcp pull port: {0}'.format(syndic_conf['tcp_pull_port'])) print('\n') print_header(' * Master configuration values', top=True) print('interface: {0}'.format(master_conf['interface'])) print('publish port: {0}'.format(master_conf['publish_port'])) print('return port: {0}'.format(master_conf['ret_port'])) - print('transport: {0}'.format(master_conf['transport'])) print('\n') print_header(' * Minion configuration values', top=True) print('interface: {0}'.format(minion_conf['interface'])) print('master: {0}'.format(minion_conf['master'])) print('master port: {0}'.format(minion_conf['master_port'])) - print('transport: {0}'.format(minion_conf['transport'])) - print('ipc mode: {0}'.format(minion_conf['ipc_mode'])) - print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port'])) - print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) + if minion_conf['ipc_mode'] == 'tcp': + print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port'])) + print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) print('\n') print_header(' * Sub Minion configuration values', top=True) print('interface: {0}'.format(sub_minion_conf['interface'])) print('master: {0}'.format(sub_minion_conf['master'])) print('master port: {0}'.format(sub_minion_conf['master_port'])) - print('transport: {0}'.format(sub_minion_conf['transport'])) - print('ipc mode: {0}'.format(sub_minion_conf['ipc_mode'])) - print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port'])) - print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port'])) + if sub_minion_conf['ipc_mode'] == 'tcp': + print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port'])) + print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port'])) print('\n') print_header(' Your client configuration is at {0}'.format(TestDaemon.config_location())) From 9b88d3a1e2787007e997c52be21854211f2f84d4 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 13 Jul 2016 10:02:58 -0600 Subject: [PATCH 22/95] Add interface to all configs This got removed during the attempt to make the tests run on Windows. I added them everywhere even though that's not strictly necessary. Without these set on the master and sub_master, Windows will throw erorrs because the networking stack will decide that anything that's bound to 0.0.0.0 represents an extestential threat to any socket that attempts to *connect* to one of those ports. (For...reasons.) --- tests/integration/files/conf/master | 1 + tests/integration/files/conf/minion | 1 + tests/integration/files/conf/sub_minion | 1 + tests/integration/files/conf/syndic | 1 + tests/integration/files/conf/syndic_master | 1 + 5 files changed, 5 insertions(+) diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 4c908068b3..76e2d80f14 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -1,6 +1,7 @@ # Master Settings # Connects to syndic_master via syndic id: master +interface: 127.0.0.1 publish_port: 64505 ret_port: 64506 worker_threads: 3 diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index 0996078f51..53bdb4ac07 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -2,6 +2,7 @@ # Connects to master master: localhost master_port: 64506 +interface: 127.0.0.1 tcp_pub_port: 64510 tcp_pull_port: 64511 sock_dir: minion_sock diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index b1f105fa1b..008ade341d 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -1,6 +1,7 @@ # basic config # Connects to master master: localhost +interface: 127.0.0.1 master_port: 64506 tcp_pub_port: 64520 tcp_pull_port: 64521 diff --git a/tests/integration/files/conf/syndic b/tests/integration/files/conf/syndic index 788828d3c1..5746d493c9 100644 --- a/tests/integration/files/conf/syndic +++ b/tests/integration/files/conf/syndic @@ -1,5 +1,6 @@ # Syndic Settings id: syndic +interface: 127.0.0.1 syndic_master: localhost syndic_master_port: 54506 syndic_log_file: syndic.log diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index eae676dbfa..203820b97e 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -1,6 +1,7 @@ # Master Settings # This is the Master of Masters id: syndic_master +interface: 127.0.0.1 publish_port: 54505 ret_port: 54506 worker_threads: 3 From 0452d0a0ec4051d7b51d32a3affbe17d43e3c809 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 18 Jul 2016 06:45:57 -0600 Subject: [PATCH 23/95] Lint --- tests/integration/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 17adac8fbb..851238b682 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1033,7 +1033,6 @@ class TestDaemon(object): syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master') syndic_master_opts['pki_dir'] = os.path.join(TMP, 'rootdir-syndic-master', 'pki', 'master') - if transport == 'raet': master_opts['transport'] = 'raet' master_opts['raet_port'] = 64506 From b1014cdefbd9748ba90df644a494429c4ce7f3f6 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 18 Jul 2016 13:01:44 -0700 Subject: [PATCH 24/95] Add code to kill process in Windows --- tests/integration/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 851238b682..4c4547c94c 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -510,7 +510,10 @@ class SaltDaemonScriptBase(SaltScriptBase, ShellTestCase): # Lets log and kill any child processes which salt left behind for child in children[:]: try: - child.send_signal(signal.SIGKILL) + if sys.platform.startswith('win'): + child.kill() + else: + child.send_signal(signal.SIGKILL) log.info('Salt left behind the following child process: %s', child.as_dict()) try: child.wait(timeout=5) From 909eee9a5abd06806f3a2fefe0f2b23be5181d4b Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Tue, 19 Jul 2016 10:07:49 -0600 Subject: [PATCH 25/95] Remove extra __salt__ work --- salt/utils/winapi.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/salt/utils/winapi.py b/salt/utils/winapi.py index 26d89d8d49..f5ef633111 100644 --- a/salt/utils/winapi.py +++ b/salt/utils/winapi.py @@ -15,8 +15,6 @@ except ImportError: log = logging.getLogger(__name__) -__salt__ = None - def __virtual__(): ''' @@ -25,9 +23,6 @@ def __virtual__(): if not HAS_LIBS: return False else: - global __salt__ - if not __salt__: - __salt__ = minion_mods(__opts__) return True From c5245f6b246723658ffef384558000171e286f6c Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 19 Jul 2016 10:04:24 -0700 Subject: [PATCH 26/95] Fix stack trace on Jenkins --- tests/integration/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 4c4547c94c..78e6430863 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -30,11 +30,6 @@ try: except ImportError: pass -try: - import salt.ext.six.moves.socketserver as socketserver -except ImportError: - import socketserver - STATE_FUNCTION_RUNNING_RE = re.compile( r'''The function (?:"|')(?P.*)(?:"|') is running as PID ''' r'(?P[\d]+) and was started at (?P.*) with jid (?P[\d]+)' @@ -81,7 +76,11 @@ except ImportError: import yaml import msgpack import salt.ext.six as six -import salt.ext.six.moves.socketserver as socketserver # pylint: disable=no-name-in-module + +try: + import salt.ext.six.moves.socketserver as socketserver # pylint: disable=name-in-module +except ImportError: + import socketserver if salt.utils.is_windows(): import win32api From f960ded7806bc3b8cd556bcb42d06c2c141f6c2c Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 19 Jul 2016 13:48:03 -0700 Subject: [PATCH 27/95] Gate SIGINT in shutdown routines for Windows SIGINT not available in Windows --- tests/integration/__init__.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 78e6430863..64c9d654ff 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -440,23 +440,24 @@ class SaltDaemonScriptBase(SaltScriptBase, ShellTestCase): pass # Let's begin the shutdown routines - if terminal.poll() is None: - try: - log.info('Sending SIGINT to %s %s DAEMON', self.display_name, self.__class__.__name__) - terminal.send_signal(signal.SIGINT) - except OSError as exc: - if exc.errno not in (errno.ESRCH, errno.EACCES): - raise - timeout = 15 - log.info('Waiting %s seconds for %s %s DAEMON to respond to SIGINT', - timeout, - self.display_name, - self.__class__.__name__) - while timeout > 0: - if terminal.poll() is not None: - break - timeout -= 0.0125 - time.sleep(0.0125) + if not sys.platform.startswith('win'): + if terminal.poll() is None: + try: + log.info('Sending SIGINT to %s %s DAEMON', self.display_name, self.__class__.__name__) + terminal.send_signal(signal.SIGINT) + except OSError as exc: + if exc.errno not in (errno.ESRCH, errno.EACCES): + raise + timeout = 15 + log.info('Waiting %s seconds for %s %s DAEMON to respond to SIGINT', + timeout, + self.display_name, + self.__class__.__name__) + while timeout > 0: + if terminal.poll() is not None: + break + timeout -= 0.0125 + time.sleep(0.0125) if terminal.poll() is None: try: log.info('Sending SIGTERM to %s %s DAEMON', self.display_name, self.__class__.__name__) From 5bcb27e477196e8ff4642abb40979da01b60175b Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 22 Jun 2016 16:56:50 -0700 Subject: [PATCH 28/95] Fix integration tests on windows --- tests/integration/__init__.py | 19 ++++++++++--------- tests/integration/files/conf/master | 7 ++----- tests/integration/files/conf/minion | 9 +++++---- tests/integration/files/conf/sub_minion | 12 +++++------- tests/integration/files/conf/syndic_master | 6 +++--- tests/integration/states/pkg.py | 1 + 6 files changed, 26 insertions(+), 28 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 92b1a4aba0..6c0d64f957 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -717,13 +717,14 @@ class TestDaemon(object): self.master_process.display_name = 'salt-master' self.minion_process = SaltMinion(self.minion_opts, TMP_CONF_DIR, SCRIPT_DIR) self.minion_process.display_name = 'salt-minion' - self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) - self.sub_minion_process.display_name = 'sub salt-minion' + #self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) + #self.sub_minion_process.display_name = 'sub salt-minion' self.smaster_process = SaltMaster(self.syndic_master_opts, TMP_SYNDIC_MASTER_CONF_DIR, SCRIPT_DIR) self.smaster_process.display_name = 'syndic salt-master' self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR) self.syndic_process.display_name = 'salt-syndic' - for process in (self.master_process, self.minion_process, self.sub_minion_process, + #for process in (self.master_process, self.minion_process, self.sub_minion_process, + for process in (self.master_process, self.minion_process, self.smaster_process, self.syndic_process): sys.stdout.write( ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format( @@ -978,19 +979,19 @@ class TestDaemon(object): running_tests_user = win32api.GetUserName() else: running_tests_user = pwd.getpwuid(os.getuid()).pw_name - master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'master')) - master_opts['user'] = running_tests_user + tests_known_hosts_file = os.path.join(TMP_CONF_DIR, 'salt_ssh_known_hosts') with salt.utils.fopen(tests_known_hosts_file, 'w') as known_hosts: known_hosts.write('') + + master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'master')) + master_opts['user'] = running_tests_user master_opts['known_hosts_file'] = tests_known_hosts_file master_opts['conf_dir'] = TMP_CONF_DIR - minion_config_path = os.path.join(CONF_DIR, 'minion') - minion_opts = salt.config._read_conf_file(minion_config_path) + minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'minion')) minion_opts['user'] = running_tests_user minion_opts['conf_dir'] = TMP_CONF_DIR - minion_opts['root_dir'] = master_opts['root_dir'] = os.path.join(TMP, 'rootdir') sub_minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'sub_minion')) @@ -1209,7 +1210,7 @@ class TestDaemon(object): ''' Kill the minion and master processes ''' - self.sub_minion_process.terminate() + #self.sub_minion_process.terminate() self.minion_process.terminate() self.master_process.terminate() try: diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index dd5f30e5bc..7ac7cb2bc4 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -5,11 +5,8 @@ publish_port: 64505 ret_port: 64506 worker_threads: 3 root_dir: /tmp/salttest -pidfile: masterpid -pki_dir: pki -cachedir: cache +pidfile: master.pid timeout: 3 -sock_dir: .salt-unix open_mode: True syndic_master: localhost fileserver_list_cache_time: 0 @@ -17,7 +14,7 @@ pillar_opts: True peer: '.*': - 'test.*' -log_file: master +log_file: master.log log_level_logfile: debug key_logfile: key token_file: /tmp/ksfjhdgiuebfgnkefvsikhfjdgvkjahcsidk diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index 0ccfe90ec0..630887efaf 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -1,17 +1,18 @@ # basic config master: localhost master_port: 64506 +tcp_pub_port: 64512 +tcp_pull_port: 64513 +interface: 127.0.0.1 root_dir: /tmp/salttest -pki_dir: pki id: minion -cachedir: cachedir -sock_dir: minion_sock #acceptance_wait_time: = 1 open_mode: True -log_file: minion +log_file: minion.log log_level_logfile: debug #loop_interval: 0.05 config_dir: /tmp/salt-tests-tmpdir +pidfile: minion.pid # module extension test.foo: baz diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index c21e4d9785..623b741599 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -1,15 +1,16 @@ # basic config master: localhost master_port: 64506 +tcp_pub_port: 64510 +tcp_pull_port: 64511 root_dir: /tmp/subsalttest -pki_dir: pki -id: sub_minion -cachedir: cachedir sock_dir: sub_minion_sock +id: sub_minion #acceptance_wait_time: 1 open_mode: True -log_file: sub_minion +log_file: sub_minion.log log_level_logfile: debug +pidfile: sub_minion.pid # module extension test.foo: baz @@ -33,6 +34,3 @@ grains: - jamie - zoe -ipc_mode: tcp -tcp_pub_port: 64510 -tcp_pull_port: 64511 diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index a88cb36b43..dbd6ed0da7 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -4,16 +4,16 @@ publish_port: 54505 ret_port: 54506 worker_threads: 3 root_dir: /tmp/saltsyndictest -pidfile: syndicmasterpid +pidfile: syndic_master.pid pki_dir: pki cachedir: cache timeout: 1 -sock_dir: .salt-unix-syndic +sock_dir: syndic_master_sock open_mode: True order_masters: True fileserver_list_cache_time: 0 pillar_opts: True tcp_master_publish_pull: 33305 tcp_master_workers: 33306 -log_file: syndic_master +log_file: syndic_master.log log_level_logfile: debug diff --git a/tests/integration/states/pkg.py b/tests/integration/states/pkg.py index 846cee0b16..39a6279652 100644 --- a/tests/integration/states/pkg.py +++ b/tests/integration/states/pkg.py @@ -37,6 +37,7 @@ _PKG_TARGETS = { 'FreeBSD': ['aalib', 'pth'], 'SUSE': ['aalib', 'python-pssh'], 'MacOS': ['libpng', 'jpeg'], + 'Windows': ['firefox', '7zip'], } _PKG_TARGETS_32 = { From 785f5470fcaf803d2f3ca9fe20a7da3f9d5f0889 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 24 Jun 2016 08:10:00 -0700 Subject: [PATCH 29/95] Fix subminion --- tests/integration/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 6c0d64f957..3b692bf676 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -717,14 +717,14 @@ class TestDaemon(object): self.master_process.display_name = 'salt-master' self.minion_process = SaltMinion(self.minion_opts, TMP_CONF_DIR, SCRIPT_DIR) self.minion_process.display_name = 'salt-minion' - #self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) - #self.sub_minion_process.display_name = 'sub salt-minion' + self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) + self.sub_minion_process.display_name = 'sub salt-minion' self.smaster_process = SaltMaster(self.syndic_master_opts, TMP_SYNDIC_MASTER_CONF_DIR, SCRIPT_DIR) self.smaster_process.display_name = 'syndic salt-master' self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR) self.syndic_process.display_name = 'salt-syndic' - #for process in (self.master_process, self.minion_process, self.sub_minion_process, - for process in (self.master_process, self.minion_process, + for process in (self.master_process, self.minion_process, self.sub_minion_process, + #for process in (self.master_process, self.minion_process, self.smaster_process, self.syndic_process): sys.stdout.write( ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format( @@ -987,22 +987,22 @@ class TestDaemon(object): master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'master')) master_opts['user'] = running_tests_user master_opts['known_hosts_file'] = tests_known_hosts_file - master_opts['conf_dir'] = TMP_CONF_DIR + master_opts['config_dir'] = TMP_CONF_DIR minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'minion')) minion_opts['user'] = running_tests_user - minion_opts['conf_dir'] = TMP_CONF_DIR + minion_opts['config_dir'] = TMP_CONF_DIR minion_opts['root_dir'] = master_opts['root_dir'] = os.path.join(TMP, 'rootdir') sub_minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'sub_minion')) sub_minion_opts['user'] = running_tests_user - sub_minion_opts['conf_dir'] = TMP_SUB_MINION_CONF_DIR + sub_minion_opts['config_dir'] = TMP_SUB_MINION_CONF_DIR sub_minion_opts['root_dir'] = os.path.join(TMP, 'rootdir-sub-minion') syndic_master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'syndic_master')) syndic_master_opts['user'] = running_tests_user syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master') - syndic_master_opts['conf_dir'] = TMP_SYNDIC_MASTER_CONF_DIR + syndic_master_opts['config_dir'] = TMP_SYNDIC_MASTER_CONF_DIR # The syndic config file has an include setting to include the master configuration # Let's start with a copy of the syndic master configuration @@ -1012,7 +1012,7 @@ class TestDaemon(object): # Lets remove the include setting syndic_opts.pop('include') syndic_opts['user'] = running_tests_user - syndic_opts['conf_dir'] = TMP_SYNDIC_MINION_CONF_DIR + syndic_opts['config_dir'] = TMP_SYNDIC_MINION_CONF_DIR if transport == 'raet': master_opts['transport'] = 'raet' @@ -1210,7 +1210,7 @@ class TestDaemon(object): ''' Kill the minion and master processes ''' - #self.sub_minion_process.terminate() + self.sub_minion_process.terminate() self.minion_process.terminate() self.master_process.terminate() try: From f7fb4985c4907f1eb4bc2dad3da38f4a41bdb96a Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 27 Jun 2016 09:10:52 -0600 Subject: [PATCH 30/95] Add Pedro's suggestions --- tests/integration/__init__.py | 1 - tests/integration/files/conf/master | 1 + tests/integration/files/conf/sub_minion | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 3b692bf676..b122a89a42 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -724,7 +724,6 @@ class TestDaemon(object): self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR) self.syndic_process.display_name = 'salt-syndic' for process in (self.master_process, self.minion_process, self.sub_minion_process, - #for process in (self.master_process, self.minion_process, self.smaster_process, self.syndic_process): sys.stdout.write( ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format( diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 7ac7cb2bc4..eaba4fcf1c 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -5,6 +5,7 @@ publish_port: 64505 ret_port: 64506 worker_threads: 3 root_dir: /tmp/salttest +pki_dir: pki pidfile: master.pid timeout: 3 open_mode: True diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 623b741599..0ebb244cb5 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -11,6 +11,7 @@ open_mode: True log_file: sub_minion.log log_level_logfile: debug pidfile: sub_minion.pid +ipc_mode: tcp # module extension test.foo: baz From 1805630fdccc5217dd4bf79eefc9ab08e94eb6fa Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 22 Jun 2016 16:56:50 -0700 Subject: [PATCH 31/95] Fix integration tests on windows --- tests/integration/__init__.py | 9 +++++---- tests/integration/files/conf/master | 1 - tests/integration/files/conf/sub_minion | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index b122a89a42..36aff9f127 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -717,13 +717,14 @@ class TestDaemon(object): self.master_process.display_name = 'salt-master' self.minion_process = SaltMinion(self.minion_opts, TMP_CONF_DIR, SCRIPT_DIR) self.minion_process.display_name = 'salt-minion' - self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) - self.sub_minion_process.display_name = 'sub salt-minion' + #self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) + #self.sub_minion_process.display_name = 'sub salt-minion' self.smaster_process = SaltMaster(self.syndic_master_opts, TMP_SYNDIC_MASTER_CONF_DIR, SCRIPT_DIR) self.smaster_process.display_name = 'syndic salt-master' self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR) self.syndic_process.display_name = 'salt-syndic' - for process in (self.master_process, self.minion_process, self.sub_minion_process, + #for process in (self.master_process, self.minion_process, self.sub_minion_process, + for process in (self.master_process, self.minion_process, self.smaster_process, self.syndic_process): sys.stdout.write( ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format( @@ -1209,7 +1210,7 @@ class TestDaemon(object): ''' Kill the minion and master processes ''' - self.sub_minion_process.terminate() + #self.sub_minion_process.terminate() self.minion_process.terminate() self.master_process.terminate() try: diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index eaba4fcf1c..7ac7cb2bc4 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -5,7 +5,6 @@ publish_port: 64505 ret_port: 64506 worker_threads: 3 root_dir: /tmp/salttest -pki_dir: pki pidfile: master.pid timeout: 3 open_mode: True diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 0ebb244cb5..623b741599 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -11,7 +11,6 @@ open_mode: True log_file: sub_minion.log log_level_logfile: debug pidfile: sub_minion.pid -ipc_mode: tcp # module extension test.foo: baz From 9bc438e3029ff4304853f7504ea0a3943c16ff2a Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 24 Jun 2016 08:10:00 -0700 Subject: [PATCH 32/95] Fix subminion --- tests/integration/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 36aff9f127..3b692bf676 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -717,14 +717,14 @@ class TestDaemon(object): self.master_process.display_name = 'salt-master' self.minion_process = SaltMinion(self.minion_opts, TMP_CONF_DIR, SCRIPT_DIR) self.minion_process.display_name = 'salt-minion' - #self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) - #self.sub_minion_process.display_name = 'sub salt-minion' + self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR) + self.sub_minion_process.display_name = 'sub salt-minion' self.smaster_process = SaltMaster(self.syndic_master_opts, TMP_SYNDIC_MASTER_CONF_DIR, SCRIPT_DIR) self.smaster_process.display_name = 'syndic salt-master' self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR) self.syndic_process.display_name = 'salt-syndic' - #for process in (self.master_process, self.minion_process, self.sub_minion_process, - for process in (self.master_process, self.minion_process, + for process in (self.master_process, self.minion_process, self.sub_minion_process, + #for process in (self.master_process, self.minion_process, self.smaster_process, self.syndic_process): sys.stdout.write( ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format( @@ -1210,7 +1210,7 @@ class TestDaemon(object): ''' Kill the minion and master processes ''' - #self.sub_minion_process.terminate() + self.sub_minion_process.terminate() self.minion_process.terminate() self.master_process.terminate() try: From 9b14dc0a6e2e98a2932fbaf87dfe60ca449a51d0 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 27 Jun 2016 09:10:52 -0600 Subject: [PATCH 33/95] Add Pedro's suggestions --- tests/integration/__init__.py | 1 - tests/integration/files/conf/master | 1 + tests/integration/files/conf/sub_minion | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 3b692bf676..b122a89a42 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -724,7 +724,6 @@ class TestDaemon(object): self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR) self.syndic_process.display_name = 'salt-syndic' for process in (self.master_process, self.minion_process, self.sub_minion_process, - #for process in (self.master_process, self.minion_process, self.smaster_process, self.syndic_process): sys.stdout.write( ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format( diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 7ac7cb2bc4..eaba4fcf1c 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -5,6 +5,7 @@ publish_port: 64505 ret_port: 64506 worker_threads: 3 root_dir: /tmp/salttest +pki_dir: pki pidfile: master.pid timeout: 3 open_mode: True diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 623b741599..0ebb244cb5 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -11,6 +11,7 @@ open_mode: True log_file: sub_minion.log log_level_logfile: debug pidfile: sub_minion.pid +ipc_mode: tcp # module extension test.foo: baz From f55ba87beede42b89beb016336fea50928f5a0b5 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 27 Jun 2016 19:26:46 -0700 Subject: [PATCH 34/95] Fix configs, remove default values --- tests/integration/files/conf/master | 4 +--- tests/integration/files/conf/minion | 9 +++------ tests/integration/files/conf/sub_minion | 8 +++----- tests/integration/files/conf/syndic | 4 ++-- tests/integration/files/conf/syndic_master | 2 -- 5 files changed, 9 insertions(+), 18 deletions(-) diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index eaba4fcf1c..f86a6e59aa 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -4,8 +4,6 @@ interface: 127.0.0.1 publish_port: 64505 ret_port: 64506 worker_threads: 3 -root_dir: /tmp/salttest -pki_dir: pki pidfile: master.pid timeout: 3 open_mode: True @@ -17,7 +15,7 @@ peer: - 'test.*' log_file: master.log log_level_logfile: debug -key_logfile: key +key_logfile: key.log token_file: /tmp/ksfjhdgiuebfgnkefvsikhfjdgvkjahcsidk file_buffer_size: 8192 diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index 630887efaf..6a49043630 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -1,18 +1,15 @@ # basic config master: localhost master_port: 64506 -tcp_pub_port: 64512 -tcp_pull_port: 64513 interface: 127.0.0.1 -root_dir: /tmp/salttest +tcp_pub_port: 64510 +tcp_pull_port: 64511 id: minion -#acceptance_wait_time: = 1 open_mode: True log_file: minion.log log_level_logfile: debug -#loop_interval: 0.05 -config_dir: /tmp/salt-tests-tmpdir pidfile: minion.pid +sock_dir: minion_sock # module extension test.foo: baz diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 0ebb244cb5..56d5e2f3e8 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -1,17 +1,15 @@ # basic config master: localhost master_port: 64506 -tcp_pub_port: 64510 -tcp_pull_port: 64511 -root_dir: /tmp/subsalttest +interface: 127.0.0.1 +tcp_pub_port: 64520 +tcp_pull_port: 64521 sock_dir: sub_minion_sock id: sub_minion -#acceptance_wait_time: 1 open_mode: True log_file: sub_minion.log log_level_logfile: debug pidfile: sub_minion.pid -ipc_mode: tcp # module extension test.foo: baz diff --git a/tests/integration/files/conf/syndic b/tests/integration/files/conf/syndic index 35b88d053d..3c2201a28b 100644 --- a/tests/integration/files/conf/syndic +++ b/tests/integration/files/conf/syndic @@ -5,9 +5,9 @@ include: master # to avoid duplication order_masters: True acceptance_wait_time: 1 -syndic_log_file: osyndic.log +syndic_log_file: syndic.log log_level_logfile: debug -syndic_pidfile: osyndic.pid +syndic_pidfile: syndic.pid syndic_master: localhost syndic_master_port: 54506 id: syndic diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index dbd6ed0da7..542430b262 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -3,9 +3,7 @@ interface: 127.0.0.1 publish_port: 54505 ret_port: 54506 worker_threads: 3 -root_dir: /tmp/saltsyndictest pidfile: syndic_master.pid -pki_dir: pki cachedir: cache timeout: 1 sock_dir: syndic_master_sock From a33c5c15da48ec935d54a98b20f2e505cf6d5165 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 27 Jun 2016 20:54:26 -0600 Subject: [PATCH 35/95] Fix master and sub_minion configs --- tests/integration/files/conf/master | 11 ++++++----- tests/integration/files/conf/sub_minion | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index f86a6e59aa..a41f2ad98a 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -1,24 +1,25 @@ id: master -user: ubuntu interface: 127.0.0.1 publish_port: 64505 ret_port: 64506 worker_threads: 3 pidfile: master.pid +user: ubuntu timeout: 3 +sock_dir: master_sock open_mode: True syndic_master: localhost fileserver_list_cache_time: 0 +file_buffer_size: 8192 pillar_opts: True -peer: - '.*': - - 'test.*' log_file: master.log log_level_logfile: debug key_logfile: key.log token_file: /tmp/ksfjhdgiuebfgnkefvsikhfjdgvkjahcsidk -file_buffer_size: 8192 +peer: + '.*': + - 'test.*' ext_pillar: - git: master https://github.com/saltstack/pillar1.git diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 56d5e2f3e8..3190618a36 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -10,6 +10,7 @@ open_mode: True log_file: sub_minion.log log_level_logfile: debug pidfile: sub_minion.pid +ipc_mode: tcp # module extension test.foo: baz From 7f3f15378e97009edf3979959e29c8a42c58b291 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 28 Jun 2016 08:54:07 -0600 Subject: [PATCH 36/95] Make additional changes to master configs --- tests/integration/files/conf/master | 5 +++-- tests/integration/files/conf/syndic_master | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index a41f2ad98a..269a2a967e 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -4,9 +4,10 @@ publish_port: 64505 ret_port: 64506 worker_threads: 3 pidfile: master.pid -user: ubuntu -timeout: 3 +cachedir: cache sock_dir: master_sock +pki_dir: pki +timeout: 3 open_mode: True syndic_master: localhost fileserver_list_cache_time: 0 diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index 542430b262..b8acf56a54 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -5,8 +5,9 @@ ret_port: 54506 worker_threads: 3 pidfile: syndic_master.pid cachedir: cache -timeout: 1 sock_dir: syndic_master_sock +pki_dir: pki +timeout: 1 open_mode: True order_masters: True fileserver_list_cache_time: 0 From 54647eb7d88d044db23667c99a3d73c6a49e565e Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 28 Jun 2016 09:08:45 -0700 Subject: [PATCH 37/95] Fix mem leak caused by pki_dir --- tests/integration/__init__.py | 4 ++++ tests/integration/files/conf/master | 2 -- tests/integration/files/conf/syndic_master | 2 -- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index b122a89a42..44b4411892 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -987,6 +987,8 @@ class TestDaemon(object): master_opts['user'] = running_tests_user master_opts['known_hosts_file'] = tests_known_hosts_file master_opts['config_dir'] = TMP_CONF_DIR + master_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') + #master_opts['pki_dir'] = os.path.join(TMP, 'rootdir','pki') minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'minion')) minion_opts['user'] = running_tests_user @@ -1002,6 +1004,8 @@ class TestDaemon(object): syndic_master_opts['user'] = running_tests_user syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master') syndic_master_opts['config_dir'] = TMP_SYNDIC_MASTER_CONF_DIR + syndic_master_opts['cachedir'] = os.path.join(TMP, 'rootdir-syndic-master', 'cache') + #syndic_master_opts['pki_dir'] = os.path.join(TMP, 'rootdir-syndic-master', 'pki') # The syndic config file has an include setting to include the master configuration # Let's start with a copy of the syndic master configuration diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 269a2a967e..0b5b09e2e4 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -4,9 +4,7 @@ publish_port: 64505 ret_port: 64506 worker_threads: 3 pidfile: master.pid -cachedir: cache sock_dir: master_sock -pki_dir: pki timeout: 3 open_mode: True syndic_master: localhost diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index b8acf56a54..817c5ba0fd 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -4,9 +4,7 @@ publish_port: 54505 ret_port: 54506 worker_threads: 3 pidfile: syndic_master.pid -cachedir: cache sock_dir: syndic_master_sock -pki_dir: pki timeout: 1 open_mode: True order_masters: True From cfc82f40048fa508567dfa09e2e7dd4b62c9b778 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 28 Jun 2016 10:08:58 -0700 Subject: [PATCH 38/95] Fix lint --- tests/integration/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 44b4411892..d3e9db8b7f 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -30,6 +30,11 @@ try: except ImportError: pass +try: + import salt.ext.six.moves.socketserver as socketserver +except ImportError: + import socketserver + STATE_FUNCTION_RUNNING_RE = re.compile( r'''The function (?:"|')(?P.*)(?:"|') is running as PID ''' r'(?P[\d]+) and was started at (?P.*) with jid (?P[\d]+)' From 79b5be69893bc7aa3ad7d6067a45b407395ebf27 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 7 Jul 2016 12:27:48 -0600 Subject: [PATCH 39/95] Add more descriptive report for interactive mode --- tests/integration/__init__.py | 21 ++++++++++++--------- tests/integration/files/conf/minion | 3 ++- tests/runtests.py | 26 ++++++++++++++++++++------ 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index d3e9db8b7f..6d3e93f293 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -170,7 +170,7 @@ def get_unused_localhost_port(): usock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) usock.bind(('127.0.0.1', 0)) port = usock.getsockname()[1] - if port in (54505, 54506, 64505, 64506, 64510, 64511): + if port in (54505, 54506, 64505, 64506, 64510, 64511, 64520, 64521): # These ports are hardcoded in the test configuration port = get_unused_localhost_port() usock.close() @@ -989,28 +989,31 @@ class TestDaemon(object): known_hosts.write('') master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'master')) - master_opts['user'] = running_tests_user master_opts['known_hosts_file'] = tests_known_hosts_file - master_opts['config_dir'] = TMP_CONF_DIR master_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') - #master_opts['pki_dir'] = os.path.join(TMP, 'rootdir','pki') + master_opts['user'] = running_tests_user + master_opts['config_dir'] = TMP_CONF_DIR + master_opts['root_dir'] = os.path.join(TMP, 'rootdir') + master_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'master') minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'minion')) minion_opts['user'] = running_tests_user minion_opts['config_dir'] = TMP_CONF_DIR - minion_opts['root_dir'] = master_opts['root_dir'] = os.path.join(TMP, 'rootdir') + minion_opts['root_dir'] = os.path.join(TMP, 'rootdir') + minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'minion') sub_minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'sub_minion')) sub_minion_opts['user'] = running_tests_user sub_minion_opts['config_dir'] = TMP_SUB_MINION_CONF_DIR sub_minion_opts['root_dir'] = os.path.join(TMP, 'rootdir-sub-minion') + sub_minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir-sub-minion', 'pki', 'minion') syndic_master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'syndic_master')) - syndic_master_opts['user'] = running_tests_user - syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master') - syndic_master_opts['config_dir'] = TMP_SYNDIC_MASTER_CONF_DIR syndic_master_opts['cachedir'] = os.path.join(TMP, 'rootdir-syndic-master', 'cache') - #syndic_master_opts['pki_dir'] = os.path.join(TMP, 'rootdir-syndic-master', 'pki') + syndic_master_opts['user'] = running_tests_user + syndic_master_opts['config_dir'] = TMP_SYNDIC_MASTER_CONF_DIR + syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master') + syndic_master_opts['pki_dir'] = os.path.join(TMP, 'rootdir-syndic-master', 'pki', 'master') # The syndic config file has an include setting to include the master configuration # Let's start with a copy of the syndic master configuration diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index 6a49043630..d4465f36ce 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -4,12 +4,13 @@ master_port: 64506 interface: 127.0.0.1 tcp_pub_port: 64510 tcp_pull_port: 64511 +sock_dir: minion_sock id: minion open_mode: True log_file: minion.log log_level_logfile: debug pidfile: minion.pid -sock_dir: minion_sock +ipc_mode: tcp # module extension test.foo: baz diff --git a/tests/runtests.py b/tests/runtests.py index edd3477352..1790fc86a6 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -389,15 +389,10 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print_header(' * Salt daemons started') master_conf = TestDaemon.config('master') minion_conf = TestDaemon.config('minion') + sub_minion_conf = TestDaemon.config('sub_minion') syndic_conf = TestDaemon.config('syndic') syndic_master_conf = TestDaemon.config('syndic_master') - print_header(' * Syndic master configuration values', top=False) - print('interface: {0}'.format(syndic_master_conf['interface'])) - print('publish port: {0}'.format(syndic_master_conf['publish_port'])) - print('return port: {0}'.format(syndic_master_conf['ret_port'])) - print('\n') - print_header(' * Master configuration values', top=True) print('interface: {0}'.format(master_conf['interface'])) print('publish port: {0}'.format(master_conf['publish_port'])) @@ -406,6 +401,25 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print_header(' * Minion configuration values', top=True) print('interface: {0}'.format(minion_conf['interface'])) + print('master port: {0}'.format(minion_conf['master_port'])) + print('ipc mode: {0}'.format(minion_conf['ipc_mode'])) + print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) + print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port'])) + print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) + print('\n') + + print_header(' * Sub Minion configuration values', top=True) + print('interface: {0}'.format(sub_minion_conf['interface'])) + print('master port: {0}'.format(sub_minion_conf['master_port'])) + print('ipc mode: {0}'.format(sub_minion_conf['ipc_mode'])) + print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port'])) + print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port'])) + print('\n') + + print_header(' * Syndic master configuration values', top=False) + print('interface: {0}'.format(syndic_master_conf['interface'])) + print('publish port: {0}'.format(syndic_master_conf['publish_port'])) + print('return port: {0}'.format(syndic_master_conf['ret_port'])) print('\n') print_header(' * Syndic configuration values', top=True) From 55b8ea664e632487fd7ab9e87375d465908c3b24 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 7 Jul 2016 12:45:14 -0600 Subject: [PATCH 40/95] Remove redundant tcp_pull_port entry for minion --- tests/runtests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/runtests.py b/tests/runtests.py index 1790fc86a6..deef73503d 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -403,7 +403,6 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print('interface: {0}'.format(minion_conf['interface'])) print('master port: {0}'.format(minion_conf['master_port'])) print('ipc mode: {0}'.format(minion_conf['ipc_mode'])) - print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port'])) print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) print('\n') From 68e5620b9251b276272ea21b08c941d82c2fc99b Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 7 Jul 2016 15:47:16 -0700 Subject: [PATCH 41/95] Check actual minion ports For some reason this fixed the PYTHONPATH issue --- tests/integration/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 6d3e93f293..504b411f92 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -585,7 +585,8 @@ class SaltMinion(SaltDaemonScriptBase): return script_args def get_check_ports(self): - return set([self.config['id']]) + return set([self.config['tcp_pub_port'], + self.config['tcp_pull_port']]) class SaltMaster(SaltDaemonScriptBase): From c1ae2537f1408079db098c08be1fea5d1e8bdbd3 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 7 Jul 2016 17:27:47 -0600 Subject: [PATCH 42/95] Remove configuration from configs --- tests/integration/files/conf/master | 1 - tests/integration/files/conf/minion | 2 -- tests/integration/files/conf/sub_minion | 1 - tests/integration/files/conf/syndic_master | 1 - tests/runtests.py | 13 ++++++++----- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 0b5b09e2e4..fcae3c2a0c 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -1,5 +1,4 @@ id: master -interface: 127.0.0.1 publish_port: 64505 ret_port: 64506 worker_threads: 3 diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index d4465f36ce..bf7429da23 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -1,7 +1,6 @@ # basic config master: localhost master_port: 64506 -interface: 127.0.0.1 tcp_pub_port: 64510 tcp_pull_port: 64511 sock_dir: minion_sock @@ -10,7 +9,6 @@ open_mode: True log_file: minion.log log_level_logfile: debug pidfile: minion.pid -ipc_mode: tcp # module extension test.foo: baz diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 3190618a36..12e6bf0a32 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -1,7 +1,6 @@ # basic config master: localhost master_port: 64506 -interface: 127.0.0.1 tcp_pub_port: 64520 tcp_pull_port: 64521 sock_dir: sub_minion_sock diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index 817c5ba0fd..146384c9f8 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -1,5 +1,4 @@ id: syndic_master -interface: 127.0.0.1 publish_port: 54505 ret_port: 54506 worker_threads: 3 diff --git a/tests/runtests.py b/tests/runtests.py index deef73503d..d8c493d8a3 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -401,6 +401,7 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print_header(' * Minion configuration values', top=True) print('interface: {0}'.format(minion_conf['interface'])) + print('master: {0}'.format(minion_conf['master'])) print('master port: {0}'.format(minion_conf['master_port'])) print('ipc mode: {0}'.format(minion_conf['ipc_mode'])) print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port'])) @@ -409,23 +410,25 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print_header(' * Sub Minion configuration values', top=True) print('interface: {0}'.format(sub_minion_conf['interface'])) + print('master: {0}'.format(sub_minion_conf['master'])) print('master port: {0}'.format(sub_minion_conf['master_port'])) print('ipc mode: {0}'.format(sub_minion_conf['ipc_mode'])) print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port'])) print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port'])) print('\n') + print_header(' * Syndic configuration values', top=True) + print('interface: {0}'.format(syndic_conf['interface'])) + print('syndic master: {0}'.format(syndic_conf['syndic_master'])) + print('syndic master port: {0}'.format(syndic_conf['syndic_master_port'])) + print('\n') + print_header(' * Syndic master configuration values', top=False) print('interface: {0}'.format(syndic_master_conf['interface'])) print('publish port: {0}'.format(syndic_master_conf['publish_port'])) print('return port: {0}'.format(syndic_master_conf['ret_port'])) print('\n') - print_header(' * Syndic configuration values', top=True) - print('interface: {0}'.format(syndic_conf['interface'])) - print('syndic master port: {0}'.format(syndic_conf['syndic_master'])) - print('\n') - print_header(' Your client configuration is at {0}'.format(TestDaemon.config_location())) print('To access the minion: `salt -c {0} minion test.ping'.format(TestDaemon.config_location())) From 81d7479dfb018a08e354fd49eeca5f2f75d9b70d Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 8 Jul 2016 10:52:04 -0600 Subject: [PATCH 43/95] Remove backtick --- tests/runtests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/runtests.py b/tests/runtests.py index d8c493d8a3..1af46f602a 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -430,7 +430,7 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print('\n') print_header(' Your client configuration is at {0}'.format(TestDaemon.config_location())) - print('To access the minion: `salt -c {0} minion test.ping'.format(TestDaemon.config_location())) + print('To access the minion: salt -c {0} minion test.ping'.format(TestDaemon.config_location())) while True: time.sleep(1) From 003e60f6c6cee40dd9def3ccd1c6602252f71b2c Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 8 Jul 2016 15:59:08 -0600 Subject: [PATCH 44/95] Add cachedir for minions and syndic --- tests/integration/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 504b411f92..8cf2ef6b35 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -998,12 +998,14 @@ class TestDaemon(object): master_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'master') minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'minion')) + minion_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') minion_opts['user'] = running_tests_user minion_opts['config_dir'] = TMP_CONF_DIR minion_opts['root_dir'] = os.path.join(TMP, 'rootdir') minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'minion') sub_minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'sub_minion')) + sub_minion_opts['cachedir'] = os.path.join(TMP, 'rootdir-sub-minion', 'cache') sub_minion_opts['user'] = running_tests_user sub_minion_opts['config_dir'] = TMP_SUB_MINION_CONF_DIR sub_minion_opts['root_dir'] = os.path.join(TMP, 'rootdir-sub-minion') @@ -1023,6 +1025,7 @@ class TestDaemon(object): syndic_opts.update(salt.config._read_conf_file(os.path.join(CONF_DIR, 'syndic'))) # Lets remove the include setting syndic_opts.pop('include') + syndic_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') syndic_opts['user'] = running_tests_user syndic_opts['config_dir'] = TMP_SYNDIC_MINION_CONF_DIR From 3833c008b21f7d6a8496e799f45e0a72c8a416c9 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 8 Jul 2016 16:14:48 -0600 Subject: [PATCH 45/95] Cleans up config parameters --- tests/integration/__init__.py | 22 ++++++++++++---------- tests/integration/files/conf/master | 3 ++- tests/integration/files/conf/minion | 1 + tests/integration/files/conf/sub_minion | 1 + tests/integration/files/conf/syndic | 17 ++++++----------- tests/integration/files/conf/syndic_master | 6 +++++- 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 8cf2ef6b35..b20dfb83e6 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -989,6 +989,7 @@ class TestDaemon(object): with salt.utils.fopen(tests_known_hosts_file, 'w') as known_hosts: known_hosts.write('') + # This master connects to syndic_master via a syndic master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'master')) master_opts['known_hosts_file'] = tests_known_hosts_file master_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') @@ -997,6 +998,15 @@ class TestDaemon(object): master_opts['root_dir'] = os.path.join(TMP, 'rootdir') master_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'master') + # This is the syndic for master + # Let's start with a copy of the syndic master configuration + syndic_opts = copy.deepcopy(master_opts) + # Let's update with the syndic configuration + syndic_opts.update(salt.config._read_conf_file(os.path.join(CONF_DIR, 'syndic'))) + syndic_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') + syndic_opts['config_dir'] = TMP_SYNDIC_MINION_CONF_DIR + + # This minion connects to master minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'minion')) minion_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') minion_opts['user'] = running_tests_user @@ -1004,6 +1014,7 @@ class TestDaemon(object): minion_opts['root_dir'] = os.path.join(TMP, 'rootdir') minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'minion') + # This sub_minion also connects to master sub_minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'sub_minion')) sub_minion_opts['cachedir'] = os.path.join(TMP, 'rootdir-sub-minion', 'cache') sub_minion_opts['user'] = running_tests_user @@ -1011,6 +1022,7 @@ class TestDaemon(object): sub_minion_opts['root_dir'] = os.path.join(TMP, 'rootdir-sub-minion') sub_minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir-sub-minion', 'pki', 'minion') + # This is the master of masters syndic_master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'syndic_master')) syndic_master_opts['cachedir'] = os.path.join(TMP, 'rootdir-syndic-master', 'cache') syndic_master_opts['user'] = running_tests_user @@ -1018,16 +1030,6 @@ class TestDaemon(object): syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master') syndic_master_opts['pki_dir'] = os.path.join(TMP, 'rootdir-syndic-master', 'pki', 'master') - # The syndic config file has an include setting to include the master configuration - # Let's start with a copy of the syndic master configuration - syndic_opts = copy.deepcopy(master_opts) - # Let's update with the syndic configuration - syndic_opts.update(salt.config._read_conf_file(os.path.join(CONF_DIR, 'syndic'))) - # Lets remove the include setting - syndic_opts.pop('include') - syndic_opts['cachedir'] = os.path.join(TMP, 'rootdir', 'cache') - syndic_opts['user'] = running_tests_user - syndic_opts['config_dir'] = TMP_SYNDIC_MINION_CONF_DIR if transport == 'raet': master_opts['transport'] = 'raet' diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index fcae3c2a0c..989d152fd3 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -1,3 +1,5 @@ +# Master Settings +# Connects to syndic_master via syndic id: master publish_port: 64505 ret_port: 64506 @@ -6,7 +8,6 @@ pidfile: master.pid sock_dir: master_sock timeout: 3 open_mode: True -syndic_master: localhost fileserver_list_cache_time: 0 file_buffer_size: 8192 pillar_opts: True diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index bf7429da23..0996078f51 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -1,4 +1,5 @@ # basic config +# Connects to master master: localhost master_port: 64506 tcp_pub_port: 64510 diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index 12e6bf0a32..b9d860a48f 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -1,4 +1,5 @@ # basic config +# Connects to master master: localhost master_port: 64506 tcp_pub_port: 64520 diff --git a/tests/integration/files/conf/syndic b/tests/integration/files/conf/syndic index 3c2201a28b..059899c45c 100644 --- a/tests/integration/files/conf/syndic +++ b/tests/integration/files/conf/syndic @@ -1,13 +1,8 @@ -include: master -# syndic basic config -# same config as master ./except the syndic bits -# in the TestCase we add at the top of the configfile the content of ./master -# to avoid duplication -order_masters: True -acceptance_wait_time: 1 -syndic_log_file: syndic.log -log_level_logfile: debug -syndic_pidfile: syndic.pid +# Syndic Settings +id: syndic syndic_master: localhost syndic_master_port: 54506 -id: syndic +syndic_log_file: syndic.log +syndic_pidfile: syndic.pid +tcp_pub_port: 64530 +tcp_pull_port: 64531 diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index 146384c9f8..676fbcf5b9 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -1,3 +1,5 @@ +# Master Settings +# This is the Master of Masters id: syndic_master publish_port: 54505 ret_port: 54506 @@ -6,10 +8,12 @@ pidfile: syndic_master.pid sock_dir: syndic_master_sock timeout: 1 open_mode: True -order_masters: True fileserver_list_cache_time: 0 pillar_opts: True tcp_master_publish_pull: 33305 tcp_master_workers: 33306 log_file: syndic_master.log log_level_logfile: debug + +# Syndic Settings +order_masters: True From b5b0b9ae4bfc9f0593a33cc667c04a7909e28c47 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 8 Jul 2016 16:21:27 -0600 Subject: [PATCH 46/95] Add more info to --interactive report --- tests/runtests.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/runtests.py b/tests/runtests.py index 1af46f602a..b0445f987d 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -393,16 +393,33 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): syndic_conf = TestDaemon.config('syndic') syndic_master_conf = TestDaemon.config('syndic_master') + print_header(' * Syndic master configuration values (MoM)', top=False) + print('interface: {0}'.format(syndic_master_conf['interface'])) + print('publish port: {0}'.format(syndic_master_conf['publish_port'])) + print('return port: {0}'.format(syndic_master_conf['ret_port'])) + print('transport: {0}'.format(syndic_master_conf['transport'])) + print('\n') + + print_header(' * Syndic configuration values', top=True) + print('interface: {0}'.format(syndic_conf['interface'])) + print('syndic master: {0}'.format(syndic_conf['syndic_master'])) + print('syndic master port: {0}'.format(syndic_conf['syndic_master_port'])) + print('tcp pub port: {0}'.format(syndic_conf['tcp_pub_port'])) + print('tcp pull port: {0}'.format(syndic_conf['tcp_pull_port'])) + print('\n') + print_header(' * Master configuration values', top=True) print('interface: {0}'.format(master_conf['interface'])) print('publish port: {0}'.format(master_conf['publish_port'])) print('return port: {0}'.format(master_conf['ret_port'])) + print('transport: {0}'.format(master_conf['transport'])) print('\n') print_header(' * Minion configuration values', top=True) print('interface: {0}'.format(minion_conf['interface'])) print('master: {0}'.format(minion_conf['master'])) print('master port: {0}'.format(minion_conf['master_port'])) + print('transport: {0}'.format(minion_conf['transport'])) print('ipc mode: {0}'.format(minion_conf['ipc_mode'])) print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port'])) print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) @@ -412,23 +429,12 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print('interface: {0}'.format(sub_minion_conf['interface'])) print('master: {0}'.format(sub_minion_conf['master'])) print('master port: {0}'.format(sub_minion_conf['master_port'])) + print('transport: {0}'.format(sub_minion_conf['transport'])) print('ipc mode: {0}'.format(sub_minion_conf['ipc_mode'])) print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port'])) print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port'])) print('\n') - print_header(' * Syndic configuration values', top=True) - print('interface: {0}'.format(syndic_conf['interface'])) - print('syndic master: {0}'.format(syndic_conf['syndic_master'])) - print('syndic master port: {0}'.format(syndic_conf['syndic_master_port'])) - print('\n') - - print_header(' * Syndic master configuration values', top=False) - print('interface: {0}'.format(syndic_master_conf['interface'])) - print('publish port: {0}'.format(syndic_master_conf['publish_port'])) - print('return port: {0}'.format(syndic_master_conf['ret_port'])) - print('\n') - print_header(' Your client configuration is at {0}'.format(TestDaemon.config_location())) print('To access the minion: salt -c {0} minion test.ping'.format(TestDaemon.config_location())) From 4e56d289938cbcd9588bbcf7cd7990c677e0179f Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 11 Jul 2016 15:15:28 -0700 Subject: [PATCH 47/95] Add tcp_master settings to both masters --- tests/integration/__init__.py | 7 +++++-- tests/integration/files/conf/master | 7 +++++++ tests/integration/files/conf/sub_minion | 1 - tests/integration/files/conf/syndic | 4 ++-- tests/integration/files/conf/syndic_master | 9 +++++++-- tests/runtests.py | 18 ++++++------------ 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index b20dfb83e6..17adac8fbb 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -585,8 +585,11 @@ class SaltMinion(SaltDaemonScriptBase): return script_args def get_check_ports(self): - return set([self.config['tcp_pub_port'], - self.config['tcp_pull_port']]) + if salt.utils.is_windows(): + return set([self.config['tcp_pub_port'], + self.config['tcp_pull_port']]) + else: + return set([self.config['id']]) class SaltMaster(SaltDaemonScriptBase): diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 989d152fd3..4c908068b3 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -16,6 +16,13 @@ log_level_logfile: debug key_logfile: key.log token_file: /tmp/ksfjhdgiuebfgnkefvsikhfjdgvkjahcsidk +# These settings needed for tests on Windows which defaults +# to ipc_mode: tcp +tcp_master_pub_port: 64512 +tcp_master_pull_port: 64513 +tcp_master_publish_pull: 64514 +tcp_master_workers: 64515 + peer: '.*': - 'test.*' diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index b9d860a48f..b1f105fa1b 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -10,7 +10,6 @@ open_mode: True log_file: sub_minion.log log_level_logfile: debug pidfile: sub_minion.pid -ipc_mode: tcp # module extension test.foo: baz diff --git a/tests/integration/files/conf/syndic b/tests/integration/files/conf/syndic index 059899c45c..788828d3c1 100644 --- a/tests/integration/files/conf/syndic +++ b/tests/integration/files/conf/syndic @@ -4,5 +4,5 @@ syndic_master: localhost syndic_master_port: 54506 syndic_log_file: syndic.log syndic_pidfile: syndic.pid -tcp_pub_port: 64530 -tcp_pull_port: 64531 +tcp_pub_port: 64510 +tcp_pull_port: 64511 diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index 676fbcf5b9..eae676dbfa 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -10,10 +10,15 @@ timeout: 1 open_mode: True fileserver_list_cache_time: 0 pillar_opts: True -tcp_master_publish_pull: 33305 -tcp_master_workers: 33306 log_file: syndic_master.log log_level_logfile: debug +# These settings needed for tests on Windows which defaults +# to ipc_mode: tcp +tcp_master_pub_port: 54512 +tcp_master_pull_port: 54513 +tcp_master_publish_pull: 54514 +tcp_master_workers: 54515 + # Syndic Settings order_masters: True diff --git a/tests/runtests.py b/tests/runtests.py index b0445f987d..0fe7966ace 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -397,42 +397,36 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): print('interface: {0}'.format(syndic_master_conf['interface'])) print('publish port: {0}'.format(syndic_master_conf['publish_port'])) print('return port: {0}'.format(syndic_master_conf['ret_port'])) - print('transport: {0}'.format(syndic_master_conf['transport'])) print('\n') print_header(' * Syndic configuration values', top=True) print('interface: {0}'.format(syndic_conf['interface'])) print('syndic master: {0}'.format(syndic_conf['syndic_master'])) print('syndic master port: {0}'.format(syndic_conf['syndic_master_port'])) - print('tcp pub port: {0}'.format(syndic_conf['tcp_pub_port'])) - print('tcp pull port: {0}'.format(syndic_conf['tcp_pull_port'])) print('\n') print_header(' * Master configuration values', top=True) print('interface: {0}'.format(master_conf['interface'])) print('publish port: {0}'.format(master_conf['publish_port'])) print('return port: {0}'.format(master_conf['ret_port'])) - print('transport: {0}'.format(master_conf['transport'])) print('\n') print_header(' * Minion configuration values', top=True) print('interface: {0}'.format(minion_conf['interface'])) print('master: {0}'.format(minion_conf['master'])) print('master port: {0}'.format(minion_conf['master_port'])) - print('transport: {0}'.format(minion_conf['transport'])) - print('ipc mode: {0}'.format(minion_conf['ipc_mode'])) - print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port'])) - print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) + if minion_conf['ipc_mode'] == 'tcp': + print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port'])) + print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port'])) print('\n') print_header(' * Sub Minion configuration values', top=True) print('interface: {0}'.format(sub_minion_conf['interface'])) print('master: {0}'.format(sub_minion_conf['master'])) print('master port: {0}'.format(sub_minion_conf['master_port'])) - print('transport: {0}'.format(sub_minion_conf['transport'])) - print('ipc mode: {0}'.format(sub_minion_conf['ipc_mode'])) - print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port'])) - print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port'])) + if sub_minion_conf['ipc_mode'] == 'tcp': + print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port'])) + print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port'])) print('\n') print_header(' Your client configuration is at {0}'.format(TestDaemon.config_location())) From 5f5d7193b0339e7940a3e759e7933f60f9dd0d2c Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 13 Jul 2016 10:02:58 -0600 Subject: [PATCH 48/95] Add interface to all configs This got removed during the attempt to make the tests run on Windows. I added them everywhere even though that's not strictly necessary. Without these set on the master and sub_master, Windows will throw erorrs because the networking stack will decide that anything that's bound to 0.0.0.0 represents an extestential threat to any socket that attempts to *connect* to one of those ports. (For...reasons.) --- tests/integration/files/conf/master | 1 + tests/integration/files/conf/minion | 1 + tests/integration/files/conf/sub_minion | 1 + tests/integration/files/conf/syndic | 1 + tests/integration/files/conf/syndic_master | 1 + 5 files changed, 5 insertions(+) diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 4c908068b3..76e2d80f14 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -1,6 +1,7 @@ # Master Settings # Connects to syndic_master via syndic id: master +interface: 127.0.0.1 publish_port: 64505 ret_port: 64506 worker_threads: 3 diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index 0996078f51..53bdb4ac07 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -2,6 +2,7 @@ # Connects to master master: localhost master_port: 64506 +interface: 127.0.0.1 tcp_pub_port: 64510 tcp_pull_port: 64511 sock_dir: minion_sock diff --git a/tests/integration/files/conf/sub_minion b/tests/integration/files/conf/sub_minion index b1f105fa1b..008ade341d 100644 --- a/tests/integration/files/conf/sub_minion +++ b/tests/integration/files/conf/sub_minion @@ -1,6 +1,7 @@ # basic config # Connects to master master: localhost +interface: 127.0.0.1 master_port: 64506 tcp_pub_port: 64520 tcp_pull_port: 64521 diff --git a/tests/integration/files/conf/syndic b/tests/integration/files/conf/syndic index 788828d3c1..5746d493c9 100644 --- a/tests/integration/files/conf/syndic +++ b/tests/integration/files/conf/syndic @@ -1,5 +1,6 @@ # Syndic Settings id: syndic +interface: 127.0.0.1 syndic_master: localhost syndic_master_port: 54506 syndic_log_file: syndic.log diff --git a/tests/integration/files/conf/syndic_master b/tests/integration/files/conf/syndic_master index eae676dbfa..203820b97e 100644 --- a/tests/integration/files/conf/syndic_master +++ b/tests/integration/files/conf/syndic_master @@ -1,6 +1,7 @@ # Master Settings # This is the Master of Masters id: syndic_master +interface: 127.0.0.1 publish_port: 54505 ret_port: 54506 worker_threads: 3 From 03039c13e2f6bba2715830ba705570c421f8b997 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 18 Jul 2016 06:45:57 -0600 Subject: [PATCH 49/95] Lint --- tests/integration/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 17adac8fbb..851238b682 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1033,7 +1033,6 @@ class TestDaemon(object): syndic_master_opts['root_dir'] = os.path.join(TMP, 'rootdir-syndic-master') syndic_master_opts['pki_dir'] = os.path.join(TMP, 'rootdir-syndic-master', 'pki', 'master') - if transport == 'raet': master_opts['transport'] = 'raet' master_opts['raet_port'] = 64506 From ec8a6ebab35c14281d591247cc7393691026d566 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 20 Jul 2016 09:44:23 -0600 Subject: [PATCH 50/95] Correct config test errors --- tests/integration/__init__.py | 2 +- tests/unit/config/config_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 851238b682..64acd23b84 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1015,7 +1015,7 @@ class TestDaemon(object): minion_opts['user'] = running_tests_user minion_opts['config_dir'] = TMP_CONF_DIR minion_opts['root_dir'] = os.path.join(TMP, 'rootdir') - minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki', 'minion') + minion_opts['pki_dir'] = os.path.join(TMP, 'rootdir', 'pki') # This sub_minion also connects to master sub_minion_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'sub_minion')) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e1ad1e24b4..1675c29481 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -324,8 +324,8 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn): self.assertEqual(syndic_opts['master'], 'localhost') self.assertEqual(syndic_opts['sock_dir'], os.path.join(root_dir, 'minion_sock')) self.assertEqual(syndic_opts['cachedir'], os.path.join(root_dir, 'cache')) - self.assertEqual(syndic_opts['log_file'], os.path.join(root_dir, 'osyndic.log')) - self.assertEqual(syndic_opts['pidfile'], os.path.join(root_dir, 'osyndic.pid')) + self.assertEqual(syndic_opts['log_file'], os.path.join(root_dir, 'syndic.log')) + self.assertEqual(syndic_opts['pidfile'], os.path.join(root_dir, 'syndic.pid')) # Show that the options of localclient that repub to local master # are not merged with syndic ones self.assertEqual(syndic_opts['_master_conf_file'], minion_conf_path) From 56838f55902cf923af3af8d559fd007068646815 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Wed, 20 Jul 2016 14:16:58 -0600 Subject: [PATCH 51/95] Remove unnecessary import --- salt/utils/winapi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/utils/winapi.py b/salt/utils/winapi.py index f5ef633111..a1fe6c6862 100644 --- a/salt/utils/winapi.py +++ b/salt/utils/winapi.py @@ -5,7 +5,6 @@ from __future__ import absolute_import # Import python libs import logging import threading -from salt.loader import minion_mods try: import pythoncom From f8753c1aa83073426c5552046ff43cabbc11d4d3 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 11 Jul 2016 22:09:49 -0500 Subject: [PATCH 52/95] salt/fileserver/svnfs.py: add file stat to find_file --- salt/fileserver/svnfs.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/salt/fileserver/svnfs.py b/salt/fileserver/svnfs.py index 42edaf2461..32987cce98 100644 --- a/salt/fileserver/svnfs.py +++ b/salt/fileserver/svnfs.py @@ -583,6 +583,22 @@ def find_file(path, tgt_env='base', **kwargs): # pylint: disable=W0613 if os.path.isfile(full): fnd['rel'] = path fnd['path'] = full + try: + # Converting the stat result to a list, the elements of the + # list correspond to the following stat_result params: + # 0 => st_mode=33188 + # 1 => st_ino=10227377 + # 2 => st_dev=65026 + # 3 => st_nlink=1 + # 4 => st_uid=1000 + # 5 => st_gid=1000 + # 6 => st_size=1056233 + # 7 => st_atime=1468284229 + # 8 => st_mtime=1456338235 + # 9 => st_ctime=1456338235 + fnd['stat'] = list(os.stat(full)) + except Exception: + pass return fnd return fnd From c11683b10204fe462ae151bd05a5997a915049eb Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 11 Jul 2016 22:10:59 -0500 Subject: [PATCH 53/95] salt/utils/gitfs.py: add file stat to find_file --- salt/utils/gitfs.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index d1badb44f0..44408ebb26 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -940,7 +940,7 @@ class GitPython(GitProvider): tree = self.get_tree(tgt_env) if not tree: # Branch/tag/SHA not found - return None, None + return None, None, None blob = None depth = 0 while True: @@ -968,7 +968,9 @@ class GitPython(GitProvider): except KeyError: # File not found or repo_path points to a directory break - return blob, blob.hexsha if blob is not None else blob + if isinstance(blob, git.Blob): + return blob, blob.hexsha, blob.mode + return None, None, None def get_tree(self, tgt_env): ''' @@ -1480,7 +1482,7 @@ class Pygit2(GitProvider): tree = self.get_tree(tgt_env) if not tree: # Branch/tag/SHA not found in repo - return None, None + return None, None, None blob = None depth = 0 while True: @@ -1502,7 +1504,9 @@ class Pygit2(GitProvider): blob = self.repo[oid] except KeyError: break - return blob, blob.hex if blob is not None else blob + if isinstance(blob, pygit2.Blob): + return blob, blob.hex, blob.mode + return None, None, None def get_tree(self, tgt_env): ''' @@ -1827,7 +1831,7 @@ class Dulwich(GitProvider): # pylint: disable=abstract-method tree = self.get_tree(tgt_env) if not tree: # Branch/tag/SHA not found - return None, None + return None, None, None blob = None depth = 0 while True: @@ -1855,7 +1859,8 @@ class Dulwich(GitProvider): # pylint: disable=abstract-method break except KeyError: break - return blob, blob.sha().hexdigest() if blob is not None else blob + if isinstance(blob, dulwich.objects.Blob): + return blob, blob.sha().hexdigest(), mode def get_conf(self): ''' @@ -2701,6 +2706,20 @@ class GitFS(GitBase): if blob is None: continue + def _add_file_stat(fnd, mode): + ''' + Add a the mode to the return dict. In other fileserver backends + we stat the file to get its mode, and add the stat result + (passed through list() for better serialization) to the 'stat' + key in the return dict. However, since we aren't using the + stat result for anything but the mode at this time, we can + avoid unnecessary work by just manually creating the list and + not running an os.stat() on all files in the repo. + ''' + if mode is not None: + fnd['stat'] = [mode] + return fnd + salt.fileserver.wait_lock(lk_fn, dest) if os.path.isfile(blobshadest) and os.path.isfile(dest): with salt.utils.fopen(blobshadest, 'r') as fp_: @@ -2708,7 +2727,7 @@ class GitFS(GitBase): if sha == blob_hexsha: fnd['rel'] = path fnd['path'] = dest - return fnd + return _add_file_stat(fnd) with salt.utils.fopen(lk_fn, 'w+') as fp_: fp_.write('') for filename in glob.glob(hashes_glob): @@ -2726,7 +2745,7 @@ class GitFS(GitBase): pass fnd['rel'] = path fnd['path'] = dest - return fnd + return _add_file_stat(fnd) # No matching file was found in tgt_env. Return a dict with empty paths # so the calling function knows the file could not be found. From 5983d990dfcef8bd26ee6dba7340a4d9aa2a5e49 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 11 Jul 2016 22:11:30 -0500 Subject: [PATCH 54/95] salt/fileserver/azurefs.py: add try/except to file stat --- salt/fileserver/azurefs.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/salt/fileserver/azurefs.py b/salt/fileserver/azurefs.py index db14a66fea..6ab9e068a9 100644 --- a/salt/fileserver/azurefs.py +++ b/salt/fileserver/azurefs.py @@ -97,7 +97,22 @@ def find_file(path, saltenv='base', **kwargs): __opts__, full): fnd['path'] = full fnd['rel'] = path - fnd['stat'] = list(os.stat(full)) + try: + # Converting the stat result to a list, the elements of the + # list correspond to the following stat_result params: + # 0 => st_mode=33188 + # 1 => st_ino=10227377 + # 2 => st_dev=65026 + # 3 => st_nlink=1 + # 4 => st_uid=1000 + # 5 => st_gid=1000 + # 6 => st_size=1056233 + # 7 => st_atime=1468284229 + # 8 => st_mtime=1456338235 + # 9 => st_ctime=1456338235 + fnd['stat'] = list(os.stat(full)) + except Exception: + pass return fnd From 150f35e2c8b2b856bf3f58bcc7e8c63746789a44 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 11 Jul 2016 22:11:30 -0500 Subject: [PATCH 55/95] salt/fileserver/hgfs.py: add try/except to file stat --- salt/fileserver/hgfs.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/salt/fileserver/hgfs.py b/salt/fileserver/hgfs.py index a39f44d64f..263490a4b2 100644 --- a/salt/fileserver/hgfs.py +++ b/salt/fileserver/hgfs.py @@ -687,7 +687,22 @@ def find_file(path, tgt_env='base', **kwargs): # pylint: disable=W0613 pass fnd['rel'] = path fnd['path'] = dest - fnd['stat'] = list(os.stat(dest)) + try: + # Converting the stat result to a list, the elements of the + # list correspond to the following stat_result params: + # 0 => st_mode=33188 + # 1 => st_ino=10227377 + # 2 => st_dev=65026 + # 3 => st_nlink=1 + # 4 => st_uid=1000 + # 5 => st_gid=1000 + # 6 => st_size=1056233 + # 7 => st_atime=1468284229 + # 8 => st_mtime=1456338235 + # 9 => st_ctime=1456338235 + fnd['stat'] = list(os.stat(dest)) + except Exception: + pass repo['repo'].close() return fnd return fnd From 46e6d4005b52474e56abcaa3808b5640129b1db1 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 11 Jul 2016 22:11:30 -0500 Subject: [PATCH 56/95] salt/fileserver/roots.py: add try/except to file stat --- salt/fileserver/roots.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/salt/fileserver/roots.py b/salt/fileserver/roots.py index 0faa3be0ff..fdb2037fc2 100644 --- a/salt/fileserver/roots.py +++ b/salt/fileserver/roots.py @@ -33,7 +33,7 @@ log = logging.getLogger(__name__) def find_file(path, saltenv='base', **kwargs): ''' - Search the environment for the relative path + Search the environment for the relative path. ''' if 'env' in kwargs: salt.utils.warn_until( @@ -51,6 +51,32 @@ def find_file(path, saltenv='base', **kwargs): return fnd if saltenv not in __opts__['file_roots']: return fnd + + def _add_file_stat(fnd): + ''' + Stat the file and, assuming no errors were found, convert the stat + result to a list of values and add to the return dict. + + Converting the stat result to a list, the elements of the list + correspond to the following stat_result params: + + 0 => st_mode=33188 + 1 => st_ino=10227377 + 2 => st_dev=65026 + 3 => st_nlink=1 + 4 => st_uid=1000 + 5 => st_gid=1000 + 6 => st_size=1056233 + 7 => st_atime=1468284229 + 8 => st_mtime=1456338235 + 9 => st_ctime=1456338235 + ''' + try: + fnd['stat'] = list(os.stat(fnd['path'])) + except Exception: + pass + return fnd + if 'index' in kwargs: try: root = __opts__['file_roots'][saltenv][int(kwargs['index'])] @@ -64,15 +90,14 @@ def find_file(path, saltenv='base', **kwargs): if os.path.isfile(full) and not salt.fileserver.is_file_ignored(__opts__, full): fnd['path'] = full fnd['rel'] = path - fnd['stat'] = list(os.stat(full)) + return _add_file_stat(fnd) return fnd for root in __opts__['file_roots'][saltenv]: full = os.path.join(root, path) if os.path.isfile(full) and not salt.fileserver.is_file_ignored(__opts__, full): fnd['path'] = full fnd['rel'] = path - fnd['stat'] = list(os.stat(full)) - return fnd + return _add_file_stat(fnd) return fnd From dbe850121fa954e05d0cea9f2f7262aae1123c42 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 12 Jul 2016 13:22:21 -0500 Subject: [PATCH 57/95] Optimize file retrieval logging logging already uses string replacement, string formatting is overkill. It'll be a modest improvement, but more impactful on file.recurse states which perform a lot of file downloads. --- salt/fileclient.py | 65 +++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index cef74c4a6c..5972a10072 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -911,9 +911,8 @@ class RemoteClient(Client): hash_server = self.hash_file(path, saltenv) if hash_server == '': log.debug( - 'Could not find file from saltenv \'{0}\', \'{1}\''.format( - saltenv, path - ) + 'Could not find file from saltenv \'%s\', \'%s\'', + saltenv, path ) return False @@ -924,32 +923,30 @@ class RemoteClient(Client): rel_path = self._check_proto(path) log.debug( - 'In saltenv \'{0}\', looking at rel_path \'{1}\' to resolve ' - '\'{2}\''.format(saltenv, rel_path, path) + 'In saltenv \'%s\', looking at rel_path \'%s\' to resolve ' + '\'%s\'', saltenv, rel_path, path ) with self._cache_loc( rel_path, saltenv, cachedir=cachedir) as cache_dest: dest2check = cache_dest log.debug( - 'In saltenv \'{0}\', ** considering ** path \'{1}\' to resolve ' - '\'{2}\''.format(saltenv, dest2check, path) + 'In saltenv \'%s\', ** considering ** path \'%s\' to resolve ' + '\'%s\'', saltenv, dest2check, path ) if dest2check and os.path.isfile(dest2check): hash_local = self.hash_file(dest2check, saltenv) if hash_local == hash_server: log.info( - 'Fetching file from saltenv \'{0}\', ** skipped ** ' - 'latest already in cache \'{1}\''.format( - saltenv, path - ) + 'Fetching file from saltenv \'%s\', ** skipped ** ' + 'latest already in cache \'%s\'', saltenv, path ) return dest2check log.debug( - 'Fetching file from saltenv \'{0}\', ** attempting ** ' - '\'{1}\''.format(saltenv, path) + 'Fetching file from saltenv \'%s\', ** attempting ** \'%s\'', + saltenv, path ) d_tries = 0 transport_tries = 0 @@ -971,7 +968,7 @@ class RemoteClient(Client): return False fn_ = salt.utils.fopen(dest, 'wb+') else: - log.debug('No dest file found {0}'.format(dest)) + log.debug('Dest file \'%s\' not found', dest) while True: if not fn_: @@ -1003,8 +1000,10 @@ class RemoteClient(Client): d_tries += 1 hsum = salt.utils.get_hash(dest, salt.utils.to_str(data.get('hash_type', b'md5'))) if hsum != data['hsum']: - log.warning('Bad download of file {0}, attempt {1} ' - 'of 3'.format(path, d_tries)) + log.warning( + 'Bad download of file %s, attempt %d of 3', + path, d_tries + ) continue break if not fn_: @@ -1023,31 +1022,37 @@ class RemoteClient(Client): else: data = data['data'] fn_.write(data) - except (TypeError, KeyError) as e: + except (TypeError, KeyError) as exc: + try: + data_type = type(data).__name__ + except AttributeError: + # Shouldn't happen, but don't let this cause a traceback. + data_type = str(type(data)) transport_tries += 1 - log.warning('Data transport is broken, got: {0}, type: {1}, ' - 'exception: {2}, attempt {3} of 3'.format( - data, type(data), e, transport_tries) - ) + log.warning( + 'Data transport is broken, got: %s, type: %s, ' + 'exception: %s, attempt %d of 3', + data, data_type, exc, transport_tries + ) self._refresh_channel() if transport_tries > 3: - log.error('Data transport is broken, got: {0}, type: {1}, ' - 'exception: {2}, ' - 'Retry attempts exhausted'.format( - data, type(data), e) - ) + log.error( + 'Data transport is broken, got: %s, type: %s, ' + 'exception: %s, retry attempts exhausted', + data, data_type, exc + ) break if fn_: fn_.close() log.info( - 'Fetching file from saltenv \'{0}\', ** done ** ' - '\'{1}\''.format(saltenv, path) + 'Fetching file from saltenv \'%s\', ** done ** \'%s\'', + saltenv, path ) else: log.debug( - 'In saltenv \'{0}\', we are ** missing ** the file ' - '\'{1}\''.format(saltenv, path) + 'In saltenv \'%s\', we are ** missing ** the file \'%s\'', + saltenv, path ) return dest From af98636ebef060c8c5d634cc1b5498465a6f6959 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 20 Jul 2016 01:09:33 -0500 Subject: [PATCH 58/95] gitfs: Include file mode in find_file results Since we're only going to be using the stat information for mode manipulation, there is no need to waste CPU resources by stat'ing the file. Besides, only the mode information is stored in the git metadata, so the remaining params from a posix.stat_result would be inaccurate and all but useless to us. --- salt/utils/gitfs.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index 44408ebb26..69dd6db589 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -1484,28 +1484,30 @@ class Pygit2(GitProvider): # Branch/tag/SHA not found in repo return None, None, None blob = None + mode = None depth = 0 while True: depth += 1 if depth > SYMLINK_RECURSE_DEPTH: break try: - if stat.S_ISLNK(tree[path].filemode): + entry = tree[path] + mode = entry.filemode + if stat.S_ISLNK(mode): # Path is a symlink. The blob data corresponding to this # path's object ID will be the target of the symlink. Follow # the symlink and set path to the location indicated # in the blob data. - link_tgt = self.repo[tree[path].oid].data + link_tgt = self.repo[entry.oid].data path = os.path.normpath( os.path.join(os.path.dirname(path), link_tgt) ) else: - oid = tree[path].oid - blob = self.repo[oid] + blob = self.repo[entry.oid] except KeyError: break if isinstance(blob, pygit2.Blob): - return blob, blob.hex, blob.mode + return blob, blob.hex, mode return None, None, None def get_tree(self, tgt_env): @@ -1833,6 +1835,7 @@ class Dulwich(GitProvider): # pylint: disable=abstract-method # Branch/tag/SHA not found return None, None, None blob = None + mode = None depth = 0 while True: depth += 1 @@ -1861,6 +1864,7 @@ class Dulwich(GitProvider): # pylint: disable=abstract-method break if isinstance(blob, dulwich.objects.Blob): return blob, blob.sha().hexdigest(), mode + return None, None, None def get_conf(self): ''' @@ -2702,7 +2706,7 @@ class GitFS(GitBase): if repo.root(tgt_env): repo_path = os.path.join(repo.root(tgt_env), repo_path) - blob, blob_hexsha = repo.find_file(repo_path, tgt_env) + blob, blob_hexsha, blob_mode = repo.find_file(repo_path, tgt_env) if blob is None: continue @@ -2727,7 +2731,7 @@ class GitFS(GitBase): if sha == blob_hexsha: fnd['rel'] = path fnd['path'] = dest - return _add_file_stat(fnd) + return _add_file_stat(fnd, blob_mode) with salt.utils.fopen(lk_fn, 'w+') as fp_: fp_.write('') for filename in glob.glob(hashes_glob): @@ -2745,7 +2749,7 @@ class GitFS(GitBase): pass fnd['rel'] = path fnd['path'] = dest - return _add_file_stat(fnd) + return _add_file_stat(fnd, blob_mode) # No matching file was found in tgt_env. Return a dict with empty paths # so the calling function knows the file could not be found. @@ -2802,7 +2806,7 @@ class GitFS(GitBase): load.pop('env') if not all(x in load for x in ('path', 'saltenv')): - return '' + return '', None ret = {'hash_type': self.opts['hash_type']} relpath = fnd['rel'] path = fnd['path'] From f609011a555a0a3737bd27c36a60720667391eef Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 20 Jul 2016 01:46:36 -0500 Subject: [PATCH 59/95] Remove unnecessary 2nd except clause Since the action is the same for both exceptions, a single except clause is sufficient here. --- salt/fileserver/azurefs.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/salt/fileserver/azurefs.py b/salt/fileserver/azurefs.py index 6ab9e068a9..9dcfd0a149 100644 --- a/salt/fileserver/azurefs.py +++ b/salt/fileserver/azurefs.py @@ -86,11 +86,8 @@ def find_file(path, saltenv='base', **kwargs): 'rel': ''} try: root = os.path.join(salt.syspaths.CACHE_DIR, 'azure') - except IndexError: - # An invalid index was passed - return fnd - except ValueError: - # An invalid index option was passed + except (IndexError, ValueError): + # An invalid index or index option was passed return fnd full = os.path.join(root, path) if os.path.isfile(full) and not salt.fileserver.is_file_ignored( From c8795ce1303584a57e2dc3ae1f0e003b56df9cdf Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 20 Jul 2016 01:48:25 -0500 Subject: [PATCH 60/95] Add a couple mode-related convenience functions --- salt/utils/__init__.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index dd6fedafc4..75bbfc9a63 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -1805,6 +1805,34 @@ def check_state_result(running, recurse=False): return ret +def st_mode_to_octal(mode): + ''' + Convert the st_mode value from a stat(2) call (as returned from os.stat()) + to an octal mode. + ''' + try: + return oct(mode)[-4:] + except (TypeError, IndexError): + return '' + + +def normalize_mode(mode): + ''' + Return a mode value, normalized to a string and containing a leading zero + if it does not have one. + + Allow "keep" as a valid mode (used by file state/module to preserve mode + from the Salt fileserver in file states). + ''' + if mode is None: + return None + if not isinstance(mode, six.string_types): + mode = str(mode) + # Strip any quotes any initial zeroes, then though zero-pad it up to 4. + # This ensures that somethign like '00644' is normalized to '0644' + return mode.strip('"').strip('\'').lstrip('0').zfill(4) + + def test_mode(**kwargs): ''' Examines the kwargs passed and returns True if any kwarg which matching From 934c6b27c626bb53ec4c8f618852ac5ed8dd5d77 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 20 Jul 2016 01:49:25 -0500 Subject: [PATCH 61/95] Add a dedicated function for returning both the file hash and stat Also add convenience functions that allow find_file to be invoked on a remote FS backend by the minion via the fileclient. --- salt/fileserver/__init__.py | 62 +++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index 9c06c3f87e..76465f4f31 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -480,6 +480,28 @@ class Fileserver(object): if fstr in self.servers: self.servers[fstr]() + def _find_file(self, load): + ''' + Convenience function for calls made using the RemoteClient + ''' + path = load.get('path') + if not path: + return {'path': '', + 'rel': ''} + tgt_env = load.get('saltenv', 'base') + return self.find_file(path, tgt_env) + + def file_find(self, load): + ''' + Convenience function for calls made using the LocalClient + ''' + path = load.get('path') + if not path: + return {'path': '', + 'rel': ''} + tgt_env = load.get('saltenv', 'base') + return self.find_file(path, tgt_env) + def find_file(self, path, saltenv, back=None): ''' Find the path and return the fnd structure, this structure is passed @@ -560,32 +582,52 @@ class Fileserver(object): return self.servers[fstr](load, fnd) return ret - def file_hash(self, load): + def __file_hash_and_stat(self, load): ''' - Return the hash of a given file + Common code for hashing and stating files ''' if 'env' in load: salt.utils.warn_until( 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt Carbon. This warning will be removed in Salt Oxygen.' - ) + 'Parameter \'env\' has been detected in the argument list. ' + 'This parameter is no longer used and has been replaced by ' + '\'saltenv\' as of Salt Carbon. This warning will be removed ' + 'in Salt Oxygen.' + ) load.pop('env') if 'path' not in load or 'saltenv' not in load: - return '' + return '', None if not isinstance(load['saltenv'], six.string_types): load['saltenv'] = six.text_type(load['saltenv']) fnd = self.find_file(salt.utils.locales.sdecode(load['path']), load['saltenv']) if not fnd.get('back'): - return '' + return '', None + stat_result = fnd.get('stat', None) fstr = '{0}.file_hash'.format(fnd['back']) if fstr in self.servers: - return self.servers[fstr](load, fnd) - return '' + return self.servers[fstr](load, fnd), stat_result + return '', None + + def file_hash(self, load): + ''' + Return the hash of a given file + ''' + try: + return self.__file_hash_and_stat(load)[0] + except (IndexError, TypeError): + return '' + + def file_hash_and_stat(self, load): + ''' + Return the hash and stat result of a given file + ''' + try: + return self.__file_hash_and_stat(load) + except (IndexError, TypeError): + return '', None def file_list(self, load): ''' From 73a156d28cafd45dcc6c2aac9f1ac3ad905695e3 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 20 Jul 2016 01:51:59 -0500 Subject: [PATCH 62/95] Support enforcement of file mode in RemoteClient and LocalClient This code modifies the file caching code in both fileclients to request the stat result for the desired file. If the locally-cached copy of the file has a mode which does not match the mode retrieved from the fileclient, then the cached copy's mode is chmod'ed to match. --- salt/fileclient.py | 232 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 195 insertions(+), 37 deletions(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index 5972a10072..d2d2d084c7 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -7,7 +7,6 @@ from __future__ import absolute_import # Import python libs import contextlib import logging -import hashlib import os import shutil import ftplib @@ -749,9 +748,27 @@ class LocalClient(Client): ''' path = self._check_proto(path) fnd = self._find_file(path, saltenv) - if not fnd['path']: + fnd_path = fnd.get('path') + if not fnd_path: return '' - return fnd['path'] + + try: + fnd_mode = fnd.get('stat', [])[0] + except (IndexError, TypeError): + fnd_mode = None + + if not salt.utils.is_windows(): + if fnd_mode is not None: + try: + if os.stat(dest).st_mode != fnd_mode: + try: + os.chmod(dest, fnd_mode) + except OSError as exc: + log.warning('Failed to chmod %s: %s', dest, exc) + except Exception: + pass + + return fnd_path def file_list(self, saltenv='base', prefix=''): ''' @@ -804,6 +821,22 @@ class LocalClient(Client): ret.append(sdecode(os.path.relpath(root, path))) return ret + def __get_file_path(self, path, saltenv='base'): + ''' + Return either a file path or the result of a remote find_file call. + ''' + try: + path = self._check_proto(path) + except MinionError as err: + # Local file path + if not os.path.isfile(path): + msg = 'specified file {0} is not present to generate hash: {1}' + log.warning(msg.format(path, err)) + return None + else: + return path + return self._find_file(path, saltenv) + def hash_file(self, path, saltenv='base'): ''' Return the hash of a file, to get the hash of a file in the file_roots @@ -811,26 +844,51 @@ class LocalClient(Client): file with / for a local file. ''' ret = {} + fnd = self.__get_file_path(path, saltenv) + if fnd is None: + return ret + try: - path = self._check_proto(path) - except MinionError as err: - if not os.path.isfile(path): - msg = 'specified file {0} is not present to generate hash: {1}' - log.warning(msg.format(path, err)) - return ret - else: - opts_hash_type = self.opts.get('hash_type', 'md5') - hash_type = getattr(hashlib, opts_hash_type) - ret['hsum'] = salt.utils.get_hash( - path, form=hash_type) - ret['hash_type'] = opts_hash_type - return ret - path = self._find_file(path, saltenv)['path'] - if not path: - return {} + # Remote file path (self._find_file() invoked) + fnd_path = fnd['path'] + except TypeError: + # Local file path + fnd_path = fnd + + hash_type = self.opts.get('hash_type', 'md5') + ret['hsum'] = salt.utils.get_hash(fnd_path, form=hash_type) + ret['hash_type'] = hash_type + return ret + + def hash_and_stat_file(self, path, saltenv='base'): + ''' + Return the hash of a file, to get the hash of a file in the file_roots + prepend the path with salt:// otherwise, prepend the + file with / for a local file. + + Additionally, return the stat result of the file, or None if no stat + results were found. + ''' ret = {} - ret['hsum'] = salt.utils.get_hash(path, self.opts['hash_type']) - ret['hash_type'] = self.opts['hash_type'] + fnd = self.__get_file_path(path, saltenv) + if fnd is None: + return ret, None + + try: + # Remote file path (self._find_file() invoked) + fnd_path = fnd['path'] + fnd_stat = fnd.get('stat') + except TypeError: + # Local file path + fnd_path = fnd + try: + fnd_stat = list(os.stat(fnd_path)) + except Exception: + fnd_stat = None + + hash_type = self.opts.get('hash_type', 'md5') + ret['hsum'] = salt.utils.get_hash(fnd_path, form=hash_type) + ret['hash_type'] = hash_type return ret def list_env(self, saltenv='base'): @@ -906,13 +964,22 @@ class RemoteClient(Client): if senv: saltenv = senv + if not salt.utils.is_windows(): + hash_server, stat_server = self.hash_and_stat_file(path, saltenv) + try: + mode_server = stat_server[0] + except (IndexError, TypeError): + mode_server = None + else: + hash_server = self.hash_file(path, saltenv) + mode_server = None + # Check if file exists on server, before creating files and # directories - hash_server = self.hash_file(path, saltenv) if hash_server == '': log.debug( - 'Could not find file from saltenv \'%s\', \'%s\'', - saltenv, path + 'Could not find file \'%s\' in saltenv \'%s\'', + path, saltenv ) return False @@ -936,13 +1003,59 @@ class RemoteClient(Client): ) if dest2check and os.path.isfile(dest2check): - hash_local = self.hash_file(dest2check, saltenv) + if not salt.utils.is_windows(): + hash_local, stat_local = \ + self.hash_and_stat_file(dest2check, saltenv) + try: + mode_local = stat_local[0] + except IndexError: + mode_local = None + else: + hash_local = self.hash_file(dest2check, saltenv) + mode_local = None + if hash_local == hash_server: - log.info( - 'Fetching file from saltenv \'%s\', ** skipped ** ' - 'latest already in cache \'%s\'', saltenv, path - ) - return dest2check + if not salt.utils.is_windows(): + if mode_server is None: + log.debug('No file mode available for \'%s\'', path) + elif mode_local is None: + log.debug( + 'No file mode available for \'%s\'', + dest2check + ) + else: + if mode_server == mode_local: + log.info( + 'Fetching file from saltenv \'%s\', ' + '** skipped ** latest already in cache ' + '\'%s\', mode up-to-date', saltenv, path + ) + else: + try: + os.chmod(dest2check, mode_server) + log.info( + 'Fetching file from saltenv \'%s\', ' + '** updated ** latest already in cache, ' + '\'%s\', mode updated from %s to %s', + saltenv, + path, + salt.utils.st_mode_to_octal(mode_local), + salt.utils.st_mode_to_octal(mode_server) + ) + except OSError as exc: + log.warning( + 'Failed to chmod %s: %s', dest2check, exc + ) + # We may not have been able to check/set the mode, but we + # don't want to re-download the file because of a failure + # in mode checking. Return the cached path. + return dest2check + else: + log.info( + 'Fetching file from saltenv \'%s\', ** skipped ** ' + 'latest already in cache \'%s\'', saltenv, path + ) + return dest2check log.debug( 'Fetching file from saltenv \'%s\', ** attempting ** \'%s\'', @@ -968,7 +1081,7 @@ class RemoteClient(Client): return False fn_ = salt.utils.fopen(dest, 'wb+') else: - log.debug('Dest file \'%s\' not found', dest) + log.debug('No dest file found') while True: if not fn_: @@ -1055,6 +1168,23 @@ class RemoteClient(Client): saltenv, path ) + if not salt.utils.is_windows(): + if mode_server is not None: + try: + if os.stat(dest).st_mode != mode_server: + try: + os.chmod(dest, mode_server) + log.info( + 'Fetching file from saltenv \'%s\', ' + '** done ** \'%s\', mode set to %s', + saltenv, + path, + salt.utils.st_mode_to_octal(mode_server) + ) + except OSError: + log.warning('Failed to chmod %s: %s', dest, exc) + except OSError: + pass return dest def file_list(self, saltenv='base', prefix=''): @@ -1094,11 +1224,9 @@ class RemoteClient(Client): 'cmd': '_symlink_list'} return self.channel.send(load) - def hash_file(self, path, saltenv='base'): + def __hash_and_stat_file(self, path, saltenv='base'): ''' - Return the hash of a file, to get the hash of a file on the salt - master file server prepend the path with salt:// - otherwise, prepend the file with / for a local file. + Common code for hashing and stating files ''' try: path = self._check_proto(path) @@ -1110,8 +1238,7 @@ class RemoteClient(Client): else: ret = {} hash_type = self.opts.get('hash_type', 'md5') - ret['hsum'] = salt.utils.get_hash( - path, form=hash_type) + ret['hsum'] = salt.utils.get_hash(path, form=hash_type) ret['hash_type'] = hash_type return ret load = {'path': path, @@ -1119,6 +1246,37 @@ class RemoteClient(Client): 'cmd': '_file_hash'} return self.channel.send(load) + def hash_file(self, path, saltenv='base'): + ''' + Return the hash of a file, to get the hash of a file on the salt + master file server prepend the path with salt:// + otherwise, prepend the file with / for a local file. + ''' + return self.__hash_and_stat_file(path, saltenv) + + def hash_and_stat_file(self, path, saltenv='base'): + ''' + The same as hash_file, but also return the file's mode, or None if no + mode data is present. + ''' + hash_result = self.hash_file(path, saltenv) + try: + path = self._check_proto(path) + except MinionError as err: + if not os.path.isfile(path): + return hash_result, None + else: + try: + return hash_result, list(os.stat(path)) + except Exception: + return hash_result, None + load = {'path': path, + 'saltenv': saltenv, + 'cmd': '_file_find'} + fnd = self.channel.send(load) + stat_result = fnd.get('stat') + return hash_result, stat_result + def list_env(self, saltenv='base'): ''' Return a list of the files in the file server's specified environment From 480a4a172ecc2bc966336bda496a31e74eceff19 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 20 Jul 2016 02:03:32 -0500 Subject: [PATCH 63/95] Add attributes to master's fileserver objects to point to new convenience function --- salt/daemons/masterapi.py | 1 + salt/master.py | 1 + 2 files changed, 2 insertions(+) diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py index f9c8e86588..68e226e7e1 100644 --- a/salt/daemons/masterapi.py +++ b/salt/daemons/masterapi.py @@ -451,6 +451,7 @@ class RemoteFuncs(object): ''' fs_ = salt.fileserver.Fileserver(self.opts) self._serve_file = fs_.serve_file + self._file_find = fs_._find_file self._file_hash = fs_.file_hash self._file_list = fs_.file_list self._file_list_emptydirs = fs_.file_list_emptydirs diff --git a/salt/master.py b/salt/master.py index 71dcec345e..2319b90977 100644 --- a/salt/master.py +++ b/salt/master.py @@ -936,6 +936,7 @@ class AESFuncs(object): ''' self.fs_ = salt.fileserver.Fileserver(self.opts) self._serve_file = self.fs_.serve_file + self._file_find = self.fs_._find_file self._file_hash = self.fs_.file_hash self._file_list = self.fs_.file_list self._file_list_emptydirs = self.fs_.file_list_emptydirs From 1c20949683640b2f46e6a4d6759b94e9db061864 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 20 Jul 2016 02:04:55 -0500 Subject: [PATCH 64/95] Remove code that was moved to salt.utils --- salt/client/ssh/wrapper/config.py | 10 ++++------ salt/modules/config.py | 15 ++++----------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/salt/client/ssh/wrapper/config.py b/salt/client/ssh/wrapper/config.py index 87681f01ef..72491816a9 100644 --- a/salt/client/ssh/wrapper/config.py +++ b/salt/client/ssh/wrapper/config.py @@ -78,12 +78,10 @@ def manage_mode(mode): salt '*' config.manage_mode ''' - if mode is None: - return None - ret = str(mode).lstrip('0').zfill(4) - if ret[0] != '0': - return '0{0}'.format(ret) - return ret + # config.manage_mode should no longer be invoked from the __salt__ dunder + # in Salt code, this function is only being left here for backwards + # compatibility. + return salt.utils.normalize_mode(mode) def valid_fileproto(uri): diff --git a/salt/modules/config.py b/salt/modules/config.py index bc7468edaa..48337f3b6c 100644 --- a/salt/modules/config.py +++ b/salt/modules/config.py @@ -97,17 +97,10 @@ def manage_mode(mode): salt '*' config.manage_mode ''' - if mode is None: - return None - if not isinstance(mode, six.string_types): - # Make it a string in case it's not - mode = str(mode) - # Strip any quotes and initial 0, though zero-pad it up to 4 - ret = mode.strip('"').strip('\'').lstrip('0').zfill(4) - if ret[0] != '0': - # Always include a leading zero - return '0{0}'.format(ret) - return ret + # config.manage_mode should no longer be invoked from the __salt__ dunder + # in Salt code, this function is only being left here for backwards + # compatibility. + return salt.utils.normalize_mode(mode) def valid_fileproto(uri): From a676cdff75e104af0bac6f989488a57d9ca333cf Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 20 Jul 2016 02:07:10 -0500 Subject: [PATCH 65/95] Update ref to config.manage_mode --- salt/states/cron.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/cron.py b/salt/states/cron.py index 256e5da748..6e32a7081d 100644 --- a/salt/states/cron.py +++ b/salt/states/cron.py @@ -469,7 +469,7 @@ def file(name, Overrides the default backup mode for the user's crontab. ''' # Initial set up - mode = __salt__['config.manage_mode']('0600') + mode = salt.utils.normalize_mode('0600') owner, group, crontab_dir = _get_cron_info() cron_path = salt.utils.mkstemp() From 44d60a3252cbeefa1defd102cd0c452776aea400 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 20 Jul 2016 02:07:56 -0500 Subject: [PATCH 66/95] Allow file.managed, file.recurse states to keep server's mode for files --- salt/modules/file.py | 41 ++++++++++++---- salt/states/file.py | 114 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 121 insertions(+), 34 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 8a4bc91c48..5f1a34847f 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1149,7 +1149,7 @@ def comment_line(path, if not salt.utils.is_windows(): pre_user = get_user(path) pre_group = get_group(path) - pre_mode = __salt__['config.manage_mode'](get_mode(path)) + pre_mode = salt.utils.normalize_mode(get_mode(path)) # Create a copy to read from and to use as a backup later try: @@ -1827,7 +1827,7 @@ def replace(path, if not salt.utils.is_windows(): pre_user = get_user(path) pre_group = get_group(path) - pre_mode = __salt__['config.manage_mode'](get_mode(path)) + pre_mode = salt.utils.normalize_mode(get_mode(path)) # Avoid TypeErrors by forcing repl to be a string repl = str(repl) @@ -2195,7 +2195,7 @@ def blockreplace(path, perms = {} perms['user'] = get_user(path) perms['group'] = get_group(path) - perms['mode'] = __salt__['config.manage_mode'](get_mode(path)) + perms['mode'] = salt.utils.normalize_mode(get_mode(path)) # write new content in the file while avoiding partial reads try: @@ -2899,7 +2899,7 @@ def copy(src, dst, recurse=False, remove_existing=False): if not salt.utils.is_windows(): pre_user = get_user(src) pre_group = get_group(src) - pre_mode = __salt__['config.manage_mode'](get_mode(src)) + pre_mode = salt.utils.normalize_mode(get_mode(src)) try: if (os.path.exists(dst) and os.path.isdir(dst)) or os.path.isdir(src): @@ -3734,7 +3734,7 @@ def check_perms(name, ret, user, group, mode, follow_symlinks=False): raise CommandExecutionError('{0} does not exist'.format(name)) perms['luser'] = cur['user'] perms['lgroup'] = cur['group'] - perms['lmode'] = __salt__['config.manage_mode'](cur['mode']) + perms['lmode'] = salt.utils.normalize_mode(cur['mode']) # Mode changes if needed if mode is not None: @@ -3743,13 +3743,13 @@ def check_perms(name, ret, user, group, mode, follow_symlinks=False): if os.path.islink(name) and not follow_symlinks: pass else: - mode = __salt__['config.manage_mode'](mode) + mode = salt.utils.normalize_mode(mode) if mode != perms['lmode']: if __opts__['test'] is True: ret['changes']['mode'] = mode else: set_mode(name, mode) - if mode != __salt__['config.manage_mode'](get_mode(name)): + if mode != salt.utils.normalize_mode(get_mode(name)): ret['result'] = False ret['comment'].append( 'Failed to change mode to {0}'.format(mode) @@ -3920,6 +3920,7 @@ def check_managed_changes( saltenv, contents=None, skip_verify=False, + keep_mode=False, **kwargs): ''' Return a dictionary of what changes need to be made for a file @@ -3956,6 +3957,13 @@ def check_managed_changes( if comments: __clean_tmp(sfn) return False, comments + if sfn and source and keep_mode: + if _urlparse(source).scheme in ('salt', 'file') \ + or source.startswith('/'): + try: + mode = salt.utils.st_mode_to_octal(os.stat(sfn).st_mode) + except Exception as exc: + log.warning('Unable to stat %s: %s', sfn, exc) changes = check_file_meta(name, sfn, source, source_sum, user, group, mode, saltenv, contents) __clean_tmp(sfn) @@ -4078,8 +4086,8 @@ def check_file_meta( and group != lstats['gid']): changes['group'] = group # Normalize the file mode - smode = __salt__['config.manage_mode'](lstats['mode']) - mode = __salt__['config.manage_mode'](mode) + smode = salt.utils.normalize_mode(lstats['mode']) + mode = salt.utils.normalize_mode(mode) if mode is not None and mode != smode: changes['mode'] = mode return changes @@ -4142,7 +4150,8 @@ def manage_file(name, contents=None, dir_mode=None, follow_symlinks=True, - skip_verify=False): + skip_verify=False, + keep_mode=False): ''' Checks the destination against what was retrieved with get_managed and makes the appropriate modifications (if necessary). @@ -4203,6 +4212,11 @@ def manage_file(name, .. versionadded:: 2016.3.0 + keep_mode : False + If ``True``, and the ``source`` is a file from the Salt fileserver (or + a local file on the minion), the mode of the destination file will be + set to the mode of the source file. + CLI Example: .. code-block:: bash @@ -4233,6 +4247,13 @@ def manage_file(name, 'hash_type': htype, 'hsum': get_hash(sfn, form=htype) } + if keep_mode: + if _urlparse(source).scheme in ('salt', 'file') \ + or source.startswith('/'): + try: + mode = salt.utils.st_mode_to_octal(os.stat(sfn).st_mode) + except Exception as exc: + log.warning('Unable to stat %s: %s', sfn, exc) # Check changes if the target file exists if os.path.isfile(name) or os.path.islink(name): diff --git a/salt/states/file.py b/salt/states/file.py index b6d4e0c692..cf3117df86 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -575,8 +575,8 @@ def _check_dir_meta(name, and group != stats.get('gid')): changes['group'] = group # Normalize the dir mode - smode = __salt__['config.manage_mode'](stats['mode']) - mode = __salt__['config.manage_mode'](mode) + smode = salt.utils.normalize_mode(stats['mode']) + mode = salt.utils.normalize_mode(mode) if mode is not None and mode != smode: changes['mode'] = mode return changes @@ -839,7 +839,7 @@ def symlink( name = os.path.expanduser(name) # Make sure that leading zeros stripped by YAML loader are added back - mode = __salt__['config.manage_mode'](mode) + mode = salt.utils.normalize_mode(mode) user = _test_owner(kwargs, user=user) ret = {'name': name, @@ -1254,8 +1254,18 @@ def managed(name, is running as on the minion On Windows, this is ignored mode - The permissions to set on this file, aka 644, 0775, 4664. Not supported - on Windows + The mode to set on this file, e.g. ``644``, ``0775``, or ``4664``. + + .. note:: + This option is **not** supported on Windows. + + .. versionchanged:: Carbon + This option can be set to ``keep``, and Salt will keep the mode + from the Salt fileserver. This is only supported when the + ``source`` URL begins with ``salt://``, or for files local to the + minion. Because the ``source`` option cannot be used with any of + the ``contents`` options, setting the ``mode`` to ``keep`` is also + incompatible with the ``contents`` options. template If this setting is applied then the named templating engine will be @@ -1270,7 +1280,8 @@ def managed(name, dir_mode If directories are to be created, passing this option specifies the permissions for those directories. If this is not set, directories - will be assigned permissions from the 'mode' argument. + will be assigned permissions by adding the execute bit to the mode of + the files. replace : True If set to ``False`` and the file already exists, the file will not be @@ -1478,9 +1489,23 @@ def managed(name, 'name': name, 'result': True} - content_sources = (contents, contents_pillar, contents_grains) + if mode is not None and salt.utils.is_windows(): + return _error(ret, 'The \'mode\' option is not supported on Windows') + + try: + keep_mode = mode.lower() == 'keep' + if keep_mode: + # We're not hard-coding the mode, so set it to None + mode = None + except AttributeError: + keep_mode = False + + # Make sure that any leading zeros stripped by YAML loader are added back + mode = salt.utils.normalize_mode(mode) + contents_count = len( - [x for x in content_sources if x is not None] + [x for x in (contents, contents_pillar, contents_grains) + if x is not None] ) if source and contents_count > 0: @@ -1489,6 +1514,12 @@ def managed(name, '\'source\' cannot be used in combination with \'contents\', ' '\'contents_pillar\', or \'contents_grains\'' ) + elif (mode or keep_mode) and contents_count > 0: + return _error( + ret, + 'Mode management cannot be used in combination with \'contents\', ' + '\'contents_pillar\', or \'contents_grains\'' + ) elif contents_count > 1: return _error( ret, @@ -1608,9 +1639,6 @@ def managed(name, ret['comment'] = 'Error while applying template on contents' return ret - # Make sure that leading zeros stripped by YAML loader are added back - mode = __salt__['config.manage_mode'](mode) - if not name: return _error(ret, 'Must provide name to file.exists') user = _test_owner(kwargs, user=user) @@ -1679,6 +1707,7 @@ def managed(name, __env__, contents, skip_verify, + keep_mode, **kwargs ) else: @@ -1766,7 +1795,8 @@ def managed(name, contents, dir_mode, follow_symlinks, - skip_verify) + skip_verify, + keep_mode) except Exception as exc: ret['changes'] = {} log.debug(traceback.format_exc()) @@ -1823,7 +1853,8 @@ def managed(name, contents, dir_mode, follow_symlinks, - skip_verify) + skip_verify, + keep_mode) except Exception as exc: ret['changes'] = {} log.debug(traceback.format_exc()) @@ -2044,8 +2075,8 @@ def directory(name, file_mode = dir_mode # Make sure that leading zeros stripped by YAML loader are added back - dir_mode = __salt__['config.manage_mode'](dir_mode) - file_mode = __salt__['config.manage_mode'](file_mode) + dir_mode = salt.utils.normalize_mode(dir_mode) + file_mode = salt.utils.normalize_mode(file_mode) u_check = _check_user(user, group) if u_check: @@ -2291,16 +2322,31 @@ def recurse(name, salt is running as on the minion. On Windows, this is ignored dir_mode - The permissions mode to set on any directories created. Not supported on - Windows + The mode to set on any directories created. + + .. note:: + This option is **not** supported on Windows. file_mode - The permissions mode to set on any files created. Not supported on + The mode to set on any files created. Windows + .. note:: + This option is **not** supported on Windows. + + .. versionchanged:: Carbon + This option can be set to ``keep``, and Salt will keep the mode + from the Salt fileserver. This is only supported when the + ``source`` URL begins with ``salt://``, or for files local to the + minion. Because the ``source`` option cannot be used with any of + the ``contents`` options, setting the ``mode`` to ``keep`` is also + incompatible with the ``contents`` options. + sym_mode - The permissions mode to set on any symlink created. Not supported on - Windows + The mode to set on any symlink created. + + .. note:: + This option is **not** supported on Windows. template If this setting is applied then the named templating engine will be @@ -2408,9 +2454,22 @@ def recurse(name, ) return ret + if any([x is not None for x in (dir_mode, file_mode, sym_mode)]) \ + and salt.utils.is_windows(): + return _error(ret, 'mode management is not supported on Windows') + # Make sure that leading zeros stripped by YAML loader are added back - dir_mode = __salt__['config.manage_mode'](dir_mode) - file_mode = __salt__['config.manage_mode'](file_mode) + dir_mode = salt.utils.normalize_mode(dir_mode) + + try: + keep_mode = file_mode.lower() == 'keep' + if keep_mode: + # We're not hard-coding the mode, so set it to None + file_mode = None + except AttributeError: + keep_mode = False + + file_mode = salt.utils.normalize_mode(file_mode) u_check = _check_user(user, group) if u_check: @@ -2509,7 +2568,7 @@ def recurse(name, source=source, user=user, group=group, - mode=file_mode, + mode='keep' if keep_mode else file_mode, template=template, makedirs=True, context=context, @@ -4777,7 +4836,11 @@ def serialize(name, salt is running as on the minion mode - The permissions to set on this file, aka 644, 0775, 4664 + The permissions to set on this file, e.g. ``644``, ``0775``, or + ``4664``. + + .. note:: + This option is **not** supported on Windows. backup Overrides the default backup mode for this specific file. @@ -4914,6 +4977,9 @@ def serialize(name, contents += '\n' + # Make sure that any leading zeros stripped by YAML loader are added back + mode = salt.utils.normalize_mode(mode) + if __opts__['test']: ret['changes'] = __salt__['file.check_managed_changes']( name=name, From 087500bde7122d66936e99f45506a356646f0e47 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 20 Jul 2016 02:14:24 -0500 Subject: [PATCH 67/95] Add new file mode preservation feature to carbon release notes --- doc/topics/releases/carbon.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/topics/releases/carbon.rst b/doc/topics/releases/carbon.rst index b2f28232b3..6cb3daf30f 100644 --- a/doc/topics/releases/carbon.rst +++ b/doc/topics/releases/carbon.rst @@ -20,6 +20,13 @@ Features SSL can be enabled by setting ``ssl_options`` for the returner. Also added support for specifying ``protocol_version`` when establishing cluster connection. +- The ``mode`` parameter in the :py:mod:`file.managed + ` state, and the ``file_mode`` parameter in the + :py:mod:`file.managed `, can both now be set to + ``keep`` and the minion will keep the mode of the file from the Salt + fileserver. This works only with files coming from sources prefixed with + ``salt://``, or files local to the minion (i.e. those which are absolute + paths, or are prefixed with ``file://``). Config Changes ============== From e9be937d8f21ba9fcdc06b93af9c66017604e558 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 20 Jul 2016 12:59:59 -0500 Subject: [PATCH 68/95] Fix incorrectly-written test This was testing to confirm incorrect behavior. We should not be having a leading zero when the setuid/setgid bit is set in the mode. --- tests/integration/modules/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/modules/config.py b/tests/integration/modules/config.py index a876640bad..8e04a881f0 100644 --- a/tests/integration/modules/config.py +++ b/tests/integration/modules/config.py @@ -50,17 +50,17 @@ class ConfigTest(integration.ModuleCase): self.assertEqual( self.run_function('config.manage_mode', ['"775"']), '0775') self.assertEqual( - self.run_function('config.manage_mode', ['"1775"']), '01775') + self.run_function('config.manage_mode', ['"1775"']), '1775') self.assertEqual( self.run_function('config.manage_mode', ['"0775"']), '0775') self.assertEqual( - self.run_function('config.manage_mode', ['"01775"']), '01775') + self.run_function('config.manage_mode', ['"01775"']), '1775') self.assertEqual( self.run_function('config.manage_mode', ['"0"']), '0000') self.assertEqual( self.run_function('config.manage_mode', ['775']), '0775') self.assertEqual( - self.run_function('config.manage_mode', ['1775']), '01775') + self.run_function('config.manage_mode', ['1775']), '1775') self.assertEqual( self.run_function('config.manage_mode', ['0']), '0000') From 8f297231604ea72794049c2e170de471e1921831 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 20 Jul 2016 14:34:58 -0700 Subject: [PATCH 69/95] Fix duplicate import, circular reference --- tests/integration/__init__.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index dd34f82b93..af72bbce5b 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -30,11 +30,6 @@ try: except ImportError: pass -try: - import salt.ext.six.moves.socketserver as socketserver -except ImportError: - import socketserver - STATE_FUNCTION_RUNNING_RE = re.compile( r'''The function (?:"|')(?P.*)(?:"|') is running as PID ''' r'(?P[\d]+) and was started at (?P.*) with jid (?P[\d]+)' @@ -67,7 +62,6 @@ import salt.utils.process import salt.log.setup as salt_log_setup from salt.utils.verify import verify_env from salt.utils.immutabletypes import freeze -from salt.utils.process import SignalHandlingMultiprocessingProcess from salt.utils.nb_popen import NonBlockingPopen from salt.exceptions import SaltClientError @@ -83,7 +77,7 @@ import msgpack import salt.ext.six as six try: - import salt.ext.six.moves.socketserver as socketserver # pylint: disable=name-in-module + import salt.ext.six.moves.socketserver as socketserver except ImportError: import socketserver @@ -410,7 +404,7 @@ class SaltDaemonScriptBase(SaltScriptBase, ShellTestCase): ''' Start the daemon subprocess ''' - self._process = SignalHandlingMultiprocessingProcess( + self._process = salt.utils.process.SignalHandlingMultiprocessingProcess( target=self._start, args=(self._running,)) self._process.start() self._running.set() From 7c402ea3d4d70802424fe1dac121075733867c10 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 20 Jul 2016 12:18:37 -0500 Subject: [PATCH 70/95] [PY3] git pillar test --- salt/pillar/git_pillar.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/pillar/git_pillar.py b/salt/pillar/git_pillar.py index c5a4226e4f..b515719215 100644 --- a/salt/pillar/git_pillar.py +++ b/salt/pillar/git_pillar.py @@ -205,6 +205,7 @@ import hashlib import os # Import salt libs +import salt.utils import salt.utils.gitfs import salt.utils.dictupdate from salt.exceptions import FileserverConfigError @@ -332,7 +333,7 @@ class _LegacyGitPillar(object): hash_type = getattr(hashlib, opts.get('hash_type', 'md5')) hash_str = '{0} {1}'.format(self.branch, self.rp_location) - repo_hash = hash_type(hash_str).hexdigest() + repo_hash = hash_type(salt.utils.to_bytes(hash_str)).hexdigest() rp_ = os.path.join(self.opts['cachedir'], 'pillar_gitfs', repo_hash) if not os.path.isdir(rp_): From 1e3429a3a22af7d5ef82b82a8a8e7873ccd7fcbe Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 20 Jul 2016 12:41:16 -0500 Subject: [PATCH 71/95] [PY3] skip mysql tests mysql-python is not compatible with python3 yet. --- tests/unit/pillar/mysql_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/pillar/mysql_test.py b/tests/unit/pillar/mysql_test.py index e9ef6e17e3..46c62e83c8 100644 --- a/tests/unit/pillar/mysql_test.py +++ b/tests/unit/pillar/mysql_test.py @@ -12,9 +12,11 @@ ensure_in_syspath('../../') # Import Salt Libs from salt.pillar import mysql +from salt.ext.six import PY3 @skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(PY3, 'MySQL-python is not compatible with python3') class MysqlPillarTestCase(TestCase): maxDiff = None From d2341654ead0411d9a29eb7a2f46aed1b7340305 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 20 Jul 2016 13:54:50 -0500 Subject: [PATCH 72/95] [PY3] actually test with nodegroup_comp function --- tests/unit/pillar/nodegroups_test.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/tests/unit/pillar/nodegroups_test.py b/tests/unit/pillar/nodegroups_test.py index 86bc0e84fa..45f69de330 100644 --- a/tests/unit/pillar/nodegroups_test.py +++ b/tests/unit/pillar/nodegroups_test.py @@ -4,8 +4,7 @@ from __future__ import absolute_import # Import Salt Testing libs -from salttesting import TestCase, skipIf -from salttesting.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, call, patch +from salttesting import TestCase from salttesting.helpers import ensure_in_syspath ensure_in_syspath('../../') @@ -16,31 +15,24 @@ from salt.pillar import nodegroups fake_minion_id = 'fake_id' fake_pillar = {} fake_nodegroups = { - 'a': 'nodegroup_a', + 'a': fake_minion_id, 'b': 'nodegroup_b', } -fake_opts = {'nodegroups': fake_nodegroups, } +fake_opts = {'nodegroups': fake_nodegroups, 'id': fake_minion_id} fake_pillar_name = 'fake_pillar_name' nodegroups.__opts__ = fake_opts -@skipIf(NO_MOCK, NO_MOCK_REASON) class NodegroupsPillarTestCase(TestCase): ''' Tests for salt.pillar.nodegroups ''' - def _runner(self, expected_ret, pillar_name=None, nodegroup_matches=None): + def _runner(self, expected_ret, pillar_name=None): pillar_name = pillar_name or fake_pillar_name - nodegroup_matches = nodegroup_matches or [True, False, ] - mock_nodegroup_match = MagicMock(side_effect=nodegroup_matches) - with patch.object(nodegroups.Matcher, 'nodegroup_match', mock_nodegroup_match): - actual_ret = nodegroups.ext_pillar(fake_minion_id, fake_pillar, pillar_name=pillar_name) + actual_ret = nodegroups.ext_pillar(fake_minion_id, fake_pillar, pillar_name=pillar_name) self.assertDictEqual(actual_ret, expected_ret) - fake_nodegroup_count = len(fake_nodegroups) - self.assertEqual(mock_nodegroup_match.call_count, fake_nodegroup_count) - mock_nodegroup_match.assert_has_calls([call(x, fake_nodegroups) for x in fake_nodegroups.keys()]) def test_succeeds(self): ret = {fake_pillar_name: ['a', ]} From ee90b443095c5b7c5b3b5efd157a7dbec28babda Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 20 Jul 2016 14:27:06 -0500 Subject: [PATCH 73/95] [PY3] add note to sqlcipher for py3 support --- salt/pillar/sqlcipher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/pillar/sqlcipher.py b/salt/pillar/sqlcipher.py index ac1d6cac06..b984235704 100644 --- a/salt/pillar/sqlcipher.py +++ b/salt/pillar/sqlcipher.py @@ -12,7 +12,7 @@ This module is a concrete implementation of the sql_base ext_pillar for SQLCipher. :maturity: new -:depends: pysqlcipher +:depends: pysqlcipher (for py2) or pysqlcipher3 (for py3) :platform: all Configuring the sqlcipher ext_pillar From a14ac8fb2f10124a4978db19049bdf932e91c49d Mon Sep 17 00:00:00 2001 From: Alejandro del Castillo Date: Wed, 29 Jun 2016 16:44:24 +0000 Subject: [PATCH 74/95] Add avahi based beacon for zeroconf announcement Signed-off-by: Alejandro del Castillo --- salt/beacons/avahi_announce.py | 117 +++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 salt/beacons/avahi_announce.py diff --git a/salt/beacons/avahi_announce.py b/salt/beacons/avahi_announce.py new file mode 100644 index 0000000000..db44dc7d95 --- /dev/null +++ b/salt/beacons/avahi_announce.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +''' + Beacon to announce via avahi (zeroconf) + +''' +# Import Python libs +from __future__ import absolute_import +import logging + +# Import 3rd Party libs +try: + import avahi + HAS_PYAVAHI = True +except ImportError: + HAS_PYAVAHI = False +import dbus + +log = logging.getLogger(__name__) + +__virtualname__ = 'avahi_announce' + +LAST_GRAINS = {} +BUS = dbus.SystemBus() +SERVER = dbus.Interface(BUS.get_object(avahi.DBUS_NAME, avahi.DBUS_PATH_SERVER), + avahi.DBUS_INTERFACE_SERVER) +GROUP = dbus.Interface(BUS.get_object(avahi.DBUS_NAME, SERVER.EntryGroupNew()), + avahi.DBUS_INTERFACE_ENTRY_GROUP) + + +def __virtual__(): + if HAS_PYAVAHI: + return __virtualname__ + return False + + +def validate(config): + ''' + Validate the beacon configuration + ''' + if not isinstance(config, dict): + return False, ('Configuration for avahi_announcement ' + 'beacon must be a dictionary') + elif not all(x in list(config.keys()) for x in ('servicetype', 'port', 'txt')): + return False, ('Configuration for avahi_announce beacon ' + 'must contain servicetype, port and txt items') + return True, 'Valid beacon configuration' + + +def beacon(config): + ''' + Broadcast values via zeroconf + + If the announced values are static, it is adviced to set run_once: True + (do not poll) on the beacon configuration. Grains can be used to define + txt values using the syntax: grains. + + The default servicename its the hostname grain value. + + Example Config + + .. code-block:: yaml + + beacons: + avahi_announce: + run_once: True + servicetype: _demo._tcp + txt: + ProdName: grains.productname + SerialNo: grains.serialnumber + Comments: 'this is a test' + ''' + ret = [] + changes = {} + txt = {} + + global LAST_GRAINS + + _validate = validate(config) + if not _validate[0]: + log.warning('Beacon {0} configuration invalid, ' + 'not adding. {1}'.format(__virtualname__, _validate[1])) + return ret + + if 'servicename' in config: + servicename = config['servicename'] + else: + servicename = __grains__['host'] + + for item in config['txt']: + if config['txt'][item].startswith('grains.'): + grain = config['txt'][item][7:] + txt[item] = __grains__[grain] + if LAST_GRAINS and (LAST_GRAINS[grain] != __grains__[grain]): + changes[str('txt.' + item)] = txt[item] + else: + txt[item] = config['txt'][item] + + if not LAST_GRAINS: + changes[str('txt.' + item)] = txt[item] + + if changes: + if not LAST_GRAINS: + changes['servicename'] = servicename + changes['servicetype'] = config['servicetype'] + changes['port'] = config['port'] + else: + GROUP.Reset() + GROUP.AddService(avahi.IF_UNSPEC, avahi.PROTO_UNSPEC, dbus.UInt32(0), + servicename, config['servicetype'], '', '', + dbus.UInt16(config['port']), avahi.dict_to_txt_array(txt)) + GROUP.Commit() + + ret.append({'tag': 'result', 'changes': changes}) + + LAST_GRAINS = __grains__ + + return ret From 1cd542edb8c76fc29afc839b1269b55103eb5a03 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 20 Jul 2016 22:08:16 -0500 Subject: [PATCH 75/95] Add integration test for mode=keep in file states --- tests/integration/states/file.py | 59 ++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/integration/states/file.py b/tests/integration/states/file.py index e7640a9d0b..830f378ba9 100644 --- a/tests/integration/states/file.py +++ b/tests/integration/states/file.py @@ -165,6 +165,65 @@ class FileTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): self.assertEqual(oct(desired_mode), oct(resulting_mode)) self.assertSaltTrueReturn(ret) + def test_managed_file_mode_keep(self): + ''' + Test using "mode: keep" in a file.managed state + ''' + rel_path = 'grail/scene33' + name = os.path.join(integration.TMP, os.path.basename(rel_path)) + grail = 'salt://' + rel_path + grail_fs_path = os.path.join(integration.FILES, 'file', 'base', rel_path) + + # Get the current mode so that we can put the file back the way we + # found it when we're done. + grail_fs_mode = os.stat(grail_fs_path).st_mode + initial_mode = 504 # 0770 octal + new_mode_1 = 384 # 0600 octal + new_mode_2 = 420 # 0644 octal + + # Set the initial mode, so we can be assured that when we set the mode + # to "keep", we're actually changing the permissions of the file to the + # new mode. + ret = self.run_state( + 'file.managed', + name=name, + mode=oct(initial_mode), + source=grail, + ) + self.assertSaltTrueReturn(ret) + try: + # Update the mode on the fileserver (pass 1) + os.chmod(grail_fs_path, new_mode_1) + ret = self.run_state( + 'file.managed', + name=name, + mode='keep', + source=grail, + ) + self.assertSaltTrueReturn(ret) + managed_mode = stat.S_IMODE(os.stat(name).st_mode) + self.assertEqual(oct(managed_mode), oct(new_mode_1)) + # Update the mode on the fileserver (pass 2) + # This assures us that if the file in file_roots was originally set + # to the same mode as new_mode_1, we definitely get an updated mode + # this time. + os.chmod(grail_fs_path, new_mode_2) + ret = self.run_state( + 'file.managed', + name=name, + mode='keep', + source=grail, + ) + self.assertSaltTrueReturn(ret) + managed_mode = stat.S_IMODE(os.stat(name).st_mode) + self.assertEqual(oct(managed_mode), oct(new_mode_2)) + except Exception: + raise + finally: + # Set the mode of the file in the file_roots back to what it + # originally was. + os.chmod(grail_fs_path, grail_fs_mode) + def test_managed_file_mode_file_exists_replace(self): ''' file.managed, existing file with replace=True, change permissions From f9cd83e8bb6128e9d8e34f21db5a8ac0a07d5899 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 20 Jul 2016 22:26:10 -0500 Subject: [PATCH 76/95] Add integration test for mode=keep when source is a local path --- tests/integration/states/file.py | 120 +++++++++++++++++-------------- 1 file changed, 67 insertions(+), 53 deletions(-) diff --git a/tests/integration/states/file.py b/tests/integration/states/file.py index 830f378ba9..f1a52130d0 100644 --- a/tests/integration/states/file.py +++ b/tests/integration/states/file.py @@ -56,6 +56,66 @@ FILEPILLARDEF = '/tmp/filepillar-defaultvalue' FILEPILLARGIT = '/tmp/filepillar-bar' +def _test_managed_file_mode_keep_helper(testcase, local=False): + ''' + DRY helper function to run the same test with a local or remote path + ''' + rel_path = 'grail/scene33' + name = os.path.join(integration.TMP, os.path.basename(rel_path)) + grail_fs_path = os.path.join(integration.FILES, 'file', 'base', rel_path) + grail = 'salt://' + rel_path if not local else grail_fs_path + + # Get the current mode so that we can put the file back the way we + # found it when we're done. + grail_fs_mode = os.stat(grail_fs_path).st_mode + initial_mode = 504 # 0770 octal + new_mode_1 = 384 # 0600 octal + new_mode_2 = 420 # 0644 octal + + # Set the initial mode, so we can be assured that when we set the mode + # to "keep", we're actually changing the permissions of the file to the + # new mode. + ret = testcase.run_state( + 'file.managed', + name=name, + mode=oct(initial_mode), + source=grail, + ) + testcase.assertSaltTrueReturn(ret) + try: + # Update the mode on the fileserver (pass 1) + os.chmod(grail_fs_path, new_mode_1) + ret = testcase.run_state( + 'file.managed', + name=name, + mode='keep', + source=grail, + ) + testcase.assertSaltTrueReturn(ret) + managed_mode = stat.S_IMODE(os.stat(name).st_mode) + testcase.assertEqual(oct(managed_mode), oct(new_mode_1)) + # Update the mode on the fileserver (pass 2) + # This assures us that if the file in file_roots was originally set + # to the same mode as new_mode_1, we definitely get an updated mode + # this time. + os.chmod(grail_fs_path, new_mode_2) + ret = testcase.run_state( + 'file.managed', + name=name, + mode='keep', + source=grail, + ) + testcase.assertSaltTrueReturn(ret) + managed_mode = stat.S_IMODE(os.stat(name).st_mode) + testcase.assertEqual(oct(managed_mode), oct(new_mode_2)) + except Exception: + raise + finally: + # Set the mode of the file in the file_roots back to what it + # originally was. + os.chmod(grail_fs_path, grail_fs_mode) + + class FileTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): ''' Validate the file state @@ -169,60 +229,14 @@ class FileTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): ''' Test using "mode: keep" in a file.managed state ''' - rel_path = 'grail/scene33' - name = os.path.join(integration.TMP, os.path.basename(rel_path)) - grail = 'salt://' + rel_path - grail_fs_path = os.path.join(integration.FILES, 'file', 'base', rel_path) + _test_managed_file_mode_keep_helper(self, local=False) - # Get the current mode so that we can put the file back the way we - # found it when we're done. - grail_fs_mode = os.stat(grail_fs_path).st_mode - initial_mode = 504 # 0770 octal - new_mode_1 = 384 # 0600 octal - new_mode_2 = 420 # 0644 octal - - # Set the initial mode, so we can be assured that when we set the mode - # to "keep", we're actually changing the permissions of the file to the - # new mode. - ret = self.run_state( - 'file.managed', - name=name, - mode=oct(initial_mode), - source=grail, - ) - self.assertSaltTrueReturn(ret) - try: - # Update the mode on the fileserver (pass 1) - os.chmod(grail_fs_path, new_mode_1) - ret = self.run_state( - 'file.managed', - name=name, - mode='keep', - source=grail, - ) - self.assertSaltTrueReturn(ret) - managed_mode = stat.S_IMODE(os.stat(name).st_mode) - self.assertEqual(oct(managed_mode), oct(new_mode_1)) - # Update the mode on the fileserver (pass 2) - # This assures us that if the file in file_roots was originally set - # to the same mode as new_mode_1, we definitely get an updated mode - # this time. - os.chmod(grail_fs_path, new_mode_2) - ret = self.run_state( - 'file.managed', - name=name, - mode='keep', - source=grail, - ) - self.assertSaltTrueReturn(ret) - managed_mode = stat.S_IMODE(os.stat(name).st_mode) - self.assertEqual(oct(managed_mode), oct(new_mode_2)) - except Exception: - raise - finally: - # Set the mode of the file in the file_roots back to what it - # originally was. - os.chmod(grail_fs_path, grail_fs_mode) + def test_managed_file_mode_keep_local_source(self): + ''' + Test using "mode: keep" in a file.managed state, with a local file path + as the source. + ''' + _test_managed_file_mode_keep_helper(self, local=True) def test_managed_file_mode_file_exists_replace(self): ''' From 66653620deab506807640836ebe6b5d343ddd555 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Wed, 20 Jul 2016 10:43:57 +0300 Subject: [PATCH 77/95] Fix problem when no acl configured --- salt/master.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/salt/master.py b/salt/master.py index 4465f68481..71dcec345e 100644 --- a/salt/master.py +++ b/salt/master.py @@ -2108,15 +2108,11 @@ class ClearFuncs(object): 'Authentication failure of type "user" occurred.' ) return '' - publisher_acl = salt.utils.get_values_of_matching_keys( - self.opts['publisher_acl'] or self.opts['client_acl'], - clear_load['user'].split('_', 1)[-1]) - if not publisher_acl: - log.warning( - 'Authentication failure of type "user" occurred.' - ) - return '' + publisher_acl = self.opts['publisher_acl'] or self.opts['client_acl'] if self.opts['sudo_acl'] and publisher_acl: + publisher_acl = salt.utils.get_values_of_matching_keys( + publisher_acl, + clear_load['user'].split('_', 1)[-1]) good = self.ckminions.auth_check( publisher_acl, clear_load['fun'], From 99c872bb7fe25d44bb25b954e1034c9a3658015a Mon Sep 17 00:00:00 2001 From: Alejandro Bednarik Date: Fri, 22 Jul 2016 12:16:59 -0300 Subject: [PATCH 78/95] modules.proxy: __virtual__ return err msg (#34879) --- salt/modules/proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/proxy.py b/salt/modules/proxy.py index 92aac095fb..6fe2bdca19 100644 --- a/salt/modules/proxy.py +++ b/salt/modules/proxy.py @@ -25,7 +25,7 @@ def __virtual__(): ''' if salt.utils.is_darwin() or salt.utils.is_windows(): return True - return False + return (False, 'Module proxy: module only works on Windows or MacOS systems') def _get_proxy_osx(function, network_service): From 5cdec8ced87b089caf0914b1eccac135a1f66e7a Mon Sep 17 00:00:00 2001 From: Alejandro Bednarik Date: Fri, 22 Jul 2016 12:18:13 -0300 Subject: [PATCH 79/95] Add VirtuozzoLinux is yumpkg enable list. (#34878) --- salt/modules/yumpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 9ba5197e3d..a92cac7a05 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -68,7 +68,7 @@ def __virtual__(): except Exception: return (False, "Module yumpkg: no yum based system detected") - enabled = ('amazon', 'xcp', 'xenserver') + enabled = ('amazon', 'xcp', 'xenserver', 'virtuozzolinux') if os_family == 'redhat' or os_grain in enabled: return __virtualname__ From 87ffd6dc1e585d57439c7a39312a14e1922df44a Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 22 Jul 2016 17:21:42 +0200 Subject: [PATCH 80/95] Regression fix: minion ID generator should use FQDN first, if available (#34876) * Regression fix: use FQDN first, if available * Adjust the tests to the new behaviour (FQDN first) --- salt/utils/network.py | 4 ++-- tests/unit/utils/network_test.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/utils/network.py b/salt/utils/network.py index ec199ee8b8..ae4033b4e0 100644 --- a/salt/utils/network.py +++ b/salt/utils/network.py @@ -93,7 +93,7 @@ def _generate_minion_id(): '::1.*', 'ipv6-.*', 'fe00::.*', 'fe02::.*', '1.0.0.*.ip6.arpa'] def append(self, p_object): - if p_object not in self and not self.filter(p_object): + if p_object and p_object not in self and not self.filter(p_object): super(self.__class__, self).append(p_object) return self @@ -111,7 +111,7 @@ def _generate_minion_id(): def first(self): return self and self[0] or None - hosts = DistinctList().append(platform.node()).append(socket.gethostname()).append(socket.getfqdn()) + hosts = DistinctList().append(socket.getfqdn()).append(platform.node()).append(socket.gethostname()) if not hosts: try: for a_nfo in socket.getaddrinfo(hosts.first(), None, socket.AF_INET, diff --git a/tests/unit/utils/network_test.py b/tests/unit/utils/network_test.py index a7d87bff6d..440e442fcc 100644 --- a/tests/unit/utils/network_test.py +++ b/tests/unit/utils/network_test.py @@ -251,7 +251,7 @@ class NetworkTestCase(TestCase): :return: ''' self.assertEqual(network._generate_minion_id(), - ['nodename', 'hostname', 'hostname.domainname.blank', '1.2.3.4', '5.6.7.8']) + ['hostname.domainname.blank', 'nodename', 'hostname', '1.2.3.4', '5.6.7.8']) @patch('platform.node', MagicMock(return_value='hostname')) @patch('socket.gethostname', MagicMock(return_value='hostname')) @@ -270,7 +270,7 @@ class NetworkTestCase(TestCase): @patch('platform.node', MagicMock(return_value='very.long.and.complex.domain.name')) @patch('socket.gethostname', MagicMock(return_value='hostname')) - @patch('socket.getfqdn', MagicMock(return_value='hostname')) + @patch('socket.getfqdn', MagicMock(return_value='')) @patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'hostname', ('127.0.1.1', 0))])) @patch('salt.utils.fopen', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=False)) @@ -286,7 +286,7 @@ class NetworkTestCase(TestCase): @patch('platform.node', MagicMock(return_value='localhost')) @patch('socket.gethostname', MagicMock(return_value='pick.me')) - @patch('socket.getfqdn', MagicMock(return_value='hostname')) + @patch('socket.getfqdn', MagicMock(return_value='hostname.domainname.blank')) @patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'hostname', ('127.0.1.1', 0))])) @patch('salt.utils.fopen', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=False)) @@ -297,7 +297,7 @@ class NetworkTestCase(TestCase): :return: ''' - self.assertEqual(network.generate_minion_id(), 'pick.me') + self.assertEqual(network.generate_minion_id(), 'hostname.domainname.blank') @patch('platform.node', MagicMock(return_value='localhost')) @patch('socket.gethostname', MagicMock(return_value='ip6-loopback')) From b9be6521dbd8f1d52fe0e4931d47c35dfefc340f Mon Sep 17 00:00:00 2001 From: Eric Radman Date: Fri, 22 Jul 2016 12:05:02 -0400 Subject: [PATCH 81/95] Preserve trailing lines in crontab This allows cron.file to install a crontab file that contains trailing newlines. Previously a change may have been detected on every run. --- salt/modules/cron.py | 8 +++++--- tests/unit/modules/cron_test.py | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/modules/cron.py b/salt/modules/cron.py index d9735ed988..c6aed88e2a 100644 --- a/salt/modules/cron.py +++ b/salt/modules/cron.py @@ -265,22 +265,24 @@ def raw_cron(user): cmd = 'crontab -l' else: cmd = 'crontab -l {0}'.format(user) + # Preserve line endings lines = __salt__['cmd.run_stdout'](cmd, runas=user, rstrip=False, - python_shell=False).splitlines() + python_shell=False).splitlines(True) else: if appUser == user: cmd = 'crontab -l' else: cmd = 'crontab -l -u {0}'.format(user) + # Preserve line endings lines = __salt__['cmd.run_stdout'](cmd, ignore_retcode=True, rstrip=False, - python_shell=False).splitlines() + python_shell=False).splitlines(True) if len(lines) != 0 and lines[0].startswith('# DO NOT EDIT THIS FILE - edit the master and reinstall.'): del lines[0:3] - return '\n'.join(lines) + return ''.join(lines) def list_tab(user): diff --git a/tests/unit/modules/cron_test.py b/tests/unit/modules/cron_test.py index fd9b62db0c..c7b39de5a3 100644 --- a/tests/unit/modules/cron_test.py +++ b/tests/unit/modules/cron_test.py @@ -161,7 +161,7 @@ class CronTestCase(TestCase): 'salt.modules.cron.raw_cron', new=MagicMock(side_effect=get_crontab) ): - set_crontab(L + '* * * * * ls\n') + set_crontab(L + '* * * * * ls\n\n') cron.set_job( user='root', minute='*', @@ -179,6 +179,7 @@ class CronTestCase(TestCase): c1, '# Lines below here are managed by Salt, do not edit\n' '* * * * * ls\n' + '\n' ) cron.set_job( user='root', From b552c853edfdac834b1220007e601af5f96ac712 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Fri, 22 Jul 2016 10:37:03 -0600 Subject: [PATCH 82/95] Add load_reg and save_reg for Thorium --- salt/returners/local_cache.py | 44 +++++++++++++++++++++++++++++++++++ salt/thorium/__init__.py | 13 ++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index 12fbc80b8a..4f73a1fe04 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -478,3 +478,47 @@ def get_endtime(jid): with salt.utils.fopen(etpath, 'r') as etfile: endtime = etfile.read().strip('\n') return endtime + + +def _reg_dir(): + ''' + Return the reg_dir for the given job id + ''' + return os.path.join(__opts__['cachedir'], 'thorium') + + +def save_reg(data): + ''' + Save the register to msgpack files + ''' + reg_dir = _reg_dir() + regfile = os.path.join(reg_dir, 'register') + try: + if not os.path.exists(): + os.makedirs(reg_dir) + except OSError as exc: + if exc.errno == errno.EEXIST: + pass + else: + raise + try: + with salt.utils.flopen(regfile, 'a') as fh_: + msgpack.dump(data, fh_) + fh_.close() + except: + log.error('Could not write to msgpack file {0}'.format(opts['outdir'])) + raise + + +def load_reg(): + ''' + Load the register from msgpack files + ''' + reg_dir = _reg_dir() + regfile = os.path.join(reg_dir, 'register') + try: + with salt.utils.flopen(regfile, 'a') as fh_: + return msgpack.load(fh_) + except: + log.error('Could not write to msgpack file {0}'.format(opts['outdir'])) + raise diff --git a/salt/thorium/__init__.py b/salt/thorium/__init__.py index 5e89107408..2f9587be31 100644 --- a/salt/thorium/__init__.py +++ b/salt/thorium/__init__.py @@ -18,6 +18,7 @@ import traceback # Import Salt libs import salt.state +import salt.loader import salt.payload from salt.exceptions import SaltRenderError @@ -43,7 +44,16 @@ class ThorState(salt.state.HighState): opts['file_client'] = 'local' self.opts = opts salt.state.HighState.__init__(self, self.opts, loader='thorium') - self.state.inject_globals = {'__reg__': {}} + + self.returners = salt.loader.returners(self.opts, {}) + self.reg_ret = self.opts.get('register_returner', 'local_cache') + try: + regdata = self.returners['{0}.load_reg'.format(self.reg_ret)]() + except Exception as exc: + log.error(exc) + regdata = {} + + self.state.inject_globals = {'__reg__': regdata} self.event = salt.utils.event.get_master_event( self.opts, self.opts['sock_dir']) @@ -174,4 +184,5 @@ class ThorState(salt.state.HighState): if (start - r_start) > recompile: cache = self.gather_cache() chunks = self.get_chunks() + self.returners['{0}.save_reg'.format(self.reg_ret)](chunks) r_start = time.time() From dcb8f2b983a8e452f17bce449bdb1922fb93ba6d Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Fri, 22 Jul 2016 11:03:04 -0600 Subject: [PATCH 83/95] Allow register_returner to be Non --- salt/thorium/__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/salt/thorium/__init__.py b/salt/thorium/__init__.py index 2f9587be31..56d3e3f2fc 100644 --- a/salt/thorium/__init__.py +++ b/salt/thorium/__init__.py @@ -46,12 +46,13 @@ class ThorState(salt.state.HighState): salt.state.HighState.__init__(self, self.opts, loader='thorium') self.returners = salt.loader.returners(self.opts, {}) - self.reg_ret = self.opts.get('register_returner', 'local_cache') - try: - regdata = self.returners['{0}.load_reg'.format(self.reg_ret)]() - except Exception as exc: - log.error(exc) - regdata = {} + self.reg_ret = self.opts.get('register_returner', None) + if self.reg_ret is not None: + try: + regdata = self.returners['{0}.load_reg'.format(self.reg_ret)]() + except Exception as exc: + log.error(exc) + regdata = {} self.state.inject_globals = {'__reg__': regdata} self.event = salt.utils.event.get_master_event( @@ -184,5 +185,6 @@ class ThorState(salt.state.HighState): if (start - r_start) > recompile: cache = self.gather_cache() chunks = self.get_chunks() - self.returners['{0}.save_reg'.format(self.reg_ret)](chunks) + if self.reg_ret is not None: + self.returners['{0}.save_reg'.format(self.reg_ret)](chunks) r_start = time.time() From 3de9dbffe5a7bdfcdd5c28d2b791eaefdac124d8 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Fri, 22 Jul 2016 11:41:31 -0600 Subject: [PATCH 84/95] Fix msgpack functions --- salt/returners/local_cache.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index 4f73a1fe04..cdc946aa9f 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -23,6 +23,7 @@ import salt.utils.jid import salt.exceptions # Import 3rd-party libs +import msgpack import salt.ext.six as six @@ -502,7 +503,7 @@ def save_reg(data): else: raise try: - with salt.utils.flopen(regfile, 'a') as fh_: + with salt.utils.fopen(regfile, 'a') as fh_: msgpack.dump(data, fh_) fh_.close() except: @@ -517,7 +518,7 @@ def load_reg(): reg_dir = _reg_dir() regfile = os.path.join(reg_dir, 'register') try: - with salt.utils.flopen(regfile, 'a') as fh_: + with salt.utils.fopen(regfile, 'r') as fh_: return msgpack.load(fh_) except: log.error('Could not write to msgpack file {0}'.format(opts['outdir'])) From 590de80010b1ee0df4d6443d0949adf6259a7e0f Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Fri, 22 Jul 2016 11:42:51 -0600 Subject: [PATCH 85/95] Fix __opts__ --- salt/returners/local_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index cdc946aa9f..7636890e79 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -507,7 +507,7 @@ def save_reg(data): msgpack.dump(data, fh_) fh_.close() except: - log.error('Could not write to msgpack file {0}'.format(opts['outdir'])) + log.error('Could not write to msgpack file {0}'.format(__opts__['outdir'])) raise @@ -521,5 +521,5 @@ def load_reg(): with salt.utils.fopen(regfile, 'r') as fh_: return msgpack.load(fh_) except: - log.error('Could not write to msgpack file {0}'.format(opts['outdir'])) + log.error('Could not write to msgpack file {0}'.format(__opts__['outdir'])) raise From 950cf79cd461a81af43b577b0d23658940ad33d4 Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Thu, 21 Jul 2016 15:15:51 -0500 Subject: [PATCH 86/95] Fix console and file logging on Windows On Windows, only the main process properly logged to console and to file. Things worked on other OSes due to the logging configuration being propogated via fork, something that won't work on Windows. The new strategy for Windows is if multiprocessing mode is used, all console and file logging will be performed through the Multiprocessing Logging Listener except for the main process. The main process does not send stuff to the Multiprocessing Logging Listener. Hence the main process will still configure file and console logging separately from the MP Logging Listener. If multiprocessing mode is off, then logging will be done as before (which should be ok since there is only one process). salt/utils/parsers.py: - Broke apart the logic that figures out the input options for `log.setup_logfile_logger` and `log.setup_console_logger`. This functionality is run before `self._setup_mp_logging_listener` so that the options passed to the MP Logging Listener are appropriate for direct use in the functions that setup the console and file logging in the MP Logging Listener process. - Moved the logic that verifies the path for the log file from `cli/daemons.py` to be able to run it before creating the MP Logging Listener. Its original place in `cli/daemons.py` would run after the MP Logging Listener is created. salt/log/setup.py: - Invoke `setup_console_logger` and `setup_logfile_logger` using the appropriate options if running on Windows. salt/minion.py: - No longer need to invoke the functions to setup the console and file logging on Windows multiprocessing mode due to the MP Logging Listener taking care of that on Windows. salt/cli/daemons.py: - Moved the logic that verifies the path for the log file to `utils/parsers.py`. Signed-off-by: Sergey Kizunov --- salt/cli/daemons.py | 27 --------------------- salt/log/setup.py | 13 +++++++++- salt/minion.py | 18 -------------- salt/utils/parsers.py | 56 +++++++++++++++++++++++++++++++++++-------- 4 files changed, 58 insertions(+), 56 deletions(-) diff --git a/salt/cli/daemons.py b/salt/cli/daemons.py index ee2ac320b1..79efd0eb65 100644 --- a/salt/cli/daemons.py +++ b/salt/cli/daemons.py @@ -46,7 +46,6 @@ from salt.utils import kinds try: from salt.utils import parsers, ip_bracket from salt.utils.verify import check_user, verify_env, verify_socket - from salt.utils.verify import verify_files except ImportError as exc: if exc.args[0] != 'No module named _msgpack': raise @@ -162,12 +161,6 @@ class Master(parsers.MasterOptionParser, DaemonsMixin): # pylint: disable=no-in permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) - logfile = self.config['log_file'] - if logfile is not None and not logfile.startswith(('tcp://', 'udp://', 'file://')): - # Logfile is not using Syslog, verify - current_umask = os.umask(0o027) - verify_files([logfile], self.config['user']) - os.umask(current_umask) # Clear out syndics from cachedir for syndic_file in os.listdir(self.config['syndic_dir']): os.remove(os.path.join(self.config['syndic_dir'], syndic_file)) @@ -288,12 +281,6 @@ class Minion(parsers.MinionOptionParser, DaemonsMixin): # pylint: disable=no-in permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) - logfile = self.config['log_file'] - if logfile is not None and not logfile.startswith(('tcp://', 'udp://', 'file://')): - # Logfile is not using Syslog, verify - current_umask = os.umask(0o027) - verify_files([logfile], self.config['user']) - os.umask(current_umask) except OSError as error: self.environment_failure(error) @@ -464,14 +451,6 @@ class ProxyMinion(parsers.ProxyMinionOptionParser, DaemonsMixin): # pylint: dis permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) - - logfile = self.config.get('proxy_log') or self.config['log_file'] - if logfile is not None and not logfile.startswith(('tcp://', 'udp://', 'file://')): - # Logfile is not using Syslog, verify - current_umask = os.umask(0o027) - verify_files([logfile], self.config['user']) - os.umask(current_umask) - except OSError as error: self.environment_failure(error) @@ -569,12 +548,6 @@ class Syndic(parsers.SyndicOptionParser, DaemonsMixin): # pylint: disable=no-in permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) - logfile = self.config['log_file'] - if logfile is not None and not logfile.startswith(('tcp://', 'udp://', 'file://')): - # Logfile is not using Syslog, verify - current_umask = os.umask(0o027) - verify_files([logfile], self.config['user']) - os.umask(current_umask) except OSError as error: self.environment_failure(error) diff --git a/salt/log/setup.py b/salt/log/setup.py index 04b2a8ffea..bdf2443afa 100644 --- a/salt/log/setup.py +++ b/salt/log/setup.py @@ -920,9 +920,20 @@ def __process_multiprocessing_logging_queue(opts, queue): salt.utils.appendproctitle('MultiprocessingLoggingQueue') if salt.utils.is_windows(): # On Windows, creating a new process doesn't fork (copy the parent - # process image). Due to this, we need to setup extended logging + # process image). Due to this, we need to setup all of our logging # inside this process. setup_temp_logger() + setup_console_logger( + log_level=opts.get('log_level'), + log_format=opts.get('log_fmt_console'), + date_format=opts.get('log_datefmt_console') + ) + setup_logfile_logger( + opts.get('log_file'), + log_level=opts.get('log_level_logfile'), + log_format=opts.get('log_fmt_logfile'), + date_format=opts.get('log_datefmt_logfile') + ) setup_extended_logging(opts) while True: try: diff --git a/salt/minion.py b/salt/minion.py index 9d90ea932e..0266e2fd0a 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1270,15 +1270,6 @@ class Minion(MinionBase): This method should be used as a threading target, start the actual minion side execution. ''' - # this seems awkward at first, but it's a workaround for Windows - # multiprocessing communication. - if sys.platform.startswith('win') and \ - opts['multiprocessing'] and \ - not salt.log.setup.is_logging_configured(): - # We have to re-init the logging system for Windows - salt.log.setup.setup_console_logger(log_level=opts.get('log_level', 'info')) - if opts.get('log_file'): - salt.log.setup.setup_logfile_logger(opts['log_file'], opts.get('log_level_logfile', 'info')) fn_ = os.path.join(minion_instance.proc_dir, data['jid']) if opts['multiprocessing'] and not salt.utils.is_windows(): @@ -1457,15 +1448,6 @@ class Minion(MinionBase): minion side execution. ''' salt.utils.appendproctitle('{0}._thread_multi_return {1}'.format(cls.__name__, data['jid'])) - # this seems awkward at first, but it's a workaround for Windows - # multiprocessing communication. - if sys.platform.startswith('win') and \ - opts['multiprocessing'] and \ - not salt.log.is_logging_configured(): - # We have to re-init the logging system for Windows - salt.log.setup_console_logger(log_level=opts.get('log_level', 'info')) - if opts.get('log_file'): - salt.log.setup_logfile_logger(opts['log_file'], opts.get('log_level_logfile', 'info')) ret = { 'return': {}, 'success': {}, diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index 46271a6279..eda98eac52 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -38,6 +38,7 @@ import salt.utils.jid from salt.utils import kinds from salt.defaults import DEFAULT_TARGET_DELIM from salt.utils.validate.path import is_writeable +from salt.utils.verify import verify_files import salt.exceptions # Import 3rd-party libs @@ -595,6 +596,10 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)): # Setup extended logging right before the last step self._mixin_after_parsed_funcs.append(self.__setup_extended_logging) + # Setup the console and log file configuration before the MP logging + # listener because the MP logging listener may need that config. + self._mixin_after_parsed_funcs.append(self.__setup_logfile_logger_config) + self._mixin_after_parsed_funcs.append(self.__setup_console_logger_config) # Setup the multiprocessing log queue listener if enabled self._mixin_after_parsed_funcs.append(self._setup_mp_logging_listener) # Setup the console as the last _mixin_after_parsed_func to run @@ -640,7 +645,7 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)): # defined default self.options.log_level = self._default_logging_level_ - def setup_logfile_logger(self): + def __setup_logfile_logger_config(self, *args): # pylint: disable=unused-argument if self._logfile_loglevel_config_setting_name_ in self.config and not \ self.config.get(self._logfile_loglevel_config_setting_name_): # Remove it from config so it inherits from log_level @@ -673,12 +678,23 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)): cli_log_path, self.config.get( # From the config setting - self._logfile_config_setting_name_, - # From the default setting - self._default_logging_logfile_ + self._logfile_config_setting_name_ ) ) + if self.config['verify_env']: + # Verify the logfile if it was explicitly set but do not try to + # verify the default + if logfile is not None and not logfile.startswith(('tcp://', 'udp://', 'file://')): + # Logfile is not using Syslog, verify + current_umask = os.umask(0o027) + verify_files([logfile], self.config['user']) + os.umask(current_umask) + + if logfile is None: + # Use the default setting if the logfile wasn't explicity set + logfile = self._default_logging_logfile_ + cli_log_file_fmt = 'cli_{0}_log_file_fmt'.format( self.get_prog_name().replace('-', '_') ) @@ -782,6 +798,18 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)): # If we haven't changed the logfile path and it's not writeable, # salt will fail once we try to setup the logfile logging. + # Save the settings back to the configuration + self.config[self._logfile_config_setting_name_] = logfile + self.config[self._logfile_loglevel_config_setting_name_] = loglevel + self.config['log_fmt_logfile'] = log_file_fmt + self.config['log_datefmt_logfile'] = log_file_datefmt + + def setup_logfile_logger(self): + logfile = self.config[self._logfile_config_setting_name_] + loglevel = self.config[self._logfile_loglevel_config_setting_name_] + log_file_fmt = self.config['log_fmt_logfile'] + log_file_datefmt = self.config['log_datefmt_logfile'] + log.setup_logfile_logger( logfile, loglevel, @@ -804,11 +832,7 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)): self._get_mp_logging_listener_queue() ) - def __setup_console_logger(self, *args): # pylint: disable=unused-argument - # If daemon is set force console logger to quiet - if getattr(self.options, 'daemon', False) is True: - return - + def __setup_console_logger_config(self, *args): # pylint: disable=unused-argument # Since we're not going to be a daemon, setup the console logger cli_log_fmt = 'cli_{0}_log_fmt'.format( self.get_prog_name().replace('-', '_') @@ -849,8 +873,20 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)): ) ) ) + + # Save the settings back to the configuration + self.config['log_fmt_console'] = logfmt + self.config['log_datefmt_console'] = datefmt + + def __setup_console_logger(self, *args): # pylint: disable=unused-argument + # If daemon is set force console logger to quiet + if getattr(self.options, 'daemon', False) is True: + return + log.setup_console_logger( - self.config['log_level'], log_format=logfmt, date_format=datefmt + self.config['log_level'], + log_format=self.config['log_fmt_console'], + date_format=self.config['log_datefmt_console'] ) for name, level in six.iteritems(self.config['log_granular_levels']): log.set_logger_level(name, level) From 9ee1ea8629995a5494392fcf77ba3fc6932c9a2a Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Fri, 22 Jul 2016 13:04:39 -0500 Subject: [PATCH 87/95] Fix unbound local variable error I have occasionally come across this error while running a salt-minion on Windows using Python 3.5.1 and the TCP transport: ''' File "...\salt\crypt.py", line 498, in _authenticate if not error: UnboundLocalError: local variable 'error' referenced before assignment [WARNING ] Minion received a SIGINT. Exiting. ''' It seems like there is an overloaded use of the local variable called `error`, each use with a different lifespan. Changed it so that `error` is no longer overloaded. This has fixed the issue. Signed-off-by: Sergey Kizunov --- salt/crypt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/crypt.py b/salt/crypt.py index 2f330b4c8b..e6d67adae5 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -475,7 +475,8 @@ class AsyncAuth(object): while True: try: creds = yield self.sign_in(channel=channel) - except SaltClientError as error: + except SaltClientError as exc: + error = exc break if creds == 'retry': if self.opts.get('caller'): From 1031f19f3dbf30e278420a463564f22acb527e46 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 22 Jul 2016 14:19:15 -0600 Subject: [PATCH 88/95] Update batch integration tests to not be flaky in Python3 --- tests/integration/cli/batch.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/tests/integration/cli/batch.py b/tests/integration/cli/batch.py index d627f4f459..4e2a0b6814 100644 --- a/tests/integration/cli/batch.py +++ b/tests/integration/cli/batch.py @@ -5,14 +5,15 @@ # Import Python libs from __future__ import absolute_import -# Import Salt Libs -import integration - # Import Salt Testing Libs from salttesting.helpers import ensure_in_syspath ensure_in_syspath('../../') +# Import Salt Libs +import integration +import salt.ext.six as six + class BatchTest(integration.ShellCase): ''' @@ -37,9 +38,11 @@ class BatchTest(integration.ShellCase): 'retcode:', ' 0', ' batch testing'] - ret = sorted(ret) - cmd = sorted(self.run_salt('\'*\' test.echo \'batch testing\' -b 50%')) - self.assertListEqual(cmd, ret) + cmd = self.run_salt('\'*\' test.echo \'batch testing\' -b 50%') + if six.PY3: + self.assertCountEqual(cmd, ret) + else: + self.assertListEqual(sorted(cmd), sorted(ret)) def test_batch_run_number(self): ''' @@ -57,8 +60,11 @@ class BatchTest(integration.ShellCase): ' True', 'retcode:', ' 0'] - cmd = sorted(self.run_salt('\'*\' test.ping --batch-size 2')) - self.assertListEqual(cmd, sorted(ret)) + cmd = self.run_salt('\'*\' test.ping --batch-size 2') + if six.PY3: + self.assertCountEqual(cmd, ret) + else: + self.assertListEqual(sorted(cmd), sorted(ret)) def test_batch_run_grains_targeting(self): ''' @@ -86,8 +92,11 @@ class BatchTest(integration.ShellCase): os_grain = item os_grain = os_grain.strip() - cmd = sorted(self.run_salt('-G \'os:{0}\' -b 25% test.ping'.format(os_grain))) - self.assertListEqual(cmd, sorted(ret)) + cmd = self.run_salt('-G \'os:{0}\' -b 25% test.ping'.format(os_grain)) + if six.PY3: + self.assertCountEqual(cmd, ret) + else: + self.assertListEqual(sorted(cmd), sorted(ret)) if __name__ == '__main__': From ed5b89655ad4c7f90055d03a76a385ff9ef755cb Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 22 Jul 2016 14:31:55 -0600 Subject: [PATCH 89/95] Transform unicode string to bytes before hashing --- salt/client/ssh/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index 896aa21652..dd02acde6a 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -1299,6 +1299,10 @@ def mod_data(fsclient): ret[ref] = mods_data if not ret: return {} + + if six.PY3: + ver_base = salt.utils.to_bytes(ver_base) + ver = hashlib.sha1(ver_base).hexdigest() ext_tar_path = os.path.join( fsclient.opts['cachedir'], From 56fb834718fe58728db087348830f949302cfcf6 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 22 Jul 2016 16:12:16 -0600 Subject: [PATCH 90/95] Remove tests that don't test anything --- tests/unit/cloud/clouds/dimensiondata_test.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/unit/cloud/clouds/dimensiondata_test.py b/tests/unit/cloud/clouds/dimensiondata_test.py index aa7f2c07fe..fb6315c6eb 100644 --- a/tests/unit/cloud/clouds/dimensiondata_test.py +++ b/tests/unit/cloud/clouds/dimensiondata_test.py @@ -149,22 +149,6 @@ class DimensionDataTestCase(ExtendedTestCase): 'default' ) - @patch('libcloud.compute.drivers.dimensiondata.DimensionDataNodeDriver.list_nodes', MagicMock(return_value=[])) - def test_list_nodes(self): - nodes = dimensiondata.list_nodes() - self.assertEqual( - nodes, - {} - ) - - @patch('libcloud.compute.drivers.dimensiondata.DimensionDataNodeDriver.list_locations', MagicMock(return_value=[])) - def test_list_locations(self): - locations = dimensiondata.avail_locations() - self.assertEqual( - locations, - {} - ) - if __name__ == '__main__': from integration import run_tests From cdfd0161fc7a74b80b33af5aed145de322112acf Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 22 Jul 2016 16:14:25 -0600 Subject: [PATCH 91/95] Set timeout for run_salt in test suite --- tests/integration/__init__.py | 4 ++-- tests/integration/minion/timeout.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index af72bbce5b..69ffa090f9 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1798,12 +1798,12 @@ class ShellCase(AdaptedConfigurationTestCaseMixIn, ShellTestCase, ScriptPathMixi except OSError: os.chdir(INTEGRATION_TEST_DIR) - def run_salt(self, arg_str, with_retcode=False, catch_stderr=False): + def run_salt(self, arg_str, with_retcode=False, catch_stderr=False, timeout=15): # pylint: disable=W0221 ''' Execute salt ''' arg_str = '-c {0} {1}'.format(self.get_config_dir(), arg_str) - return self.run_script('salt', arg_str, with_retcode=with_retcode, catch_stderr=catch_stderr) + return self.run_script('salt', arg_str, with_retcode=with_retcode, catch_stderr=catch_stderr, timeout=timeout) def run_ssh(self, arg_str, with_retcode=False, catch_stderr=False): ''' diff --git a/tests/integration/minion/timeout.py b/tests/integration/minion/timeout.py index ef6c37ee33..3b0ee0f6fe 100644 --- a/tests/integration/minion/timeout.py +++ b/tests/integration/minion/timeout.py @@ -24,7 +24,7 @@ class MinionTimeoutTestCase(integration.ShellCase): ''' # Launch the command sleep_length = 30 - ret = self.run_salt('minion test.sleep {0}'.format(sleep_length)) + ret = self.run_salt('minion test.sleep {0}'.format(sleep_length), timeout=45) self.assertTrue(isinstance(ret, list), 'Return is not a list. Minion' ' may have returned error: {0}'.format(ret)) self.assertTrue('True' in ret[1], 'Minion did not return True after ' From b1e5cdbe88fa828cd15a42e8d8cf1518a7627ba8 Mon Sep 17 00:00:00 2001 From: Jeremy McMillan Date: Fri, 22 Jul 2016 17:20:57 -0500 Subject: [PATCH 92/95] salt-cloud -f post_dns_record support (#34897) refactor arguments in salt.cloud.clouds.digital_ocean.post_dns_record() to enable use with salt-cloud --function on command line #34884 --- salt/cloud/clouds/digital_ocean.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/salt/cloud/clouds/digital_ocean.py b/salt/cloud/clouds/digital_ocean.py index 08bc52e2e9..5604b99e46 100644 --- a/salt/cloud/clouds/digital_ocean.py +++ b/salt/cloud/clouds/digital_ocean.py @@ -391,7 +391,11 @@ def create(vm_): ) if dns_hostname and dns_domain: log.info('create_dns_record: using dns_hostname="{0}", dns_domain="{1}"'.format(dns_hostname, dns_domain)) - __add_dns_addr__ = lambda t, d: post_dns_record(dns_domain, dns_hostname, t, d) + __add_dns_addr__ = lambda t, d: post_dns_record(dns_domain=dns_domain, + name=dns_hostname, + record_type=t, + record_data=d) + log.debug('create_dns_record: {0}'.format(__add_dns_addr__)) else: log.error('create_dns_record: could not determine dns_hostname and/or dns_domain') @@ -815,18 +819,30 @@ def destroy(name, call=None): return node -def post_dns_record(dns_domain, name, record_type, record_data): +def post_dns_record(**kwargs): ''' - Creates or updates a DNS record for the given name if the domain is managed with DO. + Creates a DNS record for the given name if the domain is managed with DO. ''' - domain = query(method='domains', droplet_id=dns_domain) + if 'kwargs' in kwargs: # flatten kwargs if called via salt-cloud -f + f_kwargs = kwargs['kwargs'] + del kwargs['kwargs'] + kwargs.update(f_kwargs) + mandatory_kwargs = ('dns_domain', 'name', 'record_type', 'record_data') + for i in mandatory_kwargs: + if kwargs[i]: + pass + else: + error = '{0}="{1}" ## all mandatory args must be provided: {2}'.format(i, kwargs[i], str(mandatory_kwargs)) + raise salt.exceptions.SaltInvocationError(error) + + domain = query(method='domains', droplet_id=kwargs['dns_domain']) if domain: result = query( method='domains', - droplet_id=dns_domain, + droplet_id=kwargs['dns_domain'], command='records', - args={'type': record_type, 'name': name, 'data': record_data}, + args={'type': kwargs['record_type'], 'name': kwargs['name'], 'data': kwargs['record_data']}, http_method='post' ) return result From bdafa60e6a9cd13adf0ada803ed8fde8a81fae3e Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Fri, 22 Jul 2016 16:22:30 -0600 Subject: [PATCH 93/95] Lint #34897 (#34907) --- salt/cloud/clouds/digital_ocean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/clouds/digital_ocean.py b/salt/cloud/clouds/digital_ocean.py index 5604b99e46..f45ac9e8ed 100644 --- a/salt/cloud/clouds/digital_ocean.py +++ b/salt/cloud/clouds/digital_ocean.py @@ -823,7 +823,7 @@ def post_dns_record(**kwargs): ''' Creates a DNS record for the given name if the domain is managed with DO. ''' - if 'kwargs' in kwargs: # flatten kwargs if called via salt-cloud -f + if 'kwargs' in kwargs: # flatten kwargs if called via salt-cloud -f f_kwargs = kwargs['kwargs'] del kwargs['kwargs'] kwargs.update(f_kwargs) From baee2f25584eed49fb84b34d28201ab2f60352e8 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 22 Jul 2016 18:59:31 -0600 Subject: [PATCH 94/95] Config test no longer applies to new test config --- tests/integration/modules/config.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/integration/modules/config.py b/tests/integration/modules/config.py index 8e04a881f0..de3f43d9fd 100644 --- a/tests/integration/modules/config.py +++ b/tests/integration/modules/config.py @@ -74,12 +74,6 @@ class ConfigTest(integration.ModuleCase): 'config.option', ['master_port']), 64506) - # Master conf opt - self.assertEqual( - self.run_function( - 'config.option', - ['syndic_master']), - 'localhost') # pillar conf opt self.assertEqual( self.run_function( From 92fffb06a7cf46e29a75be48688b5d159a1ef353 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 22 Jul 2016 21:21:55 -0500 Subject: [PATCH 95/95] Update unit test This unit test asserts based on the args with which a function was called, and the args were changed which broke the test. This fixes the test. --- tests/unit/states/file_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/states/file_test.py b/tests/unit/states/file_test.py index 05804ccb05..79d6bf95ee 100644 --- a/tests/unit/states/file_test.py +++ b/tests/unit/states/file_test.py @@ -131,7 +131,7 @@ class TestFileState(TestCase): # If the test is failing, check the position of the "contents" param # in the manage_file() function in salt/modules/file.py, the fix is # likely as simple as updating the 2nd index below. - self.assertEqual(expected, returner.call_args[0][-4]) + self.assertEqual(expected, returner.call_args[0][-5]) @skipIf(NO_MOCK, NO_MOCK_REASON)