From 7cef07b0d9c34ffbd061145c4d99224b679e9597 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 19 Apr 2017 13:59:36 -0500 Subject: [PATCH 01/19] Remove legacy git_pillar tests Also remove git_pillar configuration from the test suite's master configuration file. --- tests/integration/files/conf/master | 9 +-- .../files/file/base/file-pillargit.sls | 11 ---- tests/integration/modules/test_pillar.py | 59 ------------------- tests/integration/states/test_file.py | 25 -------- 4 files changed, 1 insertion(+), 103 deletions(-) delete mode 100644 tests/integration/files/file/base/file-pillargit.sls diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index 2bb7e4e21c..5da92b00b8 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -31,14 +31,7 @@ peer: '.*': - 'test.*' -git_pillar_verify_config: False - ext_pillar: - - git: - - master https://github.com/saltstack/pillar1.git - - master https://github.com/saltstack/pillar2.git - - dev https://github.com/saltstack/pillar1.git: - - env: testing - test_ext_pillar_opts: - test_issue_5951_actual_file_roots_in_opts @@ -102,4 +95,4 @@ libcloud_dns: driver: godaddy key: 12345 secret: mysecret - shopper_id: 12345 \ No newline at end of file + shopper_id: 12345 diff --git a/tests/integration/files/file/base/file-pillargit.sls b/tests/integration/files/file/base/file-pillargit.sls deleted file mode 100644 index 4e9cca4495..0000000000 --- a/tests/integration/files/file/base/file-pillargit.sls +++ /dev/null @@ -1,11 +0,0 @@ -{% if grains['kernel'] == 'Windows' %} - {% set TMP = "C:\\Windows\\Temp\\" %} -{% else %} - {% set TMP = "/tmp/" %} -{% endif %} - -{% set file = salt['pillar.get']('info', '') %} - -create_file: - file.managed: - - name: {{ TMP }}filepillar-{{ file }} diff --git a/tests/integration/modules/test_pillar.py b/tests/integration/modules/test_pillar.py index 37de0a7bc0..f7bb3f9e1a 100644 --- a/tests/integration/modules/test_pillar.py +++ b/tests/integration/modules/test_pillar.py @@ -5,22 +5,7 @@ from __future__ import absolute_import # Import Salt Testing libs from tests.support.case import ModuleCase -from tests.support.unit import skipIf from tests.support.paths import TMP_STATE_TREE -from tests.support.helpers import requires_network - -# Import salt libs -from salt.utils.versions import LooseVersion - -GIT_PYTHON = '0.3.2' -HAS_GIT_PYTHON = False - -try: - import git - if LooseVersion(git.__version__) >= LooseVersion(GIT_PYTHON): - HAS_GIT_PYTHON = True -except ImportError: - pass class PillarModuleTest(ModuleCase): @@ -40,32 +25,6 @@ class PillarModuleTest(ModuleCase): else: self.assertEqual(pillar['class'], 'other') - @requires_network() - @skipIf(HAS_GIT_PYTHON is False, - 'GitPython must be installed and >= version {0}'.format(GIT_PYTHON)) - def test_two_ext_pillar_sources_override(self): - ''' - https://github.com/saltstack/salt/issues/12647 - ''' - - self.assertEqual( - self.run_function('pillar.data')['info'], - 'bar' - ) - - @requires_network() - @skipIf(HAS_GIT_PYTHON is False, - 'GitPython must be installed and >= version {0}'.format(GIT_PYTHON)) - def test_two_ext_pillar_sources(self): - ''' - https://github.com/saltstack/salt/issues/12647 - ''' - - self.assertEqual( - self.run_function('pillar.data')['abc'], - 'def' - ) - def test_issue_5449_report_actual_file_roots_in_pillar(self): ''' pillar['master']['file_roots'] is overwritten by the master @@ -92,24 +51,6 @@ class PillarModuleTest(ModuleCase): self.run_function('pillar.data')['test_ext_pillar_opts']['file_roots']['base'] ) - def no_test_issue_10408_ext_pillar_gitfs_url_update(self): - import os - from salt.pillar import git_pillar - original_url = 'git+ssh://original@example.com/home/git/test' - changed_url = 'git+ssh://changed@example.com/home/git/test' - rp_location = os.path.join(self.master_opts['cachedir'], 'pillar_gitfs/0/.git') - opts = { - 'ext_pillar': [{'git': 'master {0}'.format(original_url)}], - 'cachedir': self.master_opts['cachedir'], - } - - git_pillar._LegacyGitPillar('master', original_url, opts) - opts['ext_pillar'] = [{'git': 'master {0}'.format(changed_url)}] - grepo = git_pillar._LegacyGitPillar('master', changed_url, opts) - repo = git.Repo(rp_location) - - self.assertEqual(grepo.rp_location, repo.remotes.origin.url) - def test_pillar_items(self): ''' Test to ensure we get expected output diff --git a/tests/integration/states/test_file.py b/tests/integration/states/test_file.py index 1d06efbcec..c479c227e1 100644 --- a/tests/integration/states/test_file.py +++ b/tests/integration/states/test_file.py @@ -34,7 +34,6 @@ from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs import salt.utils -from salt.utils.versions import LooseVersion HAS_PWD = True try: @@ -53,15 +52,6 @@ import salt.ext.six as six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin IS_WINDOWS = salt.utils.is_windows() -GIT_PYTHON = '0.3.2' -HAS_GIT_PYTHON = False - -try: - import git - if LooseVersion(git.__version__) >= LooseVersion(GIT_PYTHON): - HAS_GIT_PYTHON = True -except ImportError: - HAS_GIT_PYTHON = False STATE_DIR = os.path.join(FILES, 'file', 'base') if IS_WINDOWS: @@ -389,21 +379,6 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): check_file = self.run_function('file.file_exists', [FILEPILLARDEF]) self.assertTrue(check_file) - @skipIf(not HAS_GIT_PYTHON, "GitFS could not be loaded. Skipping test") - def test_managed_file_with_gitpillar_sls(self): - ''' - Test to ensure git pillar data in sls - file is rendered properly and is created. - ''' - state_name = 'file-pillargit' - - ret = self.run_function('state.sls', [state_name]) - self.assertSaltTrueReturn(ret) - - # Check to make sure the file was created - check_file = self.run_function('file.file_exists', [FILEPILLARGIT]) - self.assertTrue(check_file) - @skip_if_not_root def test_managed_dir_mode(self): ''' From e03698fbeb89125b1b179476ac85d735676a51e8 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 19 Apr 2017 15:13:52 -0500 Subject: [PATCH 02/19] Add git_pillar integration tests This also adds a new section to the runtests.py so that ext_pillar integration tests can be run separately. --- .../files/file/base/git_pillar/ssh/init.sls | 3 + .../ssh/server/files/ssh_host_rsa_key | 27 + .../ssh/server/files/ssh_host_rsa_key.pub | 1 + .../git_pillar/ssh/server/files/sshd_config | 10 + .../file/base/git_pillar/ssh/server/init.sls | 33 + .../git_pillar/ssh/user/files/authorized_keys | 2 + .../base/git_pillar/ssh/user/files/git_ssh | 2 + .../git_pillar/ssh/user/files/id_rsa_nopass | 51 + .../ssh/user/files/id_rsa_nopass.pub | 1 + .../git_pillar/ssh/user/files/id_rsa_withpass | 54 + .../ssh/user/files/id_rsa_withpass.pub | 1 + .../file/base/git_pillar/ssh/user/init.sls | 67 + tests/integration/pillar/__init__.py | 1 + tests/integration/pillar/test_git_pillar.py | 1336 +++++++++++++++++ tests/runtests.py | 12 + 15 files changed, 1601 insertions(+) create mode 100644 tests/integration/files/file/base/git_pillar/ssh/init.sls create mode 100644 tests/integration/files/file/base/git_pillar/ssh/server/files/ssh_host_rsa_key create mode 100644 tests/integration/files/file/base/git_pillar/ssh/server/files/ssh_host_rsa_key.pub create mode 100644 tests/integration/files/file/base/git_pillar/ssh/server/files/sshd_config create mode 100644 tests/integration/files/file/base/git_pillar/ssh/server/init.sls create mode 100644 tests/integration/files/file/base/git_pillar/ssh/user/files/authorized_keys create mode 100644 tests/integration/files/file/base/git_pillar/ssh/user/files/git_ssh create mode 100644 tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_nopass create mode 100644 tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_nopass.pub create mode 100644 tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_withpass create mode 100644 tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_withpass.pub create mode 100644 tests/integration/files/file/base/git_pillar/ssh/user/init.sls create mode 100644 tests/integration/pillar/__init__.py create mode 100644 tests/integration/pillar/test_git_pillar.py diff --git a/tests/integration/files/file/base/git_pillar/ssh/init.sls b/tests/integration/files/file/base/git_pillar/ssh/init.sls new file mode 100644 index 0000000000..3ec429814a --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/ssh/init.sls @@ -0,0 +1,3 @@ +include: + - git_pillar.ssh.server + - git_pillar.ssh.user diff --git a/tests/integration/files/file/base/git_pillar/ssh/server/files/ssh_host_rsa_key b/tests/integration/files/file/base/git_pillar/ssh/server/files/ssh_host_rsa_key new file mode 100644 index 0000000000..cb5dc456cf --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/ssh/server/files/ssh_host_rsa_key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAv7u9K4Ns5YxN/QLYkQ6HSP9jMw/3haGDqv3ryG5n6hglsOco +Jb0eapmBA0vc9C1VAMzyIuksBOqmDXIgX4NVkz57H4BwvKnXA4Rh5Ij/AEB2GNRo +QQzFPgBItPXGAWEkE4euI+r2pf1yBgILOWqZZ6xThT8J9aAbuFOaeZNWzfUOD9b/ +gnmPzRSazoh7YKydsG1uheKX078BCE02OSnWPj4GCc6OKuj2ufpmI8gY6x+GJbyh +ZG+BMmmcKtdsd6WcKZsranjY1oye3y8gWRT5mmdc98oqZEvARaHHZKotGNJo/Grt +rgG9oo3dUF/JdHU6rzqZetX4OeTSnON4JryCIwIDAQABAoIBAQCc3cnotu86Y29P +KKvtChjvRVtw5IhbwYhLNtJYutOz+CumL4luTut6xbqC6ueMsyYPsJ4Op/0GzMAs +0gnge0BhZsYvQNN71+z9iKra4qbXGuZEbEwbpIofrvXNcCOe704n2GNGKa/AoLpQ +Zg2u3SNDaf8vTiMk3eiwB16kR0LG32H5cWdBqvKlWI8LcchpBoNWRuA4XpRnfar4 +73pMnoehnb1ePwDqSOg38y4HYh4iD24XvBGzZDtHKmcGU47oRvIHwCOMjqu16tHk +k7LQwZoxM5z1APyw1qzV/yHGvlomf9K3ZQKViFOqm1CoN2uCjq/MnsVlCszjPe48 +NrCoFSypAoGBAPdJc7d9Vrzpxfhn+niIxgneYP4CwChqTyAvAHLV97/RtSL3HeNj +QmINKjW7MUx7ObmjF7ITkkY9hpsplX3P76ZytSBV3V9Be5rqfedGsQpWJb50MKoa +PfSvtWphK4hdAg6VN1YifbLWrD1h6bwUrzlfhYsvaqHqVnprwCTn8nFXAoGBAMZ9 +MHRHVlaeI5nBDM+fzTHhAEbTV1Ak0FfpHYnLCRVoBC6PwSNbTjQmsWJ0yEEgxe3I +U4YkgVuKR7NRFUuF9gf6Y5v57AhVuLuAg2kSXoj2DVkXSE8F1crzw8qVI1LTBIle +PcG/kmD622XKbtWpU8yTi1sdtAX7F67yhsligLoVAoGBAMsSc8e8U02iAKRk5wiy +8UbLawVNxvWpj78TOiAT3HeWxFSpcM76BVq2CvLC/dIb46Sx7VScw+OQxQiI1q3R +47DhxCKAwOFnyhTG+ovBvsOJSUek7Q3TrQtSe/2XPIOoNXc6TI4clvMVXa6uyJ5e +siLAcc+CKeQ7p7ay48CrBarTAoGAT4qlm0NnNwjibWAumRmJ6l4ndTqGN+i40THr +E2gY+MoZOuuC039ohH+pADKaeXb/un1X8163tA5jE1n/9ab2ZFYUCtKJowFvKTyj +7Lxew/YOfVBWOsy000MCiDFh2XQU0lPA3d6+czy0JUONTPQxT78kzlvF48uuvv4T +w6pEuc0CgYEA5CCrlOYrn4pt6unCm6K0yuGfmk9MTaChTdbpqAbgfp4uG3YLKHfb +G7WAoFCAsS8pWQZ7EVVj8COKMSYJT67CGaYFUUORjqPrguqv7SG4ZWhSzdzjls6X +zcg/f1GimEPP0QeZjdfF+wfrO2a1GGFAHCO6pbh0EEfmFuXGFo0DEG4= +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/files/file/base/git_pillar/ssh/server/files/ssh_host_rsa_key.pub b/tests/integration/files/file/base/git_pillar/ssh/server/files/ssh_host_rsa_key.pub new file mode 100644 index 0000000000..13dce46765 --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/ssh/server/files/ssh_host_rsa_key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC/u70rg2zljE39AtiRDodI/2MzD/eFoYOq/evIbmfqGCWw5yglvR5qmYEDS9z0LVUAzPIi6SwE6qYNciBfg1WTPnsfgHC8qdcDhGHkiP8AQHYY1GhBDMU+AEi09cYBYSQTh64j6val/XIGAgs5aplnrFOFPwn1oBu4U5p5k1bN9Q4P1v+CeY/NFJrOiHtgrJ2wbW6F4pfTvwEITTY5KdY+PgYJzo4q6Pa5+mYjyBjrH4YlvKFkb4EyaZwq12x3pZwpmytqeNjWjJ7fLyBZFPmaZ1z3yipkS8BFocdkqi0Y0mj8au2uAb2ijd1QX8l0dTqvOpl61fg55NKc43gmvIIj root@tardis diff --git a/tests/integration/files/file/base/git_pillar/ssh/server/files/sshd_config b/tests/integration/files/file/base/git_pillar/ssh/server/files/sshd_config new file mode 100644 index 0000000000..caf6ed5abe --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/ssh/server/files/sshd_config @@ -0,0 +1,10 @@ +Port {{ pillar['git_pillar']['sshd_port'] }} +ListenAddress 127.0.0.1 +PermitRootLogin no +UsePAM no +ChallengeResponseAuthentication no +PasswordAuthentication no +PubkeyAuthentication yes +PrintMotd no + +HostKey {{ pillar['git_pillar']['sshd_config_dir'] }}/ssh_host_rsa_key diff --git a/tests/integration/files/file/base/git_pillar/ssh/server/init.sls b/tests/integration/files/file/base/git_pillar/ssh/server/init.sls new file mode 100644 index 0000000000..0ec758bf8b --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/ssh/server/init.sls @@ -0,0 +1,33 @@ +{%- set sshd_config_dir = pillar['git_pillar']['sshd_config_dir'] %} + +{{ sshd_config_dir }}/sshd_config: + file.managed: + - source: salt://git_pillar/ssh/server/files/sshd_config + - user: root + - group: root + - mode: 644 + - template: jinja + +{{ sshd_config_dir }}/ssh_host_rsa_key: + file.managed: + - source: salt://git_pillar/ssh/server/files/ssh_host_rsa_key + - user: root + - group: root + - mode: 600 + - template: jinja + +{{ sshd_config_dir }}/ssh_host_rsa_key.pub: + file.managed: + - source: salt://git_pillar/ssh/server/files/ssh_host_rsa_key.pub + - user: root + - group: root + - mode: 644 + - template: jinja + +start_sshd: + cmd.run: + - name: '{{ pillar['git_pillar']['sshd_bin'] }} -f {{ sshd_config_dir }}/sshd_config' + - require: + - file: {{ sshd_config_dir }}/sshd_config + - file: {{ sshd_config_dir }}/ssh_host_rsa_key + - file: {{ sshd_config_dir }}/ssh_host_rsa_key.pub diff --git a/tests/integration/files/file/base/git_pillar/ssh/user/files/authorized_keys b/tests/integration/files/file/base/git_pillar/ssh/user/files/authorized_keys new file mode 100644 index 0000000000..d8d917eda2 --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/ssh/user/files/authorized_keys @@ -0,0 +1,2 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXo/n7jE3xaZRaJo6/ghm6sldUabg1G4DmZNWJx7By5XVnei6dWU3Qs2ZCKrgfDkq7lCPcujQpkOEX4Yei+k39A1kjlwkULXdi7aOquJOGVu32B0RjMQGG44SulAItmuXyu9dmmmVbXKPu35fGJtB4Ew9B5WNu1tzj4T9C8x5INgTJe0blKxTUaAeWD3/YCvAhKOhteLxMNHlcFdqzF/NnPIDzPXue6YHiPkxMZk/yiPJ49TKmC2L77ogxhJmOu2fGMj91Lh151EH7kmstDKzkuD/sU1jn21Y1bq8pYGiK3rNgb+NklF3BmjRl2fZod5nSmhYyqyaYUR3oyuyfqcpsaYbwLso92wnFAuhHA8M+iqgMJcyv13ur3rR81mH4PFw6viShBVUUDQ/jKad7PDeSA9OoO6+tjUbsmyUWqjCe89W0vmUJsGKoCX/0mpWW4+j8teoIfaS/jDfxGrFo5u6Sa2UB8DLxcfingyO9RW3ubkKAd/Saa/0H+8vLWdQYz8j+TDqZOn+eUjKuPMulcLiPpDAPtD5eewIG5su0Cs4rqRZr0qSRUMQ0Iqa+KuWfT9abh8LtXKRhiJFLVrtD5mRnnuR9vnJVyzD6S6VHsEHRFIrXJy8yBhEyzJlSE2iRh0aJ/tgfZ4aeiq8SO1SVu3jvW4uRf9j8gx3qOCMD0gSsVQ== nopass +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQClaM74C8D9DeonGmOZDylJ83bxID+R82fB4YT+wSDpA6ewdCBLZT5PcB/FmNZ3itBewPmimz3RUdKe6ms/rQ76tarzxm8msHt3BknqxSXqcv/Gq90ilHsdgNK8TMWRTOlp1+rX2LnV58Wf5pdFNfIqru8SGcf2c5ZhtOMVAY6dVVZiovgSoqnFPU9Asu8uMnLK/dar+JjwImULkzAeBCTus49Xk5Ss4RQ3yzbd6vucYsbAxJKiwlBcXbwlGCxYZ9wUEZqbktKRmVbFbZboisW5NKG0ks9OBAickRpFumYGBXMv8iItnugni/lpkyzBfsD8tfGg/BzlaAO1MAQER4qZYHaqMm+ZAmkGhFQ82oMNiRumrzWL0sfzJ18EEbY5dUPvYtEj+bn8uDvRVTIQcPzoppWnOsheV7YBnRNRUdsUgGITZGHoBFobwc006YJ2+S+ASLMFMlT9HNy5WuvvmiIaCZaqeZH55dlDBdu+r8qNn79sYm4FUIQ65f6TiReU8/D6s3Lc2AfakbB3n6IBFIRCFWDJlSYyzXXvTHman8tTesL2TmXrgCPiQtMAagVtU6uHYFDeRcyKkm1WDxsCAK8z6Oc9tINK1ZgtBaQLGq5HTlejLTK89iED5TPXFM0BW3w/aRfT6e62oq3GpFDka6hreR8fKkVOCVyibnWk+hqX7Q== withpass diff --git a/tests/integration/files/file/base/git_pillar/ssh/user/files/git_ssh b/tests/integration/files/file/base/git_pillar/ssh/user/files/git_ssh new file mode 100644 index 0000000000..1d8579b49d --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/ssh/user/files/git_ssh @@ -0,0 +1,2 @@ +#!/bin/sh +exec /usr/bin/ssh -p {{ pillar['git_pillar']['sshd_port'] }} -i /root/.ssh/{{ pillar['git_pillar']['id_rsa_nopass'] }} "$@" diff --git a/tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_nopass b/tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_nopass new file mode 100644 index 0000000000..8b1fc3e440 --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_nopass @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEA16P5+4xN8WmUWiaOv4IZurJXVGm4NRuA5mTVicewcuV1Z3ou +nVlN0LNmQiq4Hw5Ku5Qj3Lo0KZDhF+GHovpN/QNZI5cJFC13Yu2jqriThlbt9gdE +YzEBhuOErpQCLZrl8rvXZpplW1yj7t+XxibQeBMPQeVjbtbc4+E/QvMeSDYEyXtG +5SsU1GgHlg9/2ArwISjobXi8TDR5XBXasxfzZzyA8z17numB4j5MTGZP8ojyePUy +pgti++6IMYSZjrtnxjI/dS4dedRB+5JrLQys5Lg/7FNY59tWNW6vKWBoit6zYG/j +ZJRdwZo0Zdn2aHeZ0poWMqsmmFEd6Mrsn6nKbGmG8C7KPdsJxQLoRwPDPoqoDCXM +r9d7q960fNZh+DxcOr4koQVVFA0P4ymnezw3kgPTqDuvrY1G7JslFqownvPVtL5l +CbBiqAl/9JqVluPo/LXqCH2kv4w38RqxaObukmtlAfAy8XH4p4MjvUVt7m5CgHf0 +mmv9B/vLy1nUGM/I/kw6mTp/nlIyrjzLpXC4j6QwD7Q+XnsCBubLtArOK6kWa9Kk +kVDENCKmvirln0/Wm4fC7VykYYiRS1a7Q+ZkZ57kfb5yVcsw+kulR7BB0RSK1ycv +MgYRMsyZUhNokYdGif7YH2eGnoqvEjtUlbt471uLkX/Y/IMd6jgjA9IErFUCAwEA +AQKCAgBF7hpSVhSstkVy2tAuEL3RSqaBbGtdZZbuoEKTlNuG1xy0uu3E/0H57UO7 +L2lYQOVBYXAj04q49A/bE7tNwghqhZxxqzg5f+kYfuI1qffFeAlhYMfvtuO836mW +h88RBQuPJRVcY7N85lUPURlCHDI8zkmDYCVXu3wUtmYyiu8GEeaJhF3gUZFGtJnJ +MyuNXzayOjbt0VqXB+lXUIsEyz6W+wsCVqzxQt5pBTTvDbrdd0XSrgmHyWeHNbqa +/Fpj7ChiIMdtc9ABQzFGqRvylwq2fX3VYM4TGpEhcMyDCY29gyz+mCpQ4sBo2V4m +rYF4LVkH8ApE0jYI7T1a0fvcZ06KMwISF9NsDlSrJc6MS6GQyUyA0hYtFi79paVM +pRJ1qPoKb6On78Hiqzj4xe/WUKc+OCCDSO8n9oEXnEw4vl7sDmR8dxlhq6Mkr4nt +2DK6Cg7ITJXBcn0hD/FjJHRXR5NirL3tfNuan/BcLzuQU0xCMYIn8VG8Ajb/qJ7F +l6vy6lbyRxRigVSDWMFr+9HblaYoxdxQjgrmVwVNYmgIucOoOT6DQl9uTC9EQtcL +ubzMEaW+AdMo0th90i/b6WGro4+Gb2rq1THkA/ObvjjWQW56WZ+JBw2memYNRheG +dFLhH5ugq/AoVjcFxXha2sYQWESuzhHv6lVva6lEkSJ19Li1YQKCAQEA/HZde8qQ +ysuARW4gPLBKCByPzg2c36x0pO3xZOpKzC1MMMqarI/lCcx5gre/PlZ629zTJYgK +fOYnNpQBolotecjtffES7McYhymrIVNwMdgu0RRWT2clfV2Fx79M+EoYqNdcDdsH +7r7hvUyu+9/DRTiB0N4ZjZY+7FmFKt9c4tuica18mtPc5aGXg9oD4qxZZb+pYicj +ezxXq3wR8aLBxR9xP57lz5FjsVZATSjA0zCL+iCXVM+V5QDtKTgVs3V9v5agWn58 +2527qOU2ILXfBtfDrTH2NYtILHPOH2QXTcWOBsGhUciVWMFZLUqAPEg/G/lzIkzE +pp7naFw4dq2EqQKCAQEA2qmGFsa6HI6RtEpLRd17Ef1TME9W2e+eqd7M8QCi7e2t +Wr986pHE1/oFOBBKN6d0d5AqMZOpNhE43fMnq3EWpqW4TvgwR8uwoCp/XPHm8Sxl +WQkhGneeHc7g9fxjErXXjj6/hO44akq+M4EIfsPDt7M27EGZBTMTgVeUPnnXG7/I +pabuFXlsAdkAsEXrbEaN9T0Fb4ACKHQlA8PPIFT7eMzl91DG1xlkpb6B1OoNth6N +/I7NfxOUc3JTWJqMV7rXbniWMGCsuB900VvbhaL0bNv2VOrewGNTgLXomySPllQQ +UaWsIT3G8It9My0cRj1JevUenXfdULB9lcozvVuJzQKCAQAYYm5hGI2nqMQ48IwY +kIZ2Bhw1sMboK8YQcBMSxjZ3RiDHzanm5PcgXSmXYJwOL1gqiEe0plEtAyXidaU6 +wy8FRkz6DyDe0dQiqfmnfGGnztOmyioT/Uh3tWLIikeq6606EaMIi5FWlAVFvXRh +S5mWxAB15h3duRdWyMa9/1j/aGtmQ3V3luMNIvB5gcNCT5dK5po7qsAYlRl6rL8m +8at5mLHdjUFxLP/ODyCi0z7cpyG+BQvY2zwFJHPDuXEPJlgA+1F9rB3vMGsBwzHZ +MvfZt1llDyBSx6Mu9/h+u7IshtpS+LzWI2OZcQNmBn4gVHIUB6IBPBz6YvrC77Vc +cSIRAoIBAQCp6OK93gwOVqZXrwdQsaqJLwyuVGhLjsv+eZdMik8QjQiQpI0/hKet +n6TgjJ/vIRr6MTboMTJiRf2nUeN4b7bHJazTCD4T++4ydvNi2MG4k+PozJRBicN+ +rBvYaRbfGhf2e0G83JNP3OZxBQoB3sK9gu/ho5NxG+BDODeEWI7TDDKwrccBPsmz +odjMIHiwOR7j+le37YM/xghhJY1UNVT26Fil1cm8qQmxVRhzxq+C3bk9EAYUgbVw +A91J00XMge4W9HLYArcTl7XhXPx2mkpOMJn4IE2Yt1XShQfLThyZFpdbql3XsrZc +gjd2Rc5bshHgDoqMl/CMW6gqdeXAdVndAoIBAQDdSFdm3LrMGda3NHqTqyuo6Oa8 +AvmsL7MG4g4A3jGmr1KP6VomBBtI+Nvt9ZqAFhhYsVOAtU6TAvGohE21uSLrkmzR +nwCxNU9lgnUZHq00r1mFn9mXwJVjYpTLlQj+s3amt641+QNDQ5paWj06z6j/2nlQ +zYZ/BjNXZri+hsFCFcuSS1W0NwYzd6vHQJmRWIL4puXl2vK0ZIWcAX9l6pFJoWWC +uNIRjeyszvlav6pUVE5jBOKSt/nfgGiir353ArU6uG27z/h/ruqPIvNFeeWpmaWp +Ii8D2AAgG3mF61dvoEAtx1moB+795KajYvu0izw4gd5rsPUZRG350zVutvL/ +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_nopass.pub b/tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_nopass.pub new file mode 100644 index 0000000000..d1127c2ce5 --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_nopass.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXo/n7jE3xaZRaJo6/ghm6sldUabg1G4DmZNWJx7By5XVnei6dWU3Qs2ZCKrgfDkq7lCPcujQpkOEX4Yei+k39A1kjlwkULXdi7aOquJOGVu32B0RjMQGG44SulAItmuXyu9dmmmVbXKPu35fGJtB4Ew9B5WNu1tzj4T9C8x5INgTJe0blKxTUaAeWD3/YCvAhKOhteLxMNHlcFdqzF/NnPIDzPXue6YHiPkxMZk/yiPJ49TKmC2L77ogxhJmOu2fGMj91Lh151EH7kmstDKzkuD/sU1jn21Y1bq8pYGiK3rNgb+NklF3BmjRl2fZod5nSmhYyqyaYUR3oyuyfqcpsaYbwLso92wnFAuhHA8M+iqgMJcyv13ur3rR81mH4PFw6viShBVUUDQ/jKad7PDeSA9OoO6+tjUbsmyUWqjCe89W0vmUJsGKoCX/0mpWW4+j8teoIfaS/jDfxGrFo5u6Sa2UB8DLxcfingyO9RW3ubkKAd/Saa/0H+8vLWdQYz8j+TDqZOn+eUjKuPMulcLiPpDAPtD5eewIG5su0Cs4rqRZr0qSRUMQ0Iqa+KuWfT9abh8LtXKRhiJFLVrtD5mRnnuR9vnJVyzD6S6VHsEHRFIrXJy8yBhEyzJlSE2iRh0aJ/tgfZ4aeiq8SO1SVu3jvW4uRf9j8gx3qOCMD0gSsVQ== nopass diff --git a/tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_withpass b/tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_withpass new file mode 100644 index 0000000000..01dc953a5f --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_withpass @@ -0,0 +1,54 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,660A940DCC281B9A88A618C326B9B153 + +CC06yN5RjARlX6sRsbM3WL4D2IVnseOpjSULoA2Gcq/jAgm3hOjGADNJu8vRKvrH +TahYOE/ABQBiuJFxy45hp7wDF4ArFeAPN1cYhdHTSZYmx58BorBlYwszlKq6rTqc +mQYTtAVBiCP6KmXI6Uc1GEoD/v+t/XlFbbhkN08rsy7+WXIKuzff4ThmKfD9Bfsh +PXlJn1UHNPFSSZdLdPI2JwILAZSlFbfp7v/PosY4aeQn9fezOVBhpZ8AWlO+C5ZL +UJsUYmCvmP+L8UjXz/UVxFQxPVYFJrPe/QmxiKSjx7F42Tb/2dKYxCIc7IH2gaOM +qoPU2kFRDS8aUu6ctQw1q5h/uE/rggArueTf/zjTbOm0Jq95HsXmhPtvNNMFNlt+ +hOfPl8El+w9zRi+9Eduvm0l8IV4IryiPTgYDW/dQdgtniH5KNnqElZIlEUH2q0gd +5rC5y7du4rEBX3mwVvOWlyVQyuCjyEBiJ34BECNnSS+ZKTQPCyFuct5Ck9Fngcwk +WOeJKkFTw7IuBckdCRDg2UrSx6duTJQOF8yotJEtZkOgR3Igg7JhSOVC2UvVUGJ9 +rL8ILyc/BRgp6MPWhWIWE0eMAiVqGtwkp5Mwj3cwM7FQoKlwZymKJSeqPMUAjT7H +M+wJoMoIYEAPuBLYKO8QCUU7MxHlGyX9Eg+sryoIBCas9M0ES+q5P5nuxtPZLH8w +JmW3GSD6OLVuUnUfrF7IL9fo2LcBIRliz86JD3rkTu4PSE/lcMTu07bTtM1Ker3E +EXqmOGdwIc9HPT+4cZScuv4YAlqHQqMMcrhYPSqyLt8OwJTWMCWkTVSU9VSOEgzj +JubgiVciyco/0+qAdQSXDk9/2EApNjrGG7LKYJTExa7Bp+fYm5tp+RVxHUsLSQcA +kF3RfMC9+ab5wQDEKTDMj76n298i1GWQbAQXbqmncvW6AIIm+Q4TzILP5cyZtSiK +NJxWgkceGJNCBMNwGzlmCZc6Et3qtxy9Y0/KsTCdk+RXGdh2pFyxNy4tP9bls/TC +U1N4l+y3gaHoZ9afV0ijF10VF3dvia22e1znt2H1IpYtZi1oAk8VIuiztiJyfab8 +1jwTH0SN9Tu6YsC3Ma8EB+4/0kDFjxlQl+clzDgexXBG2ochckudSgUX7lpugVud +5jb+ujZnsaRkT1W4Qi8T/AweXCwZrSbRcilXzDdjEXIqK2Dd6e4TsUiuoHmvHwlH +xcZ+T1ytApRS3ztRV8sWGs5xqXXldWCPU+ArGY2L2Zshj61v40ON+3S3iZ5ponUP +XEOFSaZq1Bk5JZRZDRgp7evL7UPSMXQWHPRAsFFPhZejLBsk3uwh6tUOVXo0aRhw +d/xYqHFA/t9zP45ljPrjV0Ca4XIek3YFUxc8Tid6r9J5ymQRYSq8uZOcXL7x/b7E +h+hTI91DxpCCM+/FpiJBf2ffGUbTYgg0qV0qbSC6Wsfie8GdtDxBMh10ZGfQ40wB +BllbjeU0KvvSNGA3PjkwgesUMdZv2Jx6/shwcYiOSJvVPzhzxiiwxvXWA/o6QHem +K6b6zpqOagSbBSwymh9uNe6F2io4Olab+PoU6N+DVVv/kApycbovXS/0GJ4O6pDt +SFzsAE8mFKDiQEx7bei9BZIx/eg3vr64kw6GnKhzkncr/VVomYCBgHCEoag3y6Jj +YzdI7QvPzCj3kM4bGckpl845O/p13vqxeD/Vi6skRAiLv5AOOItaHQPB/ZoBeY/L +2+zy0AAtRc+o2E+CkrLPcdLv2WR3uQSSktgsW9suDNtDfPujvkFijhjidHoRfpN7 +aSzkJg/DkMwoxTOWp+Vx7wpsnE6iCmx9teazKBpGC1ngz8aLxxQGHZqQ9rt8DijO +MBbsYW3bwIWkwsTX3PmlH9nTHbxVz518xlBY1gKp5s0i4rdMB67l2duln/ffpfOR +N42GqO9Y+zQiA8LGrKu14EJZnsQUrycfbdY3/4t9mO4dXr1nPXrwPWyOQcxDEYF5 +R3JegGf+HSt4rKBYPTPmRiEfKL7eeLkLAiduHblCLILXfMkajGUZIKfSZboBFMnP +ewIgFUux8Yn95Z2/k/NSuGlOWv9YUvGmlx0GlaqW3jDqsenR4svV4c9CvnCpJdVd +z5EhUQmQVnZWoA8QhR0CuWDUQC3vRj2bCPROgtcbI+Is+8FFwzoWAcy7cIHKEAha +enBK4/0jpGx52+RNWq2Q17fdLNouSl+dbNB1bRHu8MXfzdZvez9HPhcwXJvqnh5/ +zffWzX6XW9fl/fLcTZIpIJfuDek43j5D4huhApwzbMp4952N4Hd2bkfzCsiJ7m/E +3//PWlhJ1nokU8wnXDQYkd7mmoyC5fNrqmqIeo5BtcGL4pkL3dkjK7PtRUkZ0Vt3 +mX/yUamrrSVF/i6voz5M+sr6SZAnZHDPZdqPJ+BN89KyuiqApr3QwdqupnALQmRO +eWNNusNasLCW4wto9R7MWW6W5WgltwqEIQARrtiCIRzNaAp12YFXDkoBrzUpzPps +4aOPBxOeu7QIpVhU1AvKNgCvaNgfmImJ1wyEHRmnGOpluJelhOKXVwYRDa6zRwVL +75cxY7cY+fFPGm5rqk2CjLouYsL1BJ2NA02Cr2n33Yp+wzZPQgvRHqPyGyQlqyf9 +zeKDfZf23xYsojxLevFoePppALndep2Gr2ofaBnLxV9cbsi5t3ZtgoO8QXUGfTPT +MUHs3/YQw1MNn0tZCMinPxF3On/VQB2E0VMKzbPjaWZ4Utes8p1x+fVbSEecxttv +90K4wab0Qz29yZdOZ6LAZVrJXoHn/tB2D7Qo7dhmrcv4Z98iqLgadFcQ01Z0zkvu +/rWfvSUQSNdEB/gEXHrgpdO5vRQ04E4Hz+yAnLksrLOwaznc0d3oRvWun7GUxRa9 +Rio4JhamfPIReVNvAPh7LMJKsHjVjWncsbnKnpcgMygNrWQ/gNe75D3Kdfsau7pc +PNpNsrcj9K3cr7hU08VAHgkwn6rx0TwXC8ohURErw1v22pFjX7mkaJVLezRuXlKX +gfICrlD/L0BoEy37dYgSoKI3mxkoUHtLkZttgrjvzfmFLsYANjvG+TEHxFSNte4w +V1C9fYc0IT44+sLg26AchJ1YwiRW50tkI/IPWJl4YAb4EUR0GI2PWxGVy9134Y2l +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_withpass.pub b/tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_withpass.pub new file mode 100644 index 0000000000..b65f3cbaa3 --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/ssh/user/files/id_rsa_withpass.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQClaM74C8D9DeonGmOZDylJ83bxID+R82fB4YT+wSDpA6ewdCBLZT5PcB/FmNZ3itBewPmimz3RUdKe6ms/rQ76tarzxm8msHt3BknqxSXqcv/Gq90ilHsdgNK8TMWRTOlp1+rX2LnV58Wf5pdFNfIqru8SGcf2c5ZhtOMVAY6dVVZiovgSoqnFPU9Asu8uMnLK/dar+JjwImULkzAeBCTus49Xk5Ss4RQ3yzbd6vucYsbAxJKiwlBcXbwlGCxYZ9wUEZqbktKRmVbFbZboisW5NKG0ks9OBAickRpFumYGBXMv8iItnugni/lpkyzBfsD8tfGg/BzlaAO1MAQER4qZYHaqMm+ZAmkGhFQ82oMNiRumrzWL0sfzJ18EEbY5dUPvYtEj+bn8uDvRVTIQcPzoppWnOsheV7YBnRNRUdsUgGITZGHoBFobwc006YJ2+S+ASLMFMlT9HNy5WuvvmiIaCZaqeZH55dlDBdu+r8qNn79sYm4FUIQ65f6TiReU8/D6s3Lc2AfakbB3n6IBFIRCFWDJlSYyzXXvTHman8tTesL2TmXrgCPiQtMAagVtU6uHYFDeRcyKkm1WDxsCAK8z6Oc9tINK1ZgtBaQLGq5HTlejLTK89iED5TPXFM0BW3w/aRfT6e62oq3GpFDka6hreR8fKkVOCVyibnWk+hqX7Q== withpass diff --git a/tests/integration/files/file/base/git_pillar/ssh/user/init.sls b/tests/integration/files/file/base/git_pillar/ssh/user/init.sls new file mode 100644 index 0000000000..0378a014cf --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/ssh/user/init.sls @@ -0,0 +1,67 @@ +{%- set user = pillar['git_pillar']['user'] %} + +{{ user }}: + user.present: + - gid_from_name: True + - password: '$6$saYbZFw2$rtmvt2LOYchvlM22y34mCs7FiIN4Fq27rmv/whr/M.oPrgfCDhP5uJqnfe6uwFj90FvwA45rhZplnRNMgiY.J.' + - require: + - group: {{ user }} + group.present: [] + +/home/{{ user }}/.ssh: + file.directory: + - user: {{ user }} + - group: {{ user }} + - dir_mode: 700 + - require: + - user: {{ user }} + - group: {{ user }} + +/home/{{ user }}/.ssh/authorized_keys: + file.managed: + - source: salt://git_pillar/ssh/user/files/authorized_keys + - user: {{ user }} + - group: {{ user }} + - mode: 600 + +# Custom SSH command +{{ pillar['git_pillar']['git_ssh'] }}: + file.managed: + - source: salt://git_pillar/ssh/user/files/git_ssh + - user: {{ user }} + - group: {{ user }} + - mode: 755 + - template: jinja + +/root/.ssh: + file.directory: + - dir_mode: 700 + - user: root + +/root/.ssh/{{ pillar['git_pillar']['id_rsa_nopass'] }}: + file.managed: + - source: salt://git_pillar/ssh/user/files/id_rsa_nopass + - user: root + - group: root + - mode: 600 + +/root/.ssh/{{ pillar['git_pillar']['id_rsa_nopass'] }}.pub: + file.managed: + - source: salt://git_pillar/ssh/user/files/id_rsa_nopass.pub + - user: root + - group: root + - mode: 644 + +/root/.ssh/{{ pillar['git_pillar']['id_rsa_withpass'] }}: + file.managed: + - source: salt://git_pillar/ssh/user/files/id_rsa_withpass + - user: root + - group: root + - mode: 600 + +/root/.ssh/{{ pillar['git_pillar']['id_rsa_withpass'] }}.pub: + file.managed: + - source: salt://git_pillar/ssh/user/files/id_rsa_withpass.pub + - user: root + - group: root + - mode: 644 diff --git a/tests/integration/pillar/__init__.py b/tests/integration/pillar/__init__.py new file mode 100644 index 0000000000..40a96afc6f --- /dev/null +++ b/tests/integration/pillar/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/integration/pillar/test_git_pillar.py b/tests/integration/pillar/test_git_pillar.py new file mode 100644 index 0000000000..305920dfd4 --- /dev/null +++ b/tests/integration/pillar/test_git_pillar.py @@ -0,0 +1,1336 @@ +# -*- coding: utf-8 -*- +''' +Tests for the salt-run command +''' +# Import Python libs +from __future__ import absolute_import +import errno +import logging +import os +import psutil +import random +import shutil +import string +import tempfile +import textwrap +import time +import yaml + +from salt.utils.gitfs import GITPYTHON_MINVER, PYGIT2_MINVER + +# Import Salt Testing libs +import tests.integration as integration +from tests.support.case import ModuleCase +from tests.support.mixins import LoaderModuleMockMixin, SaltReturnAssertsMixin +from tests.support.helpers import destructiveTest, requires_system_grains +from tests.support.unit import skipIf +from tests.support.mock import ( + patch, + NO_MOCK, + NO_MOCK_REASON +) + +# Import Salt libs +import salt.utils +from salt.pillar import git_pillar +from salt.ext import six +from salt.ext.six.moves import range # pylint: disable=redefined-builtin +from salt.utils.versions import LooseVersion + +try: + import git + HAS_GITPYTHON = \ + LooseVersion(git.__version__) >= LooseVersion(GITPYTHON_MINVER) +except ImportError: + HAS_GITPYTHON = False + +try: + import pygit2 + HAS_PYGIT2 = \ + LooseVersion(pygit2.__version__) >= LooseVersion(PYGIT2_MINVER) +except ImportError: + HAS_PYGIT2 = False + +NOTSET = object() +SSHD_PORT = 54309 +USER = 'gitpillaruser' +UID = 5920 + +log = logging.getLogger(__name__) + + +def _rand_key_name(length): + return 'id_rsa_{0}'.format( + ''.join(random.choice(string.ascii_letters) for _ in range(length)) + ) + + +class SSHTestBase(ModuleCase, LoaderModuleMockMixin, SaltReturnAssertsMixin): + ''' + Base class for GitPython and Pygit2 SSH tests + ''' + maxDiff = None + # Define a few variables and set to None so they're not culled in the + # cleanup when the test function completes, and remain available to the + # tearDownClass. The setUp will handle assigning values to these. + case = sshd_proc = bare_repo = admin_repo = None + # Creates random key names to (hopefully) ensure we're not overwriting an + # existing key in /root/.ssh. Even though these are destructive tests, we + # don't want to mess with something as important as ssh. + id_rsa_nopass = _rand_key_name(8) + id_rsa_withpass = _rand_key_name(8) + git_opts = '-c user.name="Foo Bar" -c user.email=foo@bar.com' + sshd_port = SSHD_PORT + sshd_wait = 10 + user = USER + uid = UID + passphrase = 'saltrules' + url = 'ssh://{user}@127.0.0.1:{port}/~/repo.git'.format( + user=USER, + port=SSHD_PORT) + + def setup_loader_modules(self): + return { + git_pillar: { + '__opts__': { + '__role': 'minion', + 'environment': None, + 'pillarenv': None, + 'hash_type': 'sha256', + 'file_roots': {}, + 'state_top': 'top.sls', + 'state_top_saltenv': None, + 'renderer': 'yaml_jinja', + 'renderer_whitelist': [], + 'renderer_blacklist': [], + 'pillar_merge_lists': False, + 'git_pillar_base': 'master', + 'git_pillar_branch': 'master', + 'git_pillar_env': '', + 'git_pillar_root': '', + 'git_pillar_ssl_verify': True, + 'git_pillar_global_lock': True, + 'git_pillar_user': '', + 'git_pillar_password': '', + 'git_pillar_insecure_auth': False, + 'git_pillar_privkey': '', + 'git_pillar_pubkey': '', + 'git_pillar_passphrase': '', + 'git_pillar_refspecs': [ + '+refs/heads/*:refs/remotes/origin/*', + '+refs/tags/*:refs/tags/*', + ], + 'git_pillar_includes': True, + }, + '__grains__': {}, + } + } + + @classmethod + def update_class(cls, case): + ''' + Make the test class available to the tearDownClass + ''' + if getattr(cls, 'case') is None: + setattr(cls, 'case', case) + + @classmethod + def setUpClass(cls): + cls.orig_uid = os.geteuid() + cls.orig_gid = os.getegid() + cls.environ = dict([(x, y) for x, y in six.iteritems(os.environ) + if x in ('USER', 'HOME')]) + home = '/root/.ssh' + cls.ext_opts = { + 'url': cls.url, + 'privkey_nopass': os.path.join(home, cls.id_rsa_nopass), + 'pubkey_nopass': os.path.join(home, cls.id_rsa_nopass + '.pub'), + 'privkey_withpass': os.path.join(home, cls.id_rsa_withpass), + 'pubkey_withpass': os.path.join(home, cls.id_rsa_withpass + '.pub'), + 'passphrase': cls.passphrase} + + @classmethod + def tearDownClass(cls): + ''' + Stop the SSH server, remove the user, and clean up the config dir + ''' + if cls.case.sshd_proc: + try: + cls.case.sshd_proc.kill() + except psutil.NoSuchProcess: + pass + cls.case.run_state('user.absent', name=cls.user, purge=True) + for dirname in (cls.sshd_config_dir, cls.case.admin_repo, + cls.case.bare_repo): + if dirname is not None: + shutil.rmtree(dirname, ignore_errors=True) + ssh_dir = os.path.expanduser('~/.ssh') + for key_name in (cls.id_rsa_nopass, cls.id_rsa_withpass): + try: + os.remove(os.path.join(ssh_dir, key_name)) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + + @requires_system_grains + def setUp(self, grains): + ''' + Create the SSH server and user + ''' + self.grains = grains + # Make the test class available to the tearDownClass so we can clean up + # after ourselves. This (and the gated block below) prevent us from + # needing to spend the extra time creating an ssh server and user and + # then tear them down separately for each test. + self.update_class(self) + + sshd_config_file = os.path.join(self.sshd_config_dir, 'sshd_config') + self.sshd_proc = self.find_sshd(sshd_config_file) + self.sshd_bin = salt.utils.which('sshd') + self.git_ssh = '/tmp/git_ssh' + + if self.sshd_proc is None: + user_files = os.listdir( + os.path.join( + integration.FILES, 'file/base/git_pillar/ssh/user/files' + ) + ) + ret = self.run_function( + 'state.apply', + mods='git_pillar.ssh', + pillar={'git_pillar': {'git_ssh': self.git_ssh, + 'id_rsa_nopass': self.id_rsa_nopass, + 'id_rsa_withpass': self.id_rsa_withpass, + 'sshd_bin': self.sshd_bin, + 'sshd_port': self.sshd_port, + 'sshd_config_dir': self.sshd_config_dir, + 'master_user': self.master_opts['user'], + 'user': self.user, + 'uid': self.uid, + 'user_files': user_files}} + ) + + try: + for idx in range(1, self.sshd_wait + 1): + self.sshd_proc = self.find_sshd(sshd_config_file) + if self.sshd_proc is not None: + break + else: + if idx != self.sshd_wait: + log.debug( + 'Waiting for sshd process (%d of %d)', + idx, self.sshd_wait + ) + time.sleep(1) + else: + log.debug( + 'Failed fo find sshd process after %d seconds', + self.sshd_wait + ) + else: + raise Exception( + 'Unable to find an sshd process running from temp ' + 'config file {0} using psutil. Check to see if an ' + 'instance of sshd from an earlier aborted run of ' + 'these tests is running, if so then manually kill ' + 'it and re-run test(s).'.format(sshd_config_file) + ) + finally: + # Do the assert after we check for the PID so that we can track + # it regardless of whether or not something else in the SLS + # failed (but the SSH server still started). + self.assertSaltTrueReturn(ret) + + known_hosts_ret = self.run_function( + 'ssh.set_known_host', + user=self.master_opts['user'], + hostname='127.0.0.1', + port=self.sshd_port, + enc='ssh-rsa', + fingerprint='fd:6f:7f:5d:06:6b:f2:06:0d:26:93:9e:5a:b5:19:46', + hash_known_hosts=False, + ) + if 'error' in known_hosts_ret: + raise Exception( + 'Failed to add key to {0} user\'s known_hosts ' + 'file: {1}'.format( + self.master_opts['user'], + known_hosts_ret['error'] + ) + ) + + self.make_repo() + + def make_repo(self): + self.bare_repo = os.path.expanduser('~{0}/repo.git'.format(self.user)) + if self.bare_repo.startswith('~'): + self.bare_repo = None + self.fail( + 'Unable to resolve homedir for user \'{0}\''.format(self.user)) + + # Don't need to repeat the startswith check for this one, if we were + # unable to resolve the homedir here, we'd have aborted already. + self.admin_repo = os.path.expanduser('~{0}/admin_repo'.format(self.user)) + + for dirname in (self.bare_repo, self.admin_repo): + shutil.rmtree(dirname, ignore_errors=True) + + # Create bare repo + self.run_function( + 'git.init', + [self.bare_repo], + user=self.user, + bare=True) + + # Clone bare repo + self.run_function( + 'git.clone', + [self.admin_repo], + url=self.bare_repo, + user=self.user) + + def _push(branch, message): + self.run_function( + 'git.add', + [self.admin_repo, '.'], + user=self.user) + self.run_function( + 'git.commit', + [self.admin_repo, message], + user=self.user, + git_opts=self.git_opts, + ) + self.run_function( + 'git.push', + [self.admin_repo], + remote='origin', + ref=branch, + user=self.user, + ) + + with salt.utils.fopen( + os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_: + fp_.write(textwrap.dedent('''\ + base: + '*': + - foo + ''')) + with salt.utils.fopen( + os.path.join(self.admin_repo, 'foo.sls'), 'w') as fp_: + fp_.write(textwrap.dedent('''\ + branch: master + mylist: + - master + mydict: + master: True + nested_list: + - master + nested_dict: + master: True + ''')) + # Add another file to be referenced using git_pillar_includes + with salt.utils.fopen( + os.path.join(self.admin_repo, 'bar.sls'), 'w') as fp_: + fp_.write('included_pillar: True\n') + _push('master', 'initial commit') + + # Do the same with different values for "dev" branch + self.run_function( + 'git.checkout', + [self.admin_repo], + user=self.user, + opts='-b dev') + # The bar.sls shouldn't be in any branch but master + self.run_function( + 'git.rm', + [self.admin_repo, 'bar.sls'], + user=self.user) + with salt.utils.fopen( + os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_: + fp_.write(textwrap.dedent('''\ + dev: + '*': + - foo + ''')) + with salt.utils.fopen( + os.path.join(self.admin_repo, 'foo.sls'), 'w') as fp_: + fp_.write(textwrap.dedent('''\ + branch: dev + mylist: + - dev + mydict: + dev: True + nested_list: + - dev + nested_dict: + dev: True + ''')) + _push('dev', 'add dev branch') + + # Create just a top file in a separate repo, to be mapped to the base + # env and referenced using git_pillar_includes + self.run_function( + 'git.checkout', + [self.admin_repo], + user=self.user, + opts='-b top_only') + # The top.sls should be the only file in this branch + self.run_function( + 'git.rm', + [self.admin_repo, 'foo.sls'], + user=self.user) + with salt.utils.fopen( + os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_: + fp_.write(textwrap.dedent('''\ + base: + '*': + - bar + ''')) + _push('top_only', 'add top_only branch') + + def find_sshd(self, sshd_config_file): + for proc in psutil.process_iter(): + if 'sshd' in proc.name(): + if sshd_config_file in proc.cmdline(): + return proc + return None + + def get_pillar(self, ext_pillar_conf): + ''' + Run git_pillar with the specified configuration + ''' + cachedir = tempfile.mkdtemp(dir=integration.TMP) + self.addCleanup(shutil.rmtree, cachedir, ignore_errors=True) + ext_pillar_opts = yaml.safe_load( + ext_pillar_conf.format( + cachedir=cachedir, + extmods=os.path.join(cachedir, 'extmods'), + **self.ext_opts + ) + ) + with patch.dict(git_pillar.__opts__, ext_pillar_opts): + with patch.dict(git_pillar.__grains__, self.grains): + return git_pillar.ext_pillar( + 'minion', + ext_pillar_opts['ext_pillar'][0]['git'], + {} + ) + + +@destructiveTest +@skipIf(not salt.utils.which('sshd'), 'sshd not present') +@skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) +@skipIf(salt.utils.is_windows(), 'minion is windows') +@skipIf(os.getuid() != 0, 'must be root to run this test') +@skipIf(NO_MOCK, NO_MOCK_REASON) +class TestGitPythonSSH(SSHTestBase): + ''' + Test git_pillar with GitPython using SSH authentication + + NOTE: Any tests added to this test class should have equivalent tests (if + possible) in the TestPygit2SSH class. Also, bear in mind that the pygit2 + versions of these tests need to be more complex in that they need to test + both with passphraseless and passphrase-protecteed keys, both with global + and per-remote configuration. So for every time we run a GitPython test, we + need to run that same test four different ways for pygit2. This is because + GitPython's ability to use git-over-SSH is limited to passphraseless keys. + So, unlike pygit2, we don't need to test global or per-repo credential + config params since GitPython doesn't use them. + ''' + sshd_config_dir = tempfile.mkdtemp(dir=integration.TMP) + + def get_pillar(self, ext_pillar_conf): + ''' + Wrap the parent class' get_pillar() func in logic that temporarily + changes the GIT_SSH to use our custom script, ensuring that the + passphraselsess key is used to auth without needing to modify the root + user's ssh config file. + ''' + orig_git_ssh = os.environ.pop('GIT_SSH', NOTSET) + os.environ['GIT_SSH'] = self.git_ssh + try: + return super(TestGitPythonSSH, self).get_pillar(ext_pillar_conf) + finally: + os.environ.pop('GIT_SSH', None) + if orig_git_ssh is not NOTSET: + os.environ['GIT_SSH'] = orig_git_ssh + + def test_git_pillar_single_source(self): + ''' + Test using a single ext_pillar repo + ''' + ret = self.get_pillar('''\ + git_pillar_provider: gitpython + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + ''') + self.assertEqual( + ret, + {'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}}} + ) + + def test_git_pillar_multiple_sources_master_dev_no_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the master branch followed by dev, and with + pillar_merge_lists disabled. + ''' + ret = self.get_pillar('''\ + git_pillar_provider: gitpython + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual( + ret, + {'branch': 'dev', + 'mylist': ['dev'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['dev'], + 'nested_dict': {'master': True, 'dev': True}}} + ) + + def test_git_pillar_multiple_sources_dev_master_no_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the dev branch followed by master, and with + pillar_merge_lists disabled. + ''' + ret = self.get_pillar('''\ + git_pillar_provider: gitpython + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - dev {url} + - master {url} + ''') + self.assertEqual( + ret, + {'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True, 'dev': True}}} + ) + + def test_git_pillar_multiple_sources_master_dev_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the master branch followed by dev, and with + pillar_merge_lists enabled. + ''' + ret = self.get_pillar('''\ + git_pillar_provider: gitpython + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual( + ret, + {'branch': 'dev', + 'mylist': ['master', 'dev'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['master', 'dev'], + 'nested_dict': {'master': True, 'dev': True}}} + ) + + def test_git_pillar_multiple_sources_dev_master_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the dev branch followed by master, and with + pillar_merge_lists enabled. + ''' + ret = self.get_pillar('''\ + git_pillar_provider: gitpython + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - dev {url} + - master {url} + ''') + self.assertEqual( + ret, + {'branch': 'master', + 'mylist': ['dev', 'master'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['dev', 'master'], + 'nested_dict': {'master': True, 'dev': True}}} + ) + + def test_git_pillar_multiple_sources_with_pillarenv(self): + ''' + Test using pillarenv to restrict results to those from a single branch + ''' + ret = self.get_pillar('''\ + git_pillar_provider: gitpython + cachedir: {cachedir} + extension_modules: {extmods} + pillarenv: base + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual( + ret, + {'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}}} + ) + + def test_git_pillar_includes_enabled(self): + ''' + Test with git_pillar_includes enabled. The top_only branch references + an SLS file from the master branch, so we should see the key from that + SLS file (included_pillar) in the compiled pillar data. + ''' + ret = self.get_pillar('''\ + git_pillar_provider: gitpython + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + - top_only {url}: + - env: base + ''') + self.assertEqual( + ret, + {'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}}, + 'included_pillar': True} + ) + + def test_git_pillar_includes_disabled(self): + ''' + Test with git_pillar_includes enabled. The top_only branch references + an SLS file from the master branch, but since includes are disabled it + will not find the SLS file and the "included_pillar" key should not be + present in the compiled pillar data. We should instead see an error + message in the compiled data. + ''' + ret = self.get_pillar('''\ + git_pillar_provider: gitpython + git_pillar_includes: False + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + - top_only {url}: + - env: base + ''') + self.assertEqual( + ret, + {'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}}, + '_errors': ["Specified SLS 'bar' in environment 'base' is not " + "available on the salt master"]} + ) + + +@destructiveTest +@skipIf(not salt.utils.which('sshd'), 'sshd not present') +@skipIf(not HAS_PYGIT2, 'pygit2 >= {0} required'.format(PYGIT2_MINVER)) +@skipIf(salt.utils.is_windows(), 'minion is windows') +@skipIf(os.getuid() != 0, 'must be root to run this test') +@skipIf(NO_MOCK, NO_MOCK_REASON) +class TestPygit2SSH(SSHTestBase): + ''' + Test git_pillar with pygit2 using SSH authentication + + NOTE: Any tests added to this test class should have equivalent tests (if + possible) in the TestGitPythonSSH class. + ''' + sshd_config_dir = tempfile.mkdtemp(dir=integration.TMP) + + def test_git_pillar_single_source(self): + ''' + Test using a single ext_pillar repo + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}} + } + + # Test with passphraseless key and global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_nopass} + git_pillar_privkey: {privkey_nopass} + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + ''') + self.assertEqual(ret, expected) + + # Test with passphraseless key and per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + ''') + self.assertEqual(ret, expected) + + # Test with passphrase-protected key and global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_withpass} + git_pillar_privkey: {privkey_withpass} + git_pillar_passphrase: {passphrase} + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + ''') + self.assertEqual(ret, expected) + + # Test with passphrase-protected key and per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + ''') + self.assertEqual(ret, expected) + + def test_git_pillar_multiple_sources_master_dev_no_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the master branch followed by dev, and with + pillar_merge_lists disabled. + ''' + expected = { + 'branch': 'dev', + 'mylist': ['dev'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['dev'], + 'nested_dict': {'master': True, 'dev': True}} + } + + # passphraseless key, global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_nopass} + git_pillar_privkey: {privkey_nopass} + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual(ret, expected) + + # passphraseless key, per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - master {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + - dev {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + ''') + self.assertEqual(ret, expected) + + # passphrase-protected key, global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_withpass} + git_pillar_privkey: {privkey_withpass} + git_pillar_passphrase: {passphrase} + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual(ret, expected) + + # passphrase-protected key, per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - master {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + - dev {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + ''') + self.assertEqual(ret, expected) + + def test_git_pillar_multiple_sources_dev_master_no_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the dev branch followed by master, and with + pillar_merge_lists disabled. + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True, 'dev': True}} + } + + # passphraseless key, global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_nopass} + git_pillar_privkey: {privkey_nopass} + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - dev {url} + - master {url} + ''') + self.assertEqual(ret, expected) + + # passphraseless key, per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - dev {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + - master {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + ''') + self.assertEqual(ret, expected) + + # passphrase-protected key, global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_withpass} + git_pillar_privkey: {privkey_withpass} + git_pillar_passphrase: {passphrase} + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - dev {url} + - master {url} + ''') + self.assertEqual(ret, expected) + + # passphrase-protected key, per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - dev {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + - master {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + ''') + self.assertEqual(ret, expected) + + def test_git_pillar_multiple_sources_master_dev_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the master branch followed by dev, and with + pillar_merge_lists enabled. + ''' + expected = { + 'branch': 'dev', + 'mylist': ['master', 'dev'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['master', 'dev'], + 'nested_dict': {'master': True, 'dev': True}} + } + + # passphraseless key, global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_nopass} + git_pillar_privkey: {privkey_nopass} + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual(ret, expected) + + # passphraseless key, per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - master {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + - dev {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + ''') + self.assertEqual(ret, expected) + + # passphrase-protected key, global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_withpass} + git_pillar_privkey: {privkey_withpass} + git_pillar_passphrase: {passphrase} + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual(ret, expected) + + # passphrase-protected key, per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - master {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + - dev {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + ''') + self.assertEqual(ret, expected) + + def test_git_pillar_multiple_sources_dev_master_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the dev branch followed by master, and with + pillar_merge_lists enabled. + ''' + expected = { + 'branch': 'master', + 'mylist': ['dev', 'master'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['dev', 'master'], + 'nested_dict': {'master': True, 'dev': True}} + } + + # passphraseless key, global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_nopass} + git_pillar_privkey: {privkey_nopass} + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - dev {url} + - master {url} + ''') + self.assertEqual(ret, expected) + + # passphraseless key, per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - dev {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + - master {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + ''') + self.assertEqual(ret, expected) + + # passphrase-protected key, global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_withpass} + git_pillar_privkey: {privkey_withpass} + git_pillar_passphrase: {passphrase} + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - dev {url} + - master {url} + ''') + self.assertEqual(ret, expected) + + # passphrase-protected key, per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - dev {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + - master {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + ''') + self.assertEqual(ret, expected) + + def test_git_pillar_multiple_sources_with_pillarenv(self): + ''' + Test using pillarenv to restrict results to those from a single branch + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}} + } + + # Test with passphraseless key and global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_nopass} + git_pillar_privkey: {privkey_nopass} + cachedir: {cachedir} + extension_modules: {extmods} + pillarenv: base + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual(ret, expected) + + # Test with passphraseless key and per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillarenv: base + ext_pillar: + - git: + - master {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + - dev {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + ''') + self.assertEqual(ret, expected) + + # Test with passphrase-protected key and global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_withpass} + git_pillar_privkey: {privkey_withpass} + git_pillar_passphrase: {passphrase} + cachedir: {cachedir} + extension_modules: {extmods} + pillarenv: base + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual(ret, expected) + + # Test with passphrase-protected key and per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillarenv: base + ext_pillar: + - git: + - master {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + - dev {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + - passphrase: {passphrase} + ''') + self.assertEqual(ret, expected) + + def test_git_pillar_includes_enabled(self): + ''' + Test with git_pillar_includes enabled. The top_only branch references + an SLS file from the master branch, so we should see the + "included_pillar" key from that SLS file in the compiled pillar data. + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}}, + 'included_pillar': True + } + + # passphraseless key, global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_nopass} + git_pillar_privkey: {privkey_nopass} + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + - top_only {url}: + - env: base + ''') + self.assertEqual(ret, expected) + + # passphraseless key, per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + - top_only {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + - env: base + ''') + self.assertEqual(ret, expected) + + # passphrase-protected key, global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_pubkey: {pubkey_withpass} + git_pillar_privkey: {privkey_withpass} + git_pillar_passphrase: {passphrase} + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + - top_only {url}: + - env: base + ''') + self.assertEqual(ret, expected) + + # passphrase-protected key, per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + - top_only {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + - env: base + ''') + self.assertEqual(ret, expected) + + def test_git_pillar_includes_disabled(self): + ''' + Test with git_pillar_includes enabled. The top_only branch references + an SLS file from the master branch, but since includes are disabled it + will not find the SLS file and the "included_pillar" key should not be + present in the compiled pillar data. We should instead see an error + message in the compiled data. + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}}, + '_errors': ["Specified SLS 'bar' in environment 'base' is not " + "available on the salt master"] + } + + # passphraseless key, global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_includes: False + git_pillar_pubkey: {pubkey_nopass} + git_pillar_privkey: {privkey_nopass} + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + - top_only {url}: + - env: base + ''') + self.assertEqual(ret, expected) + + # passphraseless key, per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_includes: False + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + - top_only {url}: + - pubkey: {pubkey_nopass} + - privkey: {privkey_nopass} + - env: base + ''') + self.assertEqual(ret, expected) + + # passphrase-protected key, global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_includes: False + git_pillar_pubkey: {pubkey_withpass} + git_pillar_privkey: {privkey_withpass} + git_pillar_passphrase: {passphrase} + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + - top_only {url}: + - env: base + ''') + self.assertEqual(ret, expected) + + # passphrase-protected key, per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_includes: False + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + - top_only {url}: + - pubkey: {pubkey_withpass} + - privkey: {privkey_withpass} + - passphrase: {passphrase} + - env: base + ''') + self.assertEqual(ret, expected) diff --git a/tests/runtests.py b/tests/runtests.py index 0d4899b456..439affa666 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -112,6 +112,9 @@ TEST_SUITES = { 'client': {'display_name': 'Client', 'path': 'integration/client'}, + 'ext_pillar': + {'display_name': 'External Pillar', + 'path': 'integration/pillar'}, 'grains': {'display_name': 'Grains', 'path': 'integration/grains'}, @@ -255,6 +258,15 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): action='store_true', help='Run tests for client' ) + self.test_selection_group.add_option( + '-I', + '--ext-pillar', + '--ext-pillar-tests', + dest='ext_pillar', + default=False, + action='store_true', + help='Run ext_pillar tests' + ) self.test_selection_group.add_option( '-G', '--grains', From 3881b6e0f3cf4e61a0c861c4a3260234ddd4b53d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 21 Apr 2017 15:35:51 -0500 Subject: [PATCH 03/19] Prepare git_pillar test code for adding HTTP tests This further abstracts some of the setup and teardown code so it can be used for git-over-http tests. It also moves the code that was originally added to the archive state integration tests to create a local http server into salt.support.helpers so that it can be more easily and portably used. --- tests/integration/pillar/test_git_pillar.py | 465 ++++---------------- tests/integration/states/test_archive.py | 49 +-- tests/integration/states/test_file.py | 56 +-- tests/support/git_pillar.py | 448 +++++++++++++++++++ tests/support/helpers.py | 196 ++++++++- 5 files changed, 735 insertions(+), 479 deletions(-) create mode 100644 tests/support/git_pillar.py diff --git a/tests/integration/pillar/test_git_pillar.py b/tests/integration/pillar/test_git_pillar.py index 305920dfd4..46b8f52816 100644 --- a/tests/integration/pillar/test_git_pillar.py +++ b/tests/integration/pillar/test_git_pillar.py @@ -4,37 +4,26 @@ Tests for the salt-run command ''' # Import Python libs from __future__ import absolute_import -import errno -import logging import os -import psutil -import random -import shutil -import string import tempfile -import textwrap -import time -import yaml +import tornado.web from salt.utils.gitfs import GITPYTHON_MINVER, PYGIT2_MINVER # Import Salt Testing libs -import tests.integration as integration -from tests.support.case import ModuleCase -from tests.support.mixins import LoaderModuleMockMixin, SaltReturnAssertsMixin -from tests.support.helpers import destructiveTest, requires_system_grains -from tests.support.unit import skipIf -from tests.support.mock import ( - patch, - NO_MOCK, - NO_MOCK_REASON +from tests.support.helpers import ( + destructiveTest, + http_basic_auth, + skip_if_not_root, + Webserver, ) +from tests.support.git_pillar import HTTPTestBase, SSHTestBase +from tests.support.paths import TMP +from tests.support.unit import skipIf +from tests.support.mock import NO_MOCK, NO_MOCK_REASON # Import Salt libs import salt.utils -from salt.pillar import git_pillar -from salt.ext import six -from salt.ext.six.moves import range # pylint: disable=redefined-builtin from salt.utils.versions import LooseVersion try: @@ -52,376 +41,20 @@ except ImportError: HAS_PYGIT2 = False NOTSET = object() -SSHD_PORT = 54309 -USER = 'gitpillaruser' -UID = 5920 - -log = logging.getLogger(__name__) +USERNAME = 'gitpillaruser' +PASSWORD = 'saltrules' -def _rand_key_name(length): - return 'id_rsa_{0}'.format( - ''.join(random.choice(string.ascii_letters) for _ in range(length)) - ) - - -class SSHTestBase(ModuleCase, LoaderModuleMockMixin, SaltReturnAssertsMixin): - ''' - Base class for GitPython and Pygit2 SSH tests - ''' - maxDiff = None - # Define a few variables and set to None so they're not culled in the - # cleanup when the test function completes, and remain available to the - # tearDownClass. The setUp will handle assigning values to these. - case = sshd_proc = bare_repo = admin_repo = None - # Creates random key names to (hopefully) ensure we're not overwriting an - # existing key in /root/.ssh. Even though these are destructive tests, we - # don't want to mess with something as important as ssh. - id_rsa_nopass = _rand_key_name(8) - id_rsa_withpass = _rand_key_name(8) - git_opts = '-c user.name="Foo Bar" -c user.email=foo@bar.com' - sshd_port = SSHD_PORT - sshd_wait = 10 - user = USER - uid = UID - passphrase = 'saltrules' - url = 'ssh://{user}@127.0.0.1:{port}/~/repo.git'.format( - user=USER, - port=SSHD_PORT) - - def setup_loader_modules(self): - return { - git_pillar: { - '__opts__': { - '__role': 'minion', - 'environment': None, - 'pillarenv': None, - 'hash_type': 'sha256', - 'file_roots': {}, - 'state_top': 'top.sls', - 'state_top_saltenv': None, - 'renderer': 'yaml_jinja', - 'renderer_whitelist': [], - 'renderer_blacklist': [], - 'pillar_merge_lists': False, - 'git_pillar_base': 'master', - 'git_pillar_branch': 'master', - 'git_pillar_env': '', - 'git_pillar_root': '', - 'git_pillar_ssl_verify': True, - 'git_pillar_global_lock': True, - 'git_pillar_user': '', - 'git_pillar_password': '', - 'git_pillar_insecure_auth': False, - 'git_pillar_privkey': '', - 'git_pillar_pubkey': '', - 'git_pillar_passphrase': '', - 'git_pillar_refspecs': [ - '+refs/heads/*:refs/remotes/origin/*', - '+refs/tags/*:refs/tags/*', - ], - 'git_pillar_includes': True, - }, - '__grains__': {}, - } - } - - @classmethod - def update_class(cls, case): - ''' - Make the test class available to the tearDownClass - ''' - if getattr(cls, 'case') is None: - setattr(cls, 'case', case) - - @classmethod - def setUpClass(cls): - cls.orig_uid = os.geteuid() - cls.orig_gid = os.getegid() - cls.environ = dict([(x, y) for x, y in six.iteritems(os.environ) - if x in ('USER', 'HOME')]) - home = '/root/.ssh' - cls.ext_opts = { - 'url': cls.url, - 'privkey_nopass': os.path.join(home, cls.id_rsa_nopass), - 'pubkey_nopass': os.path.join(home, cls.id_rsa_nopass + '.pub'), - 'privkey_withpass': os.path.join(home, cls.id_rsa_withpass), - 'pubkey_withpass': os.path.join(home, cls.id_rsa_withpass + '.pub'), - 'passphrase': cls.passphrase} - - @classmethod - def tearDownClass(cls): - ''' - Stop the SSH server, remove the user, and clean up the config dir - ''' - if cls.case.sshd_proc: - try: - cls.case.sshd_proc.kill() - except psutil.NoSuchProcess: - pass - cls.case.run_state('user.absent', name=cls.user, purge=True) - for dirname in (cls.sshd_config_dir, cls.case.admin_repo, - cls.case.bare_repo): - if dirname is not None: - shutil.rmtree(dirname, ignore_errors=True) - ssh_dir = os.path.expanduser('~/.ssh') - for key_name in (cls.id_rsa_nopass, cls.id_rsa_withpass): - try: - os.remove(os.path.join(ssh_dir, key_name)) - except OSError as exc: - if exc.errno != errno.ENOENT: - raise - - @requires_system_grains - def setUp(self, grains): - ''' - Create the SSH server and user - ''' - self.grains = grains - # Make the test class available to the tearDownClass so we can clean up - # after ourselves. This (and the gated block below) prevent us from - # needing to spend the extra time creating an ssh server and user and - # then tear them down separately for each test. - self.update_class(self) - - sshd_config_file = os.path.join(self.sshd_config_dir, 'sshd_config') - self.sshd_proc = self.find_sshd(sshd_config_file) - self.sshd_bin = salt.utils.which('sshd') - self.git_ssh = '/tmp/git_ssh' - - if self.sshd_proc is None: - user_files = os.listdir( - os.path.join( - integration.FILES, 'file/base/git_pillar/ssh/user/files' - ) - ) - ret = self.run_function( - 'state.apply', - mods='git_pillar.ssh', - pillar={'git_pillar': {'git_ssh': self.git_ssh, - 'id_rsa_nopass': self.id_rsa_nopass, - 'id_rsa_withpass': self.id_rsa_withpass, - 'sshd_bin': self.sshd_bin, - 'sshd_port': self.sshd_port, - 'sshd_config_dir': self.sshd_config_dir, - 'master_user': self.master_opts['user'], - 'user': self.user, - 'uid': self.uid, - 'user_files': user_files}} - ) - - try: - for idx in range(1, self.sshd_wait + 1): - self.sshd_proc = self.find_sshd(sshd_config_file) - if self.sshd_proc is not None: - break - else: - if idx != self.sshd_wait: - log.debug( - 'Waiting for sshd process (%d of %d)', - idx, self.sshd_wait - ) - time.sleep(1) - else: - log.debug( - 'Failed fo find sshd process after %d seconds', - self.sshd_wait - ) - else: - raise Exception( - 'Unable to find an sshd process running from temp ' - 'config file {0} using psutil. Check to see if an ' - 'instance of sshd from an earlier aborted run of ' - 'these tests is running, if so then manually kill ' - 'it and re-run test(s).'.format(sshd_config_file) - ) - finally: - # Do the assert after we check for the PID so that we can track - # it regardless of whether or not something else in the SLS - # failed (but the SSH server still started). - self.assertSaltTrueReturn(ret) - - known_hosts_ret = self.run_function( - 'ssh.set_known_host', - user=self.master_opts['user'], - hostname='127.0.0.1', - port=self.sshd_port, - enc='ssh-rsa', - fingerprint='fd:6f:7f:5d:06:6b:f2:06:0d:26:93:9e:5a:b5:19:46', - hash_known_hosts=False, - ) - if 'error' in known_hosts_ret: - raise Exception( - 'Failed to add key to {0} user\'s known_hosts ' - 'file: {1}'.format( - self.master_opts['user'], - known_hosts_ret['error'] - ) - ) - - self.make_repo() - - def make_repo(self): - self.bare_repo = os.path.expanduser('~{0}/repo.git'.format(self.user)) - if self.bare_repo.startswith('~'): - self.bare_repo = None - self.fail( - 'Unable to resolve homedir for user \'{0}\''.format(self.user)) - - # Don't need to repeat the startswith check for this one, if we were - # unable to resolve the homedir here, we'd have aborted already. - self.admin_repo = os.path.expanduser('~{0}/admin_repo'.format(self.user)) - - for dirname in (self.bare_repo, self.admin_repo): - shutil.rmtree(dirname, ignore_errors=True) - - # Create bare repo - self.run_function( - 'git.init', - [self.bare_repo], - user=self.user, - bare=True) - - # Clone bare repo - self.run_function( - 'git.clone', - [self.admin_repo], - url=self.bare_repo, - user=self.user) - - def _push(branch, message): - self.run_function( - 'git.add', - [self.admin_repo, '.'], - user=self.user) - self.run_function( - 'git.commit', - [self.admin_repo, message], - user=self.user, - git_opts=self.git_opts, - ) - self.run_function( - 'git.push', - [self.admin_repo], - remote='origin', - ref=branch, - user=self.user, - ) - - with salt.utils.fopen( - os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_: - fp_.write(textwrap.dedent('''\ - base: - '*': - - foo - ''')) - with salt.utils.fopen( - os.path.join(self.admin_repo, 'foo.sls'), 'w') as fp_: - fp_.write(textwrap.dedent('''\ - branch: master - mylist: - - master - mydict: - master: True - nested_list: - - master - nested_dict: - master: True - ''')) - # Add another file to be referenced using git_pillar_includes - with salt.utils.fopen( - os.path.join(self.admin_repo, 'bar.sls'), 'w') as fp_: - fp_.write('included_pillar: True\n') - _push('master', 'initial commit') - - # Do the same with different values for "dev" branch - self.run_function( - 'git.checkout', - [self.admin_repo], - user=self.user, - opts='-b dev') - # The bar.sls shouldn't be in any branch but master - self.run_function( - 'git.rm', - [self.admin_repo, 'bar.sls'], - user=self.user) - with salt.utils.fopen( - os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_: - fp_.write(textwrap.dedent('''\ - dev: - '*': - - foo - ''')) - with salt.utils.fopen( - os.path.join(self.admin_repo, 'foo.sls'), 'w') as fp_: - fp_.write(textwrap.dedent('''\ - branch: dev - mylist: - - dev - mydict: - dev: True - nested_list: - - dev - nested_dict: - dev: True - ''')) - _push('dev', 'add dev branch') - - # Create just a top file in a separate repo, to be mapped to the base - # env and referenced using git_pillar_includes - self.run_function( - 'git.checkout', - [self.admin_repo], - user=self.user, - opts='-b top_only') - # The top.sls should be the only file in this branch - self.run_function( - 'git.rm', - [self.admin_repo, 'foo.sls'], - user=self.user) - with salt.utils.fopen( - os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_: - fp_.write(textwrap.dedent('''\ - base: - '*': - - bar - ''')) - _push('top_only', 'add top_only branch') - - def find_sshd(self, sshd_config_file): - for proc in psutil.process_iter(): - if 'sshd' in proc.name(): - if sshd_config_file in proc.cmdline(): - return proc - return None - - def get_pillar(self, ext_pillar_conf): - ''' - Run git_pillar with the specified configuration - ''' - cachedir = tempfile.mkdtemp(dir=integration.TMP) - self.addCleanup(shutil.rmtree, cachedir, ignore_errors=True) - ext_pillar_opts = yaml.safe_load( - ext_pillar_conf.format( - cachedir=cachedir, - extmods=os.path.join(cachedir, 'extmods'), - **self.ext_opts - ) - ) - with patch.dict(git_pillar.__opts__, ext_pillar_opts): - with patch.dict(git_pillar.__grains__, self.grains): - return git_pillar.ext_pillar( - 'minion', - ext_pillar_opts['ext_pillar'][0]['git'], - {} - ) +@http_basic_auth(lambda u, p: u == USERNAME and p == PASSWORD) # pylint: disable=W0223 +class HTTPBasicAuthHandler(tornado.web.StaticFileHandler): + pass @destructiveTest @skipIf(not salt.utils.which('sshd'), 'sshd not present') @skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) @skipIf(salt.utils.is_windows(), 'minion is windows') -@skipIf(os.getuid() != 0, 'must be root to run this test') +@skip_if_not_root @skipIf(NO_MOCK, NO_MOCK_REASON) class TestGitPythonSSH(SSHTestBase): ''' @@ -437,7 +70,9 @@ class TestGitPythonSSH(SSHTestBase): So, unlike pygit2, we don't need to test global or per-repo credential config params since GitPython doesn't use them. ''' - sshd_config_dir = tempfile.mkdtemp(dir=integration.TMP) + username = USERNAME + passphrase = PASSWORD + sshd_config_dir = tempfile.mkdtemp(dir=TMP) def get_pillar(self, ext_pillar_conf): ''' @@ -672,7 +307,7 @@ class TestGitPythonSSH(SSHTestBase): @skipIf(not salt.utils.which('sshd'), 'sshd not present') @skipIf(not HAS_PYGIT2, 'pygit2 >= {0} required'.format(PYGIT2_MINVER)) @skipIf(salt.utils.is_windows(), 'minion is windows') -@skipIf(os.getuid() != 0, 'must be root to run this test') +@skip_if_not_root @skipIf(NO_MOCK, NO_MOCK_REASON) class TestPygit2SSH(SSHTestBase): ''' @@ -681,7 +316,9 @@ class TestPygit2SSH(SSHTestBase): NOTE: Any tests added to this test class should have equivalent tests (if possible) in the TestGitPythonSSH class. ''' - sshd_config_dir = tempfile.mkdtemp(dir=integration.TMP) + username = USERNAME + passphrase = PASSWORD + sshd_config_dir = tempfile.mkdtemp(dir=TMP) def test_git_pillar_single_source(self): ''' @@ -1334,3 +971,59 @@ class TestPygit2SSH(SSHTestBase): - env: base ''') self.assertEqual(ret, expected) + + +@skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) +@skipIf(salt.utils.is_windows(), 'minion is windows') +@skip_if_not_root +@skipIf(NO_MOCK, NO_MOCK_REASON) +class TestGitPythonHTTP(HTTPTestBase): + ''' + Test git_pillar with GitPython using unauthenticated HTTP + + NOTE: Tests will have to wait until later for this, as the current method + set up in tests.support.git_pillar uses a tornado webserver, and to serve + Git over HTTP the setup needs to be a bit more complicated as we need the + webserver to forward the request to git on the "remote" server. + ''' + username = USERNAME + password = PASSWORD + root_dir = tempfile.mkdtemp(dir=TMP) + + +class TestGitPythonAuthenticatedHTTP(TestGitPythonHTTP): + ''' + Since GitPython doesn't support passing credentials, we can test + authenticated GitPython by encoding the username:password pair into the + repository's URL. The configuration will otherwise remain the same, so we + can reuse all of the tests from TestGitPythonHTTP. + + The same cannot be done for pygit2 however, since using authentication + requires that specific params are set in the YAML config, so the YAML we + use to drive the tests will be significantly different for authenticated + repositories. + ''' + root_dir = tempfile.mkdtemp(dir=TMP) + + @classmethod + def setUpClass(cls): + ''' + Create start the webserver + ''' + super(TestGitPythonAuthenticatedHTTP, cls).setUpClass() + # Override the URL set up in the parent class + cls.url = 'http://{username}:{password}@127.0.0.1:{port}/repo.git'.format( + username=cls.username, + password=cls.password, + port=cls.port) + cls.ext_opts['url'] = cls.url + + @classmethod + def create_webserver(cls): + ''' + Use HTTPBasicAuthHandler to force an auth prompt for these tests + ''' + if cls.root_dir is None: + raise Exception('root_dir not defined in test class') + return Webserver(root=cls.root_dir, port=cls.port, + handler=HTTPBasicAuthHandler) diff --git a/tests/integration/states/test_archive.py b/tests/integration/states/test_archive.py index 783d784fe5..fc7411ec84 100644 --- a/tests/integration/states/test_archive.py +++ b/tests/integration/states/test_archive.py @@ -7,16 +7,10 @@ from __future__ import absolute_import import errno import logging import os -import socket -import threading -import tornado.httpserver -import tornado.ioloop -import tornado.web # Import Salt Testing libs from tests.support.case import ModuleCase -from tests.support.paths import FILES -from tests.support.helpers import get_unused_localhost_port, skip_if_not_root +from tests.support.helpers import skip_if_not_root, Webserver from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs @@ -32,54 +26,21 @@ else: UNTAR_FILE = os.path.join(ARCHIVE_DIR, 'custom/README') ARCHIVE_TAR_HASH = 'md5=7643861ac07c30fe7d2310e9f25ca514' -STATE_DIR = os.path.join(FILES, 'file', 'base') class ArchiveTest(ModuleCase, SaltReturnAssertsMixin): ''' Validate the archive state ''' - @classmethod - def webserver(cls): - ''' - method to start tornado - static web app - ''' - cls._ioloop = tornado.ioloop.IOLoop() - cls._ioloop.make_current() - cls._application = tornado.web.Application([(r'/(.*)', tornado.web.StaticFileHandler, - {'path': STATE_DIR})]) - cls._application.listen(cls.server_port) - cls._ioloop.start() - @classmethod def setUpClass(cls): - ''' - start tornado app on thread - and wait till its running - ''' - cls.server_port = get_unused_localhost_port() - cls.server_thread = threading.Thread(target=cls.webserver) - cls.server_thread.daemon = True - cls.server_thread.start() - cls.archive_tar_source = 'http://localhost:{0}/custom.tar.gz'.format(cls.server_port) - # check if tornado app is up - port_closed = True - while port_closed: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex(('127.0.0.1', cls.server_port)) - if result == 0: - port_closed = False + cls.webserver = Webserver() + cls.webserver.start() + cls.archive_tar_source = cls.webserver.url('custom.tar.gz') @classmethod def tearDownClass(cls): - cls._ioloop.add_callback(cls._ioloop.stop) - cls.server_thread.join() - for attrname in ('_ioloop', '_application', 'server_thread'): - try: - delattr(cls, attrname) - except AttributeError: - continue + cls.webserver.stop() def setUp(self): self._clear_archive_dir() diff --git a/tests/integration/states/test_file.py b/tests/integration/states/test_file.py index c479c227e1..b140460019 100644 --- a/tests/integration/states/test_file.py +++ b/tests/integration/states/test_file.py @@ -13,14 +13,9 @@ import os import re import sys import shutil -import socket import stat import tempfile import textwrap -import threading -import tornado.httpserver -import tornado.ioloop -import tornado.web import filecmp log = logging.getLogger(__name__) @@ -29,7 +24,11 @@ log = logging.getLogger(__name__) from tests.support.case import ModuleCase from tests.support.unit import skipIf from tests.support.paths import FILES, TMP, TMP_STATE_TREE -from tests.support.helpers import skip_if_not_root, with_system_user_and_group +from tests.support.helpers import ( + skip_if_not_root, + with_system_user_and_group, + Webserver, +) from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs @@ -2402,49 +2401,22 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): if check_file: self.run_function('file.remove', [file]) -PORT = 9999 -FILE_SOURCE = 'http://localhost:{0}/grail/scene33'.format(PORT) -FILE_HASH = 'd2feb3beb323c79fc7a0f44f1408b4a3' - class RemoteFileTest(ModuleCase, SaltReturnAssertsMixin): ''' Uses a local tornado webserver to test http(s) file.managed states with and without skip_verify ''' - @classmethod - def webserver(cls): - ''' - method to start tornado static web app - ''' - application = tornado.web.Application([ - (r'/(.*)', tornado.web.StaticFileHandler, {'path': STATE_DIR}) - ]) - cls.server = tornado.httpserver.HTTPServer(application) - cls.server.listen(PORT) - tornado.ioloop.IOLoop.instance().start() - @classmethod def setUpClass(cls): - ''' - start tornado app on thread and wait until it is running - ''' - cls.server_thread = threading.Thread(target=cls.webserver) - cls.server_thread.daemon = True - cls.server_thread.start() - # check if tornado app is up - port_closed = True - while port_closed: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex(('127.0.0.1', PORT)) - if result == 0: - port_closed = False + cls.webserver = Webserver() + cls.webserver.start() + cls.source = cls.webserver.url('grail/scene33') + cls.source_hash = 'd2feb3beb323c79fc7a0f44f1408b4a3' @classmethod def tearDownClass(cls): - tornado.ioloop.IOLoop.instance().stop() - cls.server_thread.join() - cls.server.stop() + cls.webserver.stop() def setUp(self): fd_, self.name = tempfile.mkstemp(dir=TMP) @@ -2470,7 +2442,7 @@ class RemoteFileTest(ModuleCase, SaltReturnAssertsMixin): ''' ret = self.run_state('file.managed', name=self.name, - source=FILE_SOURCE, + source=self.source, skip_verify=False) log.debug('ret = %s', ret) # This should fail because no hash was provided @@ -2482,8 +2454,8 @@ class RemoteFileTest(ModuleCase, SaltReturnAssertsMixin): ''' ret = self.run_state('file.managed', name=self.name, - source=FILE_SOURCE, - source_hash=FILE_HASH, + source=self.source, + source_hash=self.source_hash, skip_verify=False) log.debug('ret = %s', ret) self.assertSaltTrueReturn(ret) @@ -2494,7 +2466,7 @@ class RemoteFileTest(ModuleCase, SaltReturnAssertsMixin): ''' ret = self.run_state('file.managed', name=self.name, - source=FILE_SOURCE, + source=self.source, skip_verify=True) log.debug('ret = %s', ret) self.assertSaltTrueReturn(ret) diff --git a/tests/support/git_pillar.py b/tests/support/git_pillar.py new file mode 100644 index 0000000000..cc2726f875 --- /dev/null +++ b/tests/support/git_pillar.py @@ -0,0 +1,448 @@ +# -*- coding: utf-8 -*- +''' +Base classes for git_pillar integration tests +''' + +# Import python libs +from __future__ import absolute_import +import errno +import logging +import os +import psutil +import random +import shutil +import string +import tempfile +import textwrap +import time +import yaml + +# Import Salt libs +import salt.utils +from salt.pillar import git_pillar +from salt.ext.six.moves import range # pylint: disable=redefined-builtin + +# Import Salt Testing libs +from tests.support.case import ModuleCase +from tests.support.mixins import LoaderModuleMockMixin, SaltReturnAssertsMixin +from tests.support.paths import FILES, TMP +from tests.support.helpers import ( + get_unused_localhost_port, + requires_system_grains, + Webserver, +) +from tests.support.mock import patch + +log = logging.getLogger(__name__) + + +def _rand_key_name(length): + return 'id_rsa_{0}'.format( + ''.join(random.choice(string.ascii_letters) for _ in range(length)) + ) + + +class GitPillarTestBase(ModuleCase, + LoaderModuleMockMixin, + SaltReturnAssertsMixin): + ''' + Base class for all git_pillar tests + ''' + case = port = bare_repo = admin_repo = None + maxDiff = None + git_opts = '-c user.name="Foo Bar" -c user.email=foo@bar.com' + ext_opts = {} + + @requires_system_grains + def setup_loader_modules(self, grains): # pylint: disable=W0221 + return { + git_pillar: { + '__opts__': { + '__role': 'minion', + 'environment': None, + 'pillarenv': None, + 'hash_type': 'sha256', + 'file_roots': {}, + 'state_top': 'top.sls', + 'state_top_saltenv': None, + 'renderer': 'yaml_jinja', + 'renderer_whitelist': [], + 'renderer_blacklist': [], + 'pillar_merge_lists': False, + 'git_pillar_base': 'master', + 'git_pillar_branch': 'master', + 'git_pillar_env': '', + 'git_pillar_root': '', + 'git_pillar_ssl_verify': True, + 'git_pillar_global_lock': True, + 'git_pillar_user': '', + 'git_pillar_password': '', + 'git_pillar_insecure_auth': False, + 'git_pillar_privkey': '', + 'git_pillar_pubkey': '', + 'git_pillar_passphrase': '', + 'git_pillar_refspecs': [ + '+refs/heads/*:refs/remotes/origin/*', + '+refs/tags/*:refs/tags/*', + ], + 'git_pillar_includes': True, + }, + '__grains__': grains, + } + } + + @classmethod + def update_class(cls, case): + ''' + Make the test class available to the tearDownClass. Note that this + cannot be defined in a parent class and inherited, as this will cause + the parent class to be modified. + ''' + if getattr(cls, 'case') is None: + setattr(cls, 'case', case) + + @classmethod + def setUpClass(cls): + cls.port = get_unused_localhost_port() + + def setUp(self): + # Make the test class available to the tearDownClass so we can clean up + # after ourselves. This (and the gated block below) prevent us from + # needing to spend the extra time creating an ssh server and user and + # then tear them down separately for each test. + self.update_class(self) + + def get_pillar(self, ext_pillar_conf): + ''' + Run git_pillar with the specified configuration + ''' + cachedir = tempfile.mkdtemp(dir=TMP) + self.addCleanup(shutil.rmtree, cachedir, ignore_errors=True) + ext_pillar_opts = yaml.safe_load( + ext_pillar_conf.format( + cachedir=cachedir, + extmods=os.path.join(cachedir, 'extmods'), + **self.ext_opts + ) + ) + with patch.dict(git_pillar.__opts__, ext_pillar_opts): + return git_pillar.ext_pillar( + 'minion', + ext_pillar_opts['ext_pillar'][0]['git'], + {} + ) + + def make_repo(self, root_dir, user='root'): + self.bare_repo = os.path.join(root_dir, 'repo.git') + self.admin_repo = os.path.join(root_dir, 'admin') + + for dirname in (self.bare_repo, self.admin_repo): + shutil.rmtree(dirname, ignore_errors=True) + + # Create bare repo + self.run_function( + 'git.init', + [self.bare_repo], + user=user, + bare=True) + + # Clone bare repo + self.run_function( + 'git.clone', + [self.admin_repo], + url=self.bare_repo, + user=user) + + def _push(branch, message): + self.run_function( + 'git.add', + [self.admin_repo, '.'], + user=user) + self.run_function( + 'git.commit', + [self.admin_repo, message], + user=user, + git_opts=self.git_opts, + ) + self.run_function( + 'git.push', + [self.admin_repo], + remote='origin', + ref=branch, + user=user, + ) + + with salt.utils.fopen( + os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_: + fp_.write(textwrap.dedent('''\ + base: + '*': + - foo + ''')) + with salt.utils.fopen( + os.path.join(self.admin_repo, 'foo.sls'), 'w') as fp_: + fp_.write(textwrap.dedent('''\ + branch: master + mylist: + - master + mydict: + master: True + nested_list: + - master + nested_dict: + master: True + ''')) + # Add another file to be referenced using git_pillar_includes + with salt.utils.fopen( + os.path.join(self.admin_repo, 'bar.sls'), 'w') as fp_: + fp_.write('included_pillar: True\n') + _push('master', 'initial commit') + + # Do the same with different values for "dev" branch + self.run_function( + 'git.checkout', + [self.admin_repo], + user=user, + opts='-b dev') + # The bar.sls shouldn't be in any branch but master + self.run_function( + 'git.rm', + [self.admin_repo, 'bar.sls'], + user=user) + with salt.utils.fopen( + os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_: + fp_.write(textwrap.dedent('''\ + dev: + '*': + - foo + ''')) + with salt.utils.fopen( + os.path.join(self.admin_repo, 'foo.sls'), 'w') as fp_: + fp_.write(textwrap.dedent('''\ + branch: dev + mylist: + - dev + mydict: + dev: True + nested_list: + - dev + nested_dict: + dev: True + ''')) + _push('dev', 'add dev branch') + + # Create just a top file in a separate repo, to be mapped to the base + # env and referenced using git_pillar_includes + self.run_function( + 'git.checkout', + [self.admin_repo], + user=user, + opts='-b top_only') + # The top.sls should be the only file in this branch + self.run_function( + 'git.rm', + [self.admin_repo, 'foo.sls'], + user=user) + with salt.utils.fopen( + os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_: + fp_.write(textwrap.dedent('''\ + base: + '*': + - bar + ''')) + _push('top_only', 'add top_only branch') + + +class HTTPTestBase(GitPillarTestBase): + ''' + Base class for GitPython and Pygit2 HTTP tests + + NOTE: root_dir must be overridden in a subclass + ''' + goot_dir = None + + @classmethod + def setUpClass(cls): + ''' + Create start the webserver + ''' + super(HTTPTestBase, cls).setUpClass() + cls.webserver = cls.create_webserver() + cls.webserver.start() + cls.url = 'http://127.0.0.1:{port}/repo.git'.format(port=cls.port) + cls.ext_opts = { + 'url': cls.url, + 'username': cls.username, + 'password': cls.password} + + @classmethod + def tearDownClass(cls): + ''' + Stop the webserver and cleanup the repo + ''' + cls.webserver.stop() + shutil.rmtree(cls.root_dir, ignore_errors=True) + + @classmethod + def create_webserver(cls): + ''' + Override this in a subclass with the handler argument to use a custom + handler for HTTP Basic Authentication + ''' + if cls.root_dir is None: + raise Exception('root_dir not defined in test class') + return Webserver(root=cls.root_dir, port=cls.port) + + def setUp(self): + ''' + Create and start the webserver, and create the git repo + ''' + super(HTTPTestBase, self).setUp() + self.make_repo(self.root_dir) + + +class SSHTestBase(GitPillarTestBase): + ''' + Base class for GitPython and Pygit2 SSH tests + ''' + # Define a few variables and set to None so they're not culled in the + # cleanup when the test function completes, and remain available to the + # tearDownClass. + sshd_proc = None + # Creates random key names to (hopefully) ensure we're not overwriting an + # existing key in /root/.ssh. Even though these are destructive tests, we + # don't want to mess with something as important as ssh. + id_rsa_nopass = _rand_key_name(8) + id_rsa_withpass = _rand_key_name(8) + sshd_wait = 10 + + @classmethod + def setUpClass(cls): + super(SSHTestBase, cls).setUpClass() + cls.url = 'ssh://{username}@127.0.0.1:{port}/~/repo.git'.format( + username=cls.username, + port=cls.port) + home = '/root/.ssh' + cls.ext_opts = { + 'url': cls.url, + 'privkey_nopass': os.path.join(home, cls.id_rsa_nopass), + 'pubkey_nopass': os.path.join(home, cls.id_rsa_nopass + '.pub'), + 'privkey_withpass': os.path.join(home, cls.id_rsa_withpass), + 'pubkey_withpass': os.path.join(home, cls.id_rsa_withpass + '.pub'), + 'passphrase': cls.passphrase} + + @classmethod + def tearDownClass(cls): + ''' + Stop the SSH server, remove the user, and clean up the config dir + ''' + if cls.case.sshd_proc: + try: + cls.case.sshd_proc.kill() + except psutil.NoSuchProcess: + pass + cls.case.run_state('user.absent', name=cls.username, purge=True) + for dirname in (cls.sshd_config_dir, cls.case.admin_repo, + cls.case.bare_repo): + if dirname is not None: + shutil.rmtree(dirname, ignore_errors=True) + ssh_dir = os.path.expanduser('~/.ssh') + for filename in (cls.id_rsa_nopass, cls.id_rsa_withpass, cls.case.git_ssh): + try: + os.remove(os.path.join(ssh_dir, filename)) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + + def setUp(self): + ''' + Create the SSH server and user, and create the git repo + ''' + super(SSHTestBase, self).setUp() + sshd_config_file = os.path.join(self.sshd_config_dir, 'sshd_config') + self.sshd_proc = self.find_sshd(sshd_config_file) + self.sshd_bin = salt.utils.which('sshd') + self.git_ssh = '/tmp/git_ssh' + + if self.sshd_proc is None: + user_files = os.listdir( + os.path.join(FILES, 'file/base/git_pillar/ssh/user/files') + ) + ret = self.run_function( + 'state.apply', + mods='git_pillar.ssh', + pillar={'git_pillar': {'git_ssh': self.git_ssh, + 'id_rsa_nopass': self.id_rsa_nopass, + 'id_rsa_withpass': self.id_rsa_withpass, + 'sshd_bin': self.sshd_bin, + 'sshd_port': self.port, + 'sshd_config_dir': self.sshd_config_dir, + 'master_user': self.master_opts['user'], + 'user': self.username, + 'user_files': user_files}} + ) + + try: + for idx in range(1, self.sshd_wait + 1): + self.sshd_proc = self.find_sshd(sshd_config_file) + if self.sshd_proc is not None: + break + else: + if idx != self.sshd_wait: + log.debug( + 'Waiting for sshd process (%d of %d)', + idx, self.sshd_wait + ) + time.sleep(1) + else: + log.debug( + 'Failed fo find sshd process after %d seconds', + self.sshd_wait + ) + else: + raise Exception( + 'Unable to find an sshd process running from temp ' + 'config file {0} using psutil. Check to see if an ' + 'instance of sshd from an earlier aborted run of ' + 'these tests is running, if so then manually kill ' + 'it and re-run test(s).'.format(sshd_config_file) + ) + finally: + # Do the assert after we check for the PID so that we can track + # it regardless of whether or not something else in the SLS + # failed (but the SSH server still started). + self.assertSaltTrueReturn(ret) + + known_hosts_ret = self.run_function( + 'ssh.set_known_host', + user=self.master_opts['user'], + hostname='127.0.0.1', + port=self.port, + enc='ssh-rsa', + fingerprint='fd:6f:7f:5d:06:6b:f2:06:0d:26:93:9e:5a:b5:19:46', + hash_known_hosts=False, + ) + if 'error' in known_hosts_ret: + raise Exception( + 'Failed to add key to {0} user\'s known_hosts ' + 'file: {1}'.format( + self.master_opts['user'], + known_hosts_ret['error'] + ) + ) + + root_dir = os.path.expanduser('~{0}'.format(self.username)) + if root_dir.startswith('~'): + self.fail( + 'Unable to resolve homedir for user \'{0}\''.format( + self.username + ) + ) + self.make_repo(root_dir, user=self.username) + + def find_sshd(self, sshd_config_file): + for proc in psutil.process_iter(): + if 'sshd' in proc.name(): + if sshd_config_file in proc.cmdline(): + return proc + return None diff --git a/tests/support/helpers.py b/tests/support/helpers.py index af78f11bfc..cae422c0f9 100644 --- a/tests/support/helpers.py +++ b/tests/support/helpers.py @@ -13,16 +13,20 @@ # Import Python libs from __future__ import absolute_import -import os -import sys -import time +import base64 import errno -import types -import signal -import socket +import functools import inspect import logging -import functools +import os +import signal +import socket +import sys +import threading +import time +import tornado.ioloop +import tornado.web +import types # Import 3rd-party libs import psutil # pylint: disable=3rd-party-module-not-gated @@ -44,6 +48,7 @@ except ImportError: # Import Salt Tests Support libs from tests.support.unit import skip, _id from tests.support.mock import patch +from tests.support.paths import FILES log = logging.getLogger(__name__) @@ -1272,3 +1277,180 @@ def repeat(caller=None, condition=True, times=5): caller(cls) return cls return wrap + + +def http_basic_auth(login_cb=lambda username, password: False): + ''' + A crude decorator to force a handler to request HTTP Basic Authentication + + Example usage: + + .. code-block:: python + + @http_basic_auth(lambda u, p: u == 'foo' and p == 'bar') + class AuthenticatedHandler(tornado.web.RequestHandler): + pass + ''' + def wrapper(handler_class): + def wrap_execute(handler_execute): + def check_auth(handler, kwargs): + + auth = handler.request.headers.get('Authorization') + + if auth is None or not auth.startswith('Basic '): + # No username/password entered yet, we need to return a 401 + # and set the WWW-Authenticate header to request login. + handler.set_status(401) + handler.set_header( + 'WWW-Authenticate', 'Basic realm=Restricted') + + else: + # Strip the 'Basic ' from the beginning of the auth header + # leaving the base64-encoded secret + username, password = \ + base64.b64decode(auth[6:]).split(':', 1) + + if login_cb(username, password): + # Authentication successful + return + else: + # Authentication failed + handler.set_status(403) + + handler._transforms = [] + handler.finish() + + def _execute(self, transforms, *args, **kwargs): + check_auth(self, kwargs) + return handler_execute(self, transforms, *args, **kwargs) + + return _execute + + handler_class._execute = wrap_execute(handler_class._execute) + return handler_class + return wrapper + + +class Webserver(object): + ''' + Starts a tornado webserver on 127.0.0.1 on a random available port + + USAGE: + + .. code-block:: python + + from tests.support.helpers import Webserver + + webserver = Webserver('/path/to/web/root') + webserver.start() + webserver.stop() + ''' + def __init__(self, + root=None, + port=None, + wait=5, + handler=None): + ''' + root + Root directory of webserver. If not passed, it will default to the + location of the base environment of the integration suite's file + roots (tests/integration/files/file/base/) + + port + Port on which to listen. If not passed, a random one will be chosen + at the time the start() function is invoked. + + wait : 5 + Number of seconds to wait for the socket to be open before raising + an exception + + handler + Can be used to use a subclass of tornado.web.StaticFileHandler, + such as when enforcing authentication with the http_basic_auth + decorator. + ''' + if port is not None and not isinstance(port, six.integer_types): + raise ValueError('port must be an integer') + + if root is None: + root = os.path.join(FILES, 'file', 'base') + try: + self.root = os.path.realpath(root) + except AttributeError: + raise ValueError('root must be a string') + + self.port = port + self.wait = wait + self.handler = handler \ + if handler is not None \ + else tornado.web.StaticFileHandler + self.web_root = None + + def target(self): + ''' + Threading target which stands up the tornado application + ''' + self.ioloop = tornado.ioloop.IOLoop() + self.ioloop.make_current() + self.application = tornado.web.Application( + [(r'/(.*)', self.handler, {'path': self.root})]) + self.application.listen(self.port) + self.ioloop.start() + + @property + def listening(self): + if self.port is None: + return False + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + return sock.connect_ex(('127.0.0.1', self.port)) == 0 + + def url(self, path): + ''' + Convenience function which, given a file path, will return a URL that + points to that path. If the path is relative, it will just be appended + to self.web_root. + ''' + if self.web_root is None: + raise RuntimeError('Webserver instance has not been started') + err_msg = 'invalid path, must be either a relative path or a path ' \ + 'within {0}'.format(self.root) + try: + relpath = path \ + if not os.path.isabs(path) \ + else os.path.relpath(path, self.root) + if relpath.startswith('..' + os.sep): + raise ValueError(err_msg) + return '/'.join((self.web_root, relpath)) + except AttributeError: + raise ValueError(err_msg) + + def start(self): + ''' + Starts the webserver + ''' + if self.port is None: + self.port = get_unused_localhost_port() + + self.web_root = 'http://127.0.0.1:{0}'.format(self.port) + + self.server_thread = threading.Thread(target=self.target) + self.server_thread.daemon = True + self.server_thread.start() + + for idx in range(self.wait + 1): + if self.listening: + break + if idx != self.wait: + time.sleep(1) + else: + raise Exception( + 'Failed to start tornado webserver on 127.0.0.1:{0} within ' + '{1} seconds'.format(self.port, self.wait) + ) + + def stop(self): + ''' + Stops the webserver + ''' + self.ioloop.add_callback(self.ioloop.stop) + self.server_thread.join() From 75e6bc0aa325a57a7269eec122b94f58afa9c989 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 26 Apr 2017 00:03:19 -0500 Subject: [PATCH 04/19] Fix two issues with pip.install 1. The iteritems func from six was being imported directly into the module, which means that the loader picks it up and we end up with a pip.iteritems in the __salt__ dunder. 2. When env_vars is passed, it permanently modifies os.environ, when it should just be passing the values in the env argument to cmd.run_all. This fixes both of these issues. --- salt/modules/pip.py | 48 ++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index e195b7086a..143da321b8 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -87,7 +87,7 @@ import salt.utils import tempfile import salt.utils.locales import salt.utils.url -from salt.ext.six import string_types, iteritems +from salt.ext import six from salt.exceptions import CommandExecutionError, CommandNotFoundError @@ -213,7 +213,7 @@ def _resolve_requirements_chain(requirements): chain = [] - if isinstance(requirements, string_types): + if isinstance(requirements, six.string_types): requirements = [requirements] for req_file in requirements: @@ -230,7 +230,7 @@ def _process_requirements(requirements, cmd, cwd, saltenv, user): cleanup_requirements = [] if requirements is not None: - if isinstance(requirements, string_types): + if isinstance(requirements, six.string_types): requirements = [r.strip() for r in requirements.split(',')] elif not isinstance(requirements, list): raise TypeError('requirements must be a string or list') @@ -580,7 +580,7 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 'that virtualenv before using pip to install packages in it.' ) - if isinstance(__env__, string_types): + if isinstance(__env__, six.string_types): salt.utils.warn_until( 'Carbon', 'Passing a salt environment should be done using \'saltenv\' ' @@ -658,7 +658,7 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 cmd.extend(['--timeout', timeout]) if find_links: - if isinstance(find_links, string_types): + if isinstance(find_links, six.string_types): find_links = [l.strip() for l in find_links.split(',')] for link in find_links: @@ -700,7 +700,7 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 ' use index_url and/or extra_index_url instead' ) - if isinstance(mirrors, string_types): + if isinstance(mirrors, six.string_types): mirrors = [m.strip() for m in mirrors.split(',')] cmd.append('--use-mirrors') @@ -764,21 +764,21 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 cmd.extend(['--cert', cert]) if global_options: - if isinstance(global_options, string_types): + if isinstance(global_options, six.string_types): global_options = [go.strip() for go in global_options.split(',')] for opt in global_options: cmd.extend(['--global-option', opt]) if install_options: - if isinstance(install_options, string_types): + if isinstance(install_options, six.string_types): install_options = [io.strip() for io in install_options.split(',')] for opt in install_options: cmd.extend(['--install-option', opt]) if pkgs: - if isinstance(pkgs, string_types): + if isinstance(pkgs, six.string_types): pkgs = [p.strip() for p in pkgs.split(',')] # It's possible we replaced version-range commas with semicolons so @@ -789,7 +789,7 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 if editable: egg_match = re.compile(r'(?:#|#.*?&)egg=([^&]*)') - if isinstance(editable, string_types): + if isinstance(editable, six.string_types): editable = [e.strip() for e in editable.split(',')] for entry in editable: @@ -808,14 +808,14 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 cmd.append('--allow-all-external') if allow_external: - if isinstance(allow_external, string_types): + if isinstance(allow_external, six.string_types): allow_external = [p.strip() for p in allow_external.split(',')] for pkg in allow_external: cmd.extend(['--allow-external', pkg]) if allow_unverified: - if isinstance(allow_unverified, string_types): + if isinstance(allow_unverified, six.string_types): allow_unverified = \ [p.strip() for p in allow_unverified.split(',')] @@ -825,27 +825,27 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 if process_dependency_links: cmd.append('--process-dependency-links') + if trusted_host: + cmd.extend(['--trusted-host', trusted_host]) + + cmd_kwargs = dict(saltenv=saltenv, use_vt=use_vt, runas=user) + if env_vars: if isinstance(env_vars, dict): - for k, v in iteritems(env_vars): - if not isinstance(v, string_types): - env_vars[k] = str(v) - os.environ.update(env_vars) + for key, val in six.iteritems(env_vars): + if not isinstance(val, six.string_types): + val = str(val) + cmd_kwargs.setdefault('env', {})[key] = val else: raise CommandExecutionError( 'env_vars {0} is not a dictionary'.format(env_vars)) - if trusted_host: - cmd.extend(['--trusted-host', trusted_host]) - try: - cmd_kwargs = dict(saltenv=saltenv, use_vt=use_vt, runas=user) - if cwd: cmd_kwargs['cwd'] = cwd if bin_env and os.path.isdir(bin_env): - cmd_kwargs['env'] = {'VIRTUAL_ENV': bin_env} + cmd_kwargs.setdefault('env', {})['VIRTUAL_ENV'] = bin_env logger.debug( 'TRY BLOCK: end of pip.install -- cmd: %s, cmd_kwargs: %s', @@ -926,7 +926,7 @@ def uninstall(pkgs=None, cmd = [pip_bin, 'uninstall', '-y'] - if isinstance(__env__, string_types): + if isinstance(__env__, six.string_types): salt.utils.warn_until( 'Carbon', 'Passing a salt environment should be done using \'saltenv\' ' @@ -971,7 +971,7 @@ def uninstall(pkgs=None, cmd.extend(['--timeout', timeout]) if pkgs: - if isinstance(pkgs, string_types): + if isinstance(pkgs, six.string_types): pkgs = [p.strip() for p in pkgs.split(',')] if requirements: for requirement in requirements: From 4f9c92a012550175efbdc842e12cfe9ac525ef06 Mon Sep 17 00:00:00 2001 From: rkgrunt Date: Wed, 26 Apr 2017 14:38:14 -0400 Subject: [PATCH 05/19] Fixed issue with parsing of master minion returns when batching is enabled. --- salt/states/saltmod.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/states/saltmod.py b/salt/states/saltmod.py index a9d1f6be93..35cd01fb4e 100644 --- a/salt/states/saltmod.py +++ b/salt/states/saltmod.py @@ -300,7 +300,7 @@ def state( except KeyError: m_state = False if m_state: - m_state = salt.utils.check_state_result(m_ret) + m_state = salt.utils.check_state_result(m_ret, recurse=True) if not m_state: if minion not in fail_minions: @@ -309,9 +309,10 @@ def state( continue try: for state_item in six.itervalues(m_ret): - if 'changes' in state_item and state_item['changes']: - changes[minion] = m_ret - break + if isinstance(state_item, dict): + if 'changes' in state_item and state_item['changes']: + changes[minion] = m_ret + break else: no_change.add(minion) except AttributeError: From 9e6361c6c8d20aae1f884e93530de8e7cf6ad192 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 27 Apr 2017 00:48:24 -0500 Subject: [PATCH 06/19] Add GitPython HTTP git_pillar tests --- .../base/git_pillar/http/files/nginx.conf | 40 ++ .../file/base/git_pillar/http/files/users | 1 + .../file/base/git_pillar/http/files/uwsgi.yml | 13 + .../files/file/base/git_pillar/http/init.sls | 62 +++ tests/integration/pillar/test_git_pillar.py | 179 +++---- tests/support/git_pillar.py | 476 +++++++++++------- 6 files changed, 484 insertions(+), 287 deletions(-) create mode 100644 tests/integration/files/file/base/git_pillar/http/files/nginx.conf create mode 100644 tests/integration/files/file/base/git_pillar/http/files/users create mode 100644 tests/integration/files/file/base/git_pillar/http/files/uwsgi.yml create mode 100644 tests/integration/files/file/base/git_pillar/http/init.sls diff --git a/tests/integration/files/file/base/git_pillar/http/files/nginx.conf b/tests/integration/files/file/base/git_pillar/http/files/nginx.conf new file mode 100644 index 0000000000..c8ad0debe3 --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/http/files/nginx.conf @@ -0,0 +1,40 @@ +worker_processes 1; +error_log {{ pillar['git_pillar']['config_dir'] }}/error.log; +pid {{ pillar['git_pillar']['config_dir'] }}/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + access_log {{ pillar['git_pillar']['config_dir'] }}/git_access.log; + error_log {{ pillar['git_pillar']['config_dir'] }}/git_error.log; + + #sendfile on; + #keepalive_timeout 65; + + server { + listen {{ pillar['git_pillar']['nginx_port'] }} default_server; + server_name git.local; + root {{ pillar['git_pillar']['git_dir'] }}/repos; + + location / { +{%- if salt['pillar.get']('git_pillar:auth_enabled', False) %} + auth_basic "YOU... SHALL NOT... PASS!!!"; + auth_basic_user_file {{ pillar['git_pillar']['git_dir'] }}/users; +{%- endif %} + + include /etc/nginx/uwsgi_params; + + uwsgi_param GIT_HTTP_EXPORT_ALL ""; + uwsgi_param GIT_PROJECT_ROOT {{ pillar['git_pillar']['git_dir'] }}/repos; + uwsgi_param REMOTE_USER $remote_user; + + uwsgi_modifier1 9; + uwsgi_pass 127.0.0.1:{{ pillar['git_pillar']['uwsgi_port'] }}; + } + } +} diff --git a/tests/integration/files/file/base/git_pillar/http/files/users b/tests/integration/files/file/base/git_pillar/http/files/users new file mode 100644 index 0000000000..cb1102e0d6 --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/http/files/users @@ -0,0 +1 @@ +gitpillaruser:$apr1$H2XscjOs$EY8ZeOFX2NqR3XVsOEUM71 diff --git a/tests/integration/files/file/base/git_pillar/http/files/uwsgi.yml b/tests/integration/files/file/base/git_pillar/http/files/uwsgi.yml new file mode 100644 index 0000000000..97f95a489d --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/http/files/uwsgi.yml @@ -0,0 +1,13 @@ +uwsgi: + socket: 127.0.0.1:{{ pillar['git_pillar']['uwsgi_port'] }} + cgi: {{ salt['pillar.get']('git_pillar:libexec_dir', '/usr/libexec') }}/git-core/git-http-backend + chdir: %d + daemonize: {{ pillar['git_pillar']['config_dir'] }}/uwsgi.log + pidfile: {{ pillar['git_pillar']['config_dir'] }}/uwsgi.pid + # This is required to work around a bug in git-http-backend, introduced in + # git 2.4.4 and worked around with cgi-close-stdin-on-eof in uwsgi >= 2.0.13. + # + # See: + # https://github.com/git/git/commit/6bc0cb5 + # https://github.com/unbit/uwsgi/commit/ac1e354 + cgi-close-stdin-on-eof: 1 diff --git a/tests/integration/files/file/base/git_pillar/http/init.sls b/tests/integration/files/file/base/git_pillar/http/init.sls new file mode 100644 index 0000000000..595fda9d2c --- /dev/null +++ b/tests/integration/files/file/base/git_pillar/http/init.sls @@ -0,0 +1,62 @@ +{%- set config_dir = pillar['git_pillar']['config_dir'] %} +{%- set git_dir = pillar['git_pillar']['git_dir'] %} +{%- set venv_dir = pillar['git_pillar']['venv_dir'] %} +{%- set root_dir = pillar['git_pillar']['root_dir'] %} + +{{ config_dir }}/nginx.conf: + file.managed: + - source: salt://git_pillar/http/files/nginx.conf + - user: root + - group: root + - mode: 644 + - makedirs: True + - template: jinja + +{{ config_dir }}/uwsgi.yml: + file.managed: + - source: salt://git_pillar/http/files/uwsgi.yml + - user: root + - group: root + - mode: 644 + - makedirs: True + - template: jinja + +{{ root_dir }}: + file.directory: + - user: root + - group: root + - mode: 755 + +{{ git_dir }}/users: + file.managed: + - source: salt://git_pillar/http/files/users + - user: root + - group: root + - makedirs: True + - mode: 644 + +{{ venv_dir }}: + virtualenv.managed: + - system_site_packages: False + +uwsgi: + pip.installed: + - name: 'uwsgi >= 2.0.13' + - bin_env: {{ venv_dir }} + - env_vars: + UWSGI_PROFILE: cgi + - require: + - virtualenv: {{ venv_dir }} + +start_uwsgi: + cmd.run: + - name: '{{ venv_dir }}/bin/uwsgi --yaml {{ config_dir }}/uwsgi.yml' + - require: + - pip: uwsgi + - file: {{ config_dir }}/uwsgi.yml + +start_nginx: + cmd.run: + - name: 'nginx -c {{ config_dir }}/nginx.conf' + - require: + - file: {{ config_dir }}/nginx.conf diff --git a/tests/integration/pillar/test_git_pillar.py b/tests/integration/pillar/test_git_pillar.py index 46b8f52816..a6f73297cc 100644 --- a/tests/integration/pillar/test_git_pillar.py +++ b/tests/integration/pillar/test_git_pillar.py @@ -17,7 +17,7 @@ from tests.support.helpers import ( skip_if_not_root, Webserver, ) -from tests.support.git_pillar import HTTPTestBase, SSHTestBase +from tests.support.git_pillar import GitPillarSSHTestBase, GitPillarHTTPTestBase from tests.support.paths import TMP from tests.support.unit import skipIf from tests.support.mock import NO_MOCK, NO_MOCK_REASON @@ -40,56 +40,15 @@ try: except ImportError: HAS_PYGIT2 = False -NOTSET = object() USERNAME = 'gitpillaruser' PASSWORD = 'saltrules' -@http_basic_auth(lambda u, p: u == USERNAME and p == PASSWORD) # pylint: disable=W0223 -class HTTPBasicAuthHandler(tornado.web.StaticFileHandler): - pass - - -@destructiveTest -@skipIf(not salt.utils.which('sshd'), 'sshd not present') -@skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) -@skipIf(salt.utils.is_windows(), 'minion is windows') -@skip_if_not_root -@skipIf(NO_MOCK, NO_MOCK_REASON) -class TestGitPythonSSH(SSHTestBase): +class GitPythonMixin(object): ''' - Test git_pillar with GitPython using SSH authentication - - NOTE: Any tests added to this test class should have equivalent tests (if - possible) in the TestPygit2SSH class. Also, bear in mind that the pygit2 - versions of these tests need to be more complex in that they need to test - both with passphraseless and passphrase-protecteed keys, both with global - and per-remote configuration. So for every time we run a GitPython test, we - need to run that same test four different ways for pygit2. This is because - GitPython's ability to use git-over-SSH is limited to passphraseless keys. - So, unlike pygit2, we don't need to test global or per-repo credential - config params since GitPython doesn't use them. + GitPython doesn't support anything fancy in terms of authentication + options, so all of the tests for GitPython can be re-used via this mixin. ''' - username = USERNAME - passphrase = PASSWORD - sshd_config_dir = tempfile.mkdtemp(dir=TMP) - - def get_pillar(self, ext_pillar_conf): - ''' - Wrap the parent class' get_pillar() func in logic that temporarily - changes the GIT_SSH to use our custom script, ensuring that the - passphraselsess key is used to auth without needing to modify the root - user's ssh config file. - ''' - orig_git_ssh = os.environ.pop('GIT_SSH', NOTSET) - os.environ['GIT_SSH'] = self.git_ssh - try: - return super(TestGitPythonSSH, self).get_pillar(ext_pillar_conf) - finally: - os.environ.pop('GIT_SSH', None) - if orig_git_ssh is not NOTSET: - os.environ['GIT_SSH'] = orig_git_ssh - def test_git_pillar_single_source(self): ''' Test using a single ext_pillar repo @@ -303,13 +262,85 @@ class TestGitPythonSSH(SSHTestBase): ) +@destructiveTest +@skipIf(not salt.utils.which('sshd'), 'sshd not present') +@skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) +@skipIf(salt.utils.is_windows(), 'minion is windows') +@skip_if_not_root +@skipIf(NO_MOCK, NO_MOCK_REASON) +class TestGitPythonSSH(GitPillarSSHTestBase, GitPythonMixin): + ''' + Test git_pillar with GitPython using SSH authentication + + NOTE: Any tests added to this test class should have equivalent tests (if + possible) in the TestPygit2SSH class. Also, bear in mind that the pygit2 + versions of these tests need to be more complex in that they need to test + both with passphraseless and passphrase-protecteed keys, both with global + and per-remote configuration. So for every time we run a GitPython test, we + need to run that same test four different ways for pygit2. This is because + GitPython's ability to use git-over-SSH is limited to passphraseless keys. + So, unlike pygit2, we don't need to test global or per-repo credential + config params since GitPython doesn't use them. + ''' + sshd_config_dir = tempfile.mkdtemp(dir=TMP) + username = USERNAME + passphrase = PASSWORD + + +@skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) +@skipIf(salt.utils.is_windows(), 'minion is windows') +@skip_if_not_root +@skipIf(NO_MOCK, NO_MOCK_REASON) +class TestGitPythonHTTP(GitPillarHTTPTestBase, GitPythonMixin): + ''' + Test git_pillar with GitPython using unauthenticated HTTP + ''' + root_dir = tempfile.mkdtemp(dir=TMP) + + +@skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) +@skipIf(salt.utils.is_windows(), 'minion is windows') +@skip_if_not_root +@skipIf(NO_MOCK, NO_MOCK_REASON) +class TestGitPythonAuthenticatedHTTP(TestGitPythonHTTP, GitPythonMixin): + ''' + Since GitPython doesn't support passing credentials, we can test + authenticated GitPython by encoding the username:password pair into the + repository's URL. The configuration will otherwise remain the same, so we + can reuse all of the tests from TestGitPythonHTTP. + + The same cannot be done for pygit2 however, since using authentication + requires that specific params are set in the YAML config, so the YAML we + use to drive the tests will be significantly different for authenticated + repositories. + ''' + root_dir = tempfile.mkdtemp(dir=TMP) + username = USERNAME + password = PASSWORD + + @classmethod + def setUpClass(cls): + ''' + Create start the webserver + ''' + super(TestGitPythonAuthenticatedHTTP, cls).setUpClass() + # Override the URL set up in the parent class + cls.url = 'http://{username}:{password}@127.0.0.1:{port}/repo.git'.format( + username=cls.username, + password=cls.password, + port=cls.nginx_port) + cls.ext_opts['url'] = cls.url + cls.ext_opts['username'] = cls.username + cls.ext_opts['password'] = cls.password + + @destructiveTest @skipIf(not salt.utils.which('sshd'), 'sshd not present') @skipIf(not HAS_PYGIT2, 'pygit2 >= {0} required'.format(PYGIT2_MINVER)) @skipIf(salt.utils.is_windows(), 'minion is windows') @skip_if_not_root @skipIf(NO_MOCK, NO_MOCK_REASON) -class TestPygit2SSH(SSHTestBase): +class TestPygit2SSH(GitPillarSSHTestBase): ''' Test git_pillar with pygit2 using SSH authentication @@ -971,59 +1002,3 @@ class TestPygit2SSH(SSHTestBase): - env: base ''') self.assertEqual(ret, expected) - - -@skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) -@skipIf(salt.utils.is_windows(), 'minion is windows') -@skip_if_not_root -@skipIf(NO_MOCK, NO_MOCK_REASON) -class TestGitPythonHTTP(HTTPTestBase): - ''' - Test git_pillar with GitPython using unauthenticated HTTP - - NOTE: Tests will have to wait until later for this, as the current method - set up in tests.support.git_pillar uses a tornado webserver, and to serve - Git over HTTP the setup needs to be a bit more complicated as we need the - webserver to forward the request to git on the "remote" server. - ''' - username = USERNAME - password = PASSWORD - root_dir = tempfile.mkdtemp(dir=TMP) - - -class TestGitPythonAuthenticatedHTTP(TestGitPythonHTTP): - ''' - Since GitPython doesn't support passing credentials, we can test - authenticated GitPython by encoding the username:password pair into the - repository's URL. The configuration will otherwise remain the same, so we - can reuse all of the tests from TestGitPythonHTTP. - - The same cannot be done for pygit2 however, since using authentication - requires that specific params are set in the YAML config, so the YAML we - use to drive the tests will be significantly different for authenticated - repositories. - ''' - root_dir = tempfile.mkdtemp(dir=TMP) - - @classmethod - def setUpClass(cls): - ''' - Create start the webserver - ''' - super(TestGitPythonAuthenticatedHTTP, cls).setUpClass() - # Override the URL set up in the parent class - cls.url = 'http://{username}:{password}@127.0.0.1:{port}/repo.git'.format( - username=cls.username, - password=cls.password, - port=cls.port) - cls.ext_opts['url'] = cls.url - - @classmethod - def create_webserver(cls): - ''' - Use HTTPBasicAuthHandler to force an auth prompt for these tests - ''' - if cls.root_dir is None: - raise Exception('root_dir not defined in test class') - return Webserver(root=cls.root_dir, port=cls.port, - handler=HTTPBasicAuthHandler) diff --git a/tests/support/git_pillar.py b/tests/support/git_pillar.py index cc2726f875..68bf44b07d 100644 --- a/tests/support/git_pillar.py +++ b/tests/support/git_pillar.py @@ -5,12 +5,14 @@ Base classes for git_pillar integration tests # Import python libs from __future__ import absolute_import +import copy import errno import logging import os import psutil import random import shutil +import signal import string import tempfile import textwrap @@ -19,6 +21,7 @@ import yaml # Import Salt libs import salt.utils +from salt.fileserver import gitfs from salt.pillar import git_pillar from salt.ext.six.moves import range # pylint: disable=redefined-builtin @@ -36,60 +39,222 @@ from tests.support.mock import patch log = logging.getLogger(__name__) +_OPTS = { + '__role': 'minion', + 'environment': None, + 'pillarenv': None, + 'hash_type': 'sha256', + 'file_roots': {}, + 'state_top': 'top.sls', + 'state_top_saltenv': None, + 'renderer': 'yaml_jinja', + 'renderer_whitelist': [], + 'renderer_blacklist': [], + 'pillar_merge_lists': False, + 'git_pillar_base': 'master', + 'git_pillar_branch': 'master', + 'git_pillar_env': '', + 'git_pillar_root': '', + 'git_pillar_ssl_verify': True, + 'git_pillar_global_lock': True, + 'git_pillar_user': '', + 'git_pillar_password': '', + 'git_pillar_insecure_auth': False, + 'git_pillar_privkey': '', + 'git_pillar_pubkey': '', + 'git_pillar_passphrase': '', + 'git_pillar_refspecs': [ + '+refs/heads/*:refs/remotes/origin/*', + '+refs/tags/*:refs/tags/*', + ], + 'git_pillar_includes': True, +} +PROC_TIMEOUT = 10 +NOTSET = object() + + def _rand_key_name(length): return 'id_rsa_{0}'.format( ''.join(random.choice(string.ascii_letters) for _ in range(length)) ) -class GitPillarTestBase(ModuleCase, - LoaderModuleMockMixin, - SaltReturnAssertsMixin): +class ProcessManager(object): ''' - Base class for all git_pillar tests + Functions used both to set up self-contained SSH/HTTP servers for testing + ''' + wait = 10 + + def find_proc(self, name=None, search=None): + def _search(proc): + return any([search in x for x in proc.cmdline()]) + if name is None and search is None: + raise ValueError('one of name or search is required') + for proc in psutil.process_iter(): + if name is not None: + if search is None: + if name in proc.name(): + return proc + elif name in proc.name() and _search(proc): + return proc + else: + if _search(proc): + return proc + return None + + def wait_proc(self, name=None, search=None, timeout=PROC_TIMEOUT): + for idx in range(1, self.wait + 1): + proc = self.find_proc(name=name, search=search) + if proc is not None: + return proc + else: + if idx != self.wait: + log.debug( + 'Waiting for %s process (%d of %d)', + name, idx, self.wait + ) + time.sleep(1) + else: + log.debug( + 'Failed fo find %s process after %d seconds', + name, self.wait + ) + else: + raise Exception( + 'Unable to find {0} process running from temp config file ' + '{1} using psutil. Check to see if an instance of nginx from ' + 'an earlier aborted run of these tests is running, if so then ' + 'manually kill it and re-run test(s).'.format(name, search) + ) + + +class SSHDMixin(ModuleCase, ProcessManager, SaltReturnAssertsMixin): + ''' + Functions to stand up an SSHD server to serve up git repos for tests. + ''' + sshd_proc = None + + @classmethod + def prep_server(cls): + cls.sshd_config = os.path.join(cls.sshd_config_dir, 'sshd_config') + cls.sshd_port = get_unused_localhost_port() + cls.url = 'ssh://{username}@127.0.0.1:{port}/~/repo.git'.format( + username=cls.username, + port=cls.sshd_port) + home = '/root/.ssh' + cls.ext_opts = { + 'url': cls.url, + 'privkey_nopass': os.path.join(home, cls.id_rsa_nopass), + 'pubkey_nopass': os.path.join(home, cls.id_rsa_nopass + '.pub'), + 'privkey_withpass': os.path.join(home, cls.id_rsa_withpass), + 'pubkey_withpass': os.path.join(home, cls.id_rsa_withpass + '.pub'), + 'passphrase': cls.passphrase} + + def spawn_server(self): + ret = self.run_function( + 'state.apply', + mods='git_pillar.ssh', + pillar={'git_pillar': {'git_ssh': self.git_ssh, + 'id_rsa_nopass': self.id_rsa_nopass, + 'id_rsa_withpass': self.id_rsa_withpass, + 'sshd_bin': self.sshd_bin, + 'sshd_port': self.sshd_port, + 'sshd_config_dir': self.sshd_config_dir, + 'master_user': self.master_opts['user'], + 'user': self.username}} + ) + + try: + self.sshd_proc = self.wait_proc(name='sshd', + search=self.sshd_config) + finally: + # Do the assert after we check for the PID so that we can track + # it regardless of whether or not something else in the SLS + # failed (but the SSH server still started). + self.assertSaltTrueReturn(ret) + + +class WebserverMixin(ModuleCase, ProcessManager, SaltReturnAssertsMixin): + ''' + Functions to stand up an nginx + uWSGI + git-http-backend webserver to + serve up git repos for tests. + ''' + nginx_proc = uwsgi_proc = None + + @classmethod + def prep_server(cls): + ''' + Set up all the webserver paths. Designed to be run once in a + setUpClass function. + ''' + cls.root_dir = tempfile.mkdtemp(dir=TMP) + cls.config_dir = os.path.join(cls.root_dir, 'config') + cls.nginx_conf = os.path.join(cls.config_dir, 'nginx.conf') + cls.uwsgi_conf = os.path.join(cls.config_dir, 'uwsgi.yml') + cls.git_dir = os.path.join(cls.root_dir, 'git') + cls.repo_dir = os.path.join(cls.git_dir, 'repos') + cls.venv_dir = os.path.join(cls.root_dir, 'venv') + cls.uwsgi_bin = os.path.join(cls.venv_dir, 'bin', 'uwsgi') + cls.nginx_port = cls.uwsgi_port = get_unused_localhost_port() + while cls.uwsgi_port == cls.nginx_port: + # Ensure we don't hit a corner case in which two sucessive calls to + # get_unused_localhost_port() return identical port numbers. + cls.uwsgi_port = get_unused_localhost_port() + cls.url = 'http://127.0.0.1:{port}/repo.git'.format(port=cls.nginx_port) + cls.ext_opts = {'url': cls.url} + + @requires_system_grains + def spawn_server(self, grains): + auth_enabled = hasattr(self, 'username') and hasattr(self, 'password') + pillar = {'git_pillar': {'config_dir': self.config_dir, + 'git_dir': self.git_dir, + 'venv_dir': self.venv_dir, + 'root_dir': self.root_dir, + 'nginx_port': self.nginx_port, + 'uwsgi_port': self.uwsgi_port, + 'auth_enabled': auth_enabled}} + + if grains['os_family'] in ('Debian',): + # Different libexec dir for git backend on Debian-based systems + pillar['git_pillar']['libexec_dir'] = '/usr/lib' + + ret = self.run_function( + 'state.apply', + mods='git_pillar.http', + pillar=pillar) + + try: + self.nginx_proc = self.wait_proc(name='nginx', + search=self.nginx_conf) + self.uwsgi_proc = self.wait_proc(name='uwsgi', + search=self.uwsgi_conf) + finally: + # Do the assert after we check for the PID so that we can track + # it regardless of whether or not something else in the SLS + # failed (but the webserver still started). + self.assertSaltTrueReturn(ret) + + +class GitTestBase(ModuleCase): + ''' + Base class for all gitfs/git_pillar tests. Must be subclassed and paired + with either SSHDMixin or WebserverMixin to provide the server. ''' case = port = bare_repo = admin_repo = None maxDiff = None git_opts = '-c user.name="Foo Bar" -c user.email=foo@bar.com' ext_opts = {} - @requires_system_grains - def setup_loader_modules(self, grains): # pylint: disable=W0221 - return { - git_pillar: { - '__opts__': { - '__role': 'minion', - 'environment': None, - 'pillarenv': None, - 'hash_type': 'sha256', - 'file_roots': {}, - 'state_top': 'top.sls', - 'state_top_saltenv': None, - 'renderer': 'yaml_jinja', - 'renderer_whitelist': [], - 'renderer_blacklist': [], - 'pillar_merge_lists': False, - 'git_pillar_base': 'master', - 'git_pillar_branch': 'master', - 'git_pillar_env': '', - 'git_pillar_root': '', - 'git_pillar_ssl_verify': True, - 'git_pillar_global_lock': True, - 'git_pillar_user': '', - 'git_pillar_password': '', - 'git_pillar_insecure_auth': False, - 'git_pillar_privkey': '', - 'git_pillar_pubkey': '', - 'git_pillar_passphrase': '', - 'git_pillar_refspecs': [ - '+refs/heads/*:refs/remotes/origin/*', - '+refs/tags/*:refs/tags/*', - ], - 'git_pillar_includes': True, - }, - '__grains__': grains, - } - } + @classmethod + def setUpClass(cls): + cls.prep_server() + + def setUp(self): + # Make the test class available to the tearDownClass so we can clean up + # after ourselves. This (and the gated block below) prevent us from + # needing to spend the extra time creating an ssh server and user and + # then tear them down separately for each test. + self.update_class(self) @classmethod def update_class(cls, case): @@ -101,16 +266,36 @@ class GitPillarTestBase(ModuleCase, if getattr(cls, 'case') is None: setattr(cls, 'case', case) - @classmethod - def setUpClass(cls): - cls.port = get_unused_localhost_port() + def make_repo(self, root_dir, user='root'): + raise NotImplementedError() - def setUp(self): - # Make the test class available to the tearDownClass so we can clean up - # after ourselves. This (and the gated block below) prevent us from - # needing to spend the extra time creating an ssh server and user and - # then tear them down separately for each test. - self.update_class(self) + +class GitFSTestBase(GitTestBase, LoaderModuleMockMixin): + ''' + Base class for all gitfs tests + ''' + @requires_system_grains + def setup_loader_modules(self, grains): # pylint: disable=W0221 + return { + gitfs: { + '__opts__': copy.copy(_OPTS), + '__grains__': grains, + } + } + + +class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin): + ''' + Base class for all git_pillar tests + ''' + @requires_system_grains + def setup_loader_modules(self, grains): # pylint: disable=W0221 + return { + git_pillar: { + '__opts__': copy.copy(_OPTS), + '__grains__': grains, + } + } def get_pillar(self, ext_pillar_conf): ''' @@ -253,94 +438,22 @@ class GitPillarTestBase(ModuleCase, _push('top_only', 'add top_only branch') -class HTTPTestBase(GitPillarTestBase): +class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin): ''' - Base class for GitPython and Pygit2 HTTP tests - - NOTE: root_dir must be overridden in a subclass + Base class for GitPython and Pygit2 SSH tests. Redefine sshd_config_dir in + a subclass using tempfile.mkdtemp() ''' - goot_dir = None - - @classmethod - def setUpClass(cls): - ''' - Create start the webserver - ''' - super(HTTPTestBase, cls).setUpClass() - cls.webserver = cls.create_webserver() - cls.webserver.start() - cls.url = 'http://127.0.0.1:{port}/repo.git'.format(port=cls.port) - cls.ext_opts = { - 'url': cls.url, - 'username': cls.username, - 'password': cls.password} - - @classmethod - def tearDownClass(cls): - ''' - Stop the webserver and cleanup the repo - ''' - cls.webserver.stop() - shutil.rmtree(cls.root_dir, ignore_errors=True) - - @classmethod - def create_webserver(cls): - ''' - Override this in a subclass with the handler argument to use a custom - handler for HTTP Basic Authentication - ''' - if cls.root_dir is None: - raise Exception('root_dir not defined in test class') - return Webserver(root=cls.root_dir, port=cls.port) - - def setUp(self): - ''' - Create and start the webserver, and create the git repo - ''' - super(HTTPTestBase, self).setUp() - self.make_repo(self.root_dir) - - -class SSHTestBase(GitPillarTestBase): - ''' - Base class for GitPython and Pygit2 SSH tests - ''' - # Define a few variables and set to None so they're not culled in the - # cleanup when the test function completes, and remain available to the - # tearDownClass. - sshd_proc = None + sshd_config_dir = None # Creates random key names to (hopefully) ensure we're not overwriting an # existing key in /root/.ssh. Even though these are destructive tests, we # don't want to mess with something as important as ssh. id_rsa_nopass = _rand_key_name(8) id_rsa_withpass = _rand_key_name(8) - sshd_wait = 10 - - @classmethod - def setUpClass(cls): - super(SSHTestBase, cls).setUpClass() - cls.url = 'ssh://{username}@127.0.0.1:{port}/~/repo.git'.format( - username=cls.username, - port=cls.port) - home = '/root/.ssh' - cls.ext_opts = { - 'url': cls.url, - 'privkey_nopass': os.path.join(home, cls.id_rsa_nopass), - 'pubkey_nopass': os.path.join(home, cls.id_rsa_nopass + '.pub'), - 'privkey_withpass': os.path.join(home, cls.id_rsa_withpass), - 'pubkey_withpass': os.path.join(home, cls.id_rsa_withpass + '.pub'), - 'passphrase': cls.passphrase} @classmethod def tearDownClass(cls): - ''' - Stop the SSH server, remove the user, and clean up the config dir - ''' - if cls.case.sshd_proc: - try: - cls.case.sshd_proc.kill() - except psutil.NoSuchProcess: - pass + if cls.case.sshd_proc is not None: + cls.case.sshd_proc.send_signal(signal.SIGTERM) cls.case.run_state('user.absent', name=cls.username, purge=True) for dirname in (cls.sshd_config_dir, cls.case.admin_repo, cls.case.bare_repo): @@ -358,66 +471,20 @@ class SSHTestBase(GitPillarTestBase): ''' Create the SSH server and user, and create the git repo ''' - super(SSHTestBase, self).setUp() - sshd_config_file = os.path.join(self.sshd_config_dir, 'sshd_config') - self.sshd_proc = self.find_sshd(sshd_config_file) + super(GitPillarSSHTestBase, self).setUp() + self.sshd_proc = self.find_proc(name='sshd', + search=self.sshd_config) self.sshd_bin = salt.utils.which('sshd') self.git_ssh = '/tmp/git_ssh' if self.sshd_proc is None: - user_files = os.listdir( - os.path.join(FILES, 'file/base/git_pillar/ssh/user/files') - ) - ret = self.run_function( - 'state.apply', - mods='git_pillar.ssh', - pillar={'git_pillar': {'git_ssh': self.git_ssh, - 'id_rsa_nopass': self.id_rsa_nopass, - 'id_rsa_withpass': self.id_rsa_withpass, - 'sshd_bin': self.sshd_bin, - 'sshd_port': self.port, - 'sshd_config_dir': self.sshd_config_dir, - 'master_user': self.master_opts['user'], - 'user': self.username, - 'user_files': user_files}} - ) - - try: - for idx in range(1, self.sshd_wait + 1): - self.sshd_proc = self.find_sshd(sshd_config_file) - if self.sshd_proc is not None: - break - else: - if idx != self.sshd_wait: - log.debug( - 'Waiting for sshd process (%d of %d)', - idx, self.sshd_wait - ) - time.sleep(1) - else: - log.debug( - 'Failed fo find sshd process after %d seconds', - self.sshd_wait - ) - else: - raise Exception( - 'Unable to find an sshd process running from temp ' - 'config file {0} using psutil. Check to see if an ' - 'instance of sshd from an earlier aborted run of ' - 'these tests is running, if so then manually kill ' - 'it and re-run test(s).'.format(sshd_config_file) - ) - finally: - # Do the assert after we check for the PID so that we can track - # it regardless of whether or not something else in the SLS - # failed (but the SSH server still started). - self.assertSaltTrueReturn(ret) + self.spawn_server() known_hosts_ret = self.run_function( 'ssh.set_known_host', user=self.master_opts['user'], hostname='127.0.0.1', - port=self.port, + port=self.sshd_port, enc='ssh-rsa', fingerprint='fd:6f:7f:5d:06:6b:f2:06:0d:26:93:9e:5a:b5:19:46', hash_known_hosts=False, @@ -440,9 +507,48 @@ class SSHTestBase(GitPillarTestBase): ) self.make_repo(root_dir, user=self.username) - def find_sshd(self, sshd_config_file): - for proc in psutil.process_iter(): - if 'sshd' in proc.name(): - if sshd_config_file in proc.cmdline(): - return proc - return None + def get_pillar(self, ext_pillar_conf): + ''' + Wrap the parent class' get_pillar() func in logic that temporarily + changes the GIT_SSH to use our custom script, ensuring that the + passphraselsess key is used to auth without needing to modify the root + user's ssh config file. + ''' + orig_git_ssh = os.environ.pop('GIT_SSH', NOTSET) + os.environ['GIT_SSH'] = self.git_ssh + try: + return super(GitPillarSSHTestBase, self).get_pillar(ext_pillar_conf) + finally: + os.environ.pop('GIT_SSH', None) + if orig_git_ssh is not NOTSET: + os.environ['GIT_SSH'] = orig_git_ssh + + +class GitPillarHTTPTestBase(GitPillarTestBase, WebserverMixin): + ''' + Base class for GitPython and Pygit2 HTTP tests + ''' + @classmethod + def tearDownClass(cls): + for proc in (cls.case.nginx_proc, cls.case.uwsgi_proc): + if proc is not None: + try: + proc.send_signal(signal.SIGQUIT) + except psutil.NoSuchProcess: + pass + shutil.rmtree(cls.root_dir, ignore_errors=True) + + def setUp(self): + ''' + Create and start the webserver, and create the git repo + ''' + super(GitPillarHTTPTestBase, self).setUp() + self.nginx_proc = self.find_proc(name='nginx', + search=self.nginx_conf) + self.uwsgi_proc = self.find_proc(name='uwsgi', + search=self.uwsgi_conf) + + if self.nginx_proc is None and self.uwsgi_proc is None: + self.spawn_server() + + self.make_repo(self.repo_dir) From 123b5cdc11de8c1d1430b38dadc185c971abdb40 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 27 Apr 2017 10:59:49 -0500 Subject: [PATCH 07/19] Add documentation for PyYAML's loading of time expressions Also remove the notice about the handling of integer scalars starting with zeroes, as this was fixed in 0.10.0 which is roughly 5 years old now. --- .../troubleshooting/yaml_idiosyncrasies.rst | 43 ++++--------------- 1 file changed, 8 insertions(+), 35 deletions(-) diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst index 6c16f35969..74980d1bd2 100644 --- a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -129,44 +129,17 @@ string literal: - source: salt://ssh_keys/chease.pub - config: '%h/.ssh/authorized_keys' -Integers are Parsed as Integers -=============================== +Time Expressions +================ -NOTE: This has been fixed in salt 0.10.0, as of this release passing an -integer that is preceded by a 0 will be correctly parsed +PyYAML will load a time expression as the integer value of that, assuming +``HH:MM``. So for example, ``12:00`` is loaded by PyYAML as ``720``. An +excellent explanation for why can be found here__. -When passing :func:`integers ` into an SLS file, they are -passed as integers. This means that if a state accepts a string value -and an integer is passed, that an integer will be sent. The solution here -is to send the integer as a string. - -This is best explained when setting the mode for a file: - -.. code-block:: yaml - - /etc/vimrc: - file: - - managed - - source: salt://edit/vimrc - - user: root - - group: root - - mode: 644 - -Salt manages this well, since the mode is passed as 644, but if the mode is -zero padded as 0644, then it is read by YAML as an integer and evaluated as -an octal value, 0644 becomes 420. Therefore, if the file mode is -preceded by a 0 then it needs to be passed as a string: - -.. code-block:: yaml - - /etc/vimrc: - file: - - managed - - source: salt://edit/vimrc - - user: root - - group: root - - mode: '0644' +To keep time expressions like this from being loaded as integers, always quote +them. +.. __: http://stackoverflow.com/a/31007425 YAML does not like "Double Short Decs" ====================================== From 8c078f144cdebdf7f8a06d5582a372e872f68c1b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 27 Apr 2017 11:12:42 -0500 Subject: [PATCH 08/19] Add additional note about quoting within load_yaml --- .../troubleshooting/yaml_idiosyncrasies.rst | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst index 74980d1bd2..e1c6383459 100644 --- a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -139,6 +139,31 @@ excellent explanation for why can be found here__. To keep time expressions like this from being loaded as integers, always quote them. +.. note:: + When using a jinja ``load_yaml`` map, items must be quoted twice. For + example: + + .. code-block:: yaml + + {% load_yaml as wsus_schedule %} + + FRI_10: + time: '"23:00"' + day: 6 - Every Friday + SAT_10: + time: '"06:00"' + day: 7 - Every Saturday + SAT_20: + time: '"14:00"' + day: 7 - Every Saturday + SAT_30: + time: '"22:00"' + day: 7 - Every Saturday + SUN_10: + time: '"06:00"' + day: 1 - Every Sunday + {% endload %} + .. __: http://stackoverflow.com/a/31007425 YAML does not like "Double Short Decs" From 3ccb553f9f124336ad42eaa7647a0f05a357780f Mon Sep 17 00:00:00 2001 From: David Boucha Date: Thu, 20 Apr 2017 17:20:19 -0600 Subject: [PATCH 09/19] get config_dir based off conf_file get config_dir based off conf_file if config_dir not in __opts__ ZD 1353 --- salt/renderers/gpg.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/renderers/gpg.py b/salt/renderers/gpg.py index 7974ad1ba5..ff0c3f12d5 100644 --- a/salt/renderers/gpg.py +++ b/salt/renderers/gpg.py @@ -244,11 +244,16 @@ def _get_key_dir(): ''' return the location of the GPG key directory ''' + gpg_keydir = None if 'config.get' in __salt__: gpg_keydir = __salt__['config.get']('gpg_keydir') - else: + if not gpg_keydir: gpg_keydir = __opts__.get('gpg_keydir') - return gpg_keydir or os.path.join(__opts__['config_dir'], 'gpgkeys') + if not gpg_keydir and 'config_dir' in __opts__: + gpg_keydir = os.path.join(__opts__['config_dir'], 'gpgkeys') + else: + gpg_keydir = os.path.join(os.path.split(__opts__['conf_file'])[0], 'gpgkeys') + return gpg_keydir def _decrypt_ciphertext(cipher, translate_newlines=False): From 9f27f362cac517e3e2ec770384ea8fd72b659537 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 27 Apr 2017 17:07:00 -0500 Subject: [PATCH 10/19] Add HTTP git_pillar integration tests for pygit2 Also work around ssh auth issues in Ubuntu --- salt/utils/gitfs.py | 2 +- .../git_pillar/ssh/server/files/sshd_config | 1 - .../file/base/git_pillar/ssh/server/init.sls | 11 + tests/integration/pillar/test_git_pillar.py | 925 ++++++++++++++++-- tests/support/{git_pillar.py => gitfs.py} | 55 +- 5 files changed, 875 insertions(+), 119 deletions(-) rename tests/support/{git_pillar.py => gitfs.py} (93%) diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index 811b76b496..9eaf73a9be 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -1403,7 +1403,7 @@ class Pygit2(GitProvider): try: try: self.repo = pygit2.Repository(self.cachedir) - except pygit2.GitError as exc: + except GitError as exc: import pwd # https://github.com/libgit2/pygit2/issues/339 # https://github.com/libgit2/libgit2/issues/2122 diff --git a/tests/integration/files/file/base/git_pillar/ssh/server/files/sshd_config b/tests/integration/files/file/base/git_pillar/ssh/server/files/sshd_config index caf6ed5abe..ecc7c0efad 100644 --- a/tests/integration/files/file/base/git_pillar/ssh/server/files/sshd_config +++ b/tests/integration/files/file/base/git_pillar/ssh/server/files/sshd_config @@ -1,7 +1,6 @@ Port {{ pillar['git_pillar']['sshd_port'] }} ListenAddress 127.0.0.1 PermitRootLogin no -UsePAM no ChallengeResponseAuthentication no PasswordAuthentication no PubkeyAuthentication yes diff --git a/tests/integration/files/file/base/git_pillar/ssh/server/init.sls b/tests/integration/files/file/base/git_pillar/ssh/server/init.sls index 0ec758bf8b..506f7d15da 100644 --- a/tests/integration/files/file/base/git_pillar/ssh/server/init.sls +++ b/tests/integration/files/file/base/git_pillar/ssh/server/init.sls @@ -24,6 +24,14 @@ - mode: 644 - template: jinja +{%- if grains['os_family'] == 'Debian' %} +/var/run/sshd: + file.directory: + - user: root + - group: root + - mode: 755 +{%- endif %} + start_sshd: cmd.run: - name: '{{ pillar['git_pillar']['sshd_bin'] }} -f {{ sshd_config_dir }}/sshd_config' @@ -31,3 +39,6 @@ start_sshd: - file: {{ sshd_config_dir }}/sshd_config - file: {{ sshd_config_dir }}/ssh_host_rsa_key - file: {{ sshd_config_dir }}/ssh_host_rsa_key.pub +{%- if grains['os_family'] == 'Debian' %} + - file: /var/run/sshd +{%- endif %} diff --git a/tests/integration/pillar/test_git_pillar.py b/tests/integration/pillar/test_git_pillar.py index a6f73297cc..4e6b65d51c 100644 --- a/tests/integration/pillar/test_git_pillar.py +++ b/tests/integration/pillar/test_git_pillar.py @@ -1,31 +1,97 @@ # -*- coding: utf-8 -*- ''' -Tests for the salt-run command +Integration tests for git_pillar + +The base classes for all of these tests are in tests/support/gitfs.py. +Repositories for the tests are generated on the fly (look for the "make_repo" +function). + +Where possible, a test case in this module should be reproduced in the +following ways: + +1. GitPython over SSH (TestGitPythonSSH) +2. GitPython over HTTP (TestGitPythonHTTP) +3. GitPython over HTTP w/basic auth (TestGitPythonAuthenticatedHTTP) +4. pygit2 over SSH (TestPygit2SSH) +5. pygit2 over HTTP (TestPygit2HTTP) +6. pygit2 over HTTP w/basic auth (TestPygit2AuthenticatedHTTP) + +For GitPython, this is easy, since it does not support the authentication +configuration parameters that pygit2 does. Therefore, this test module includes +a GitPythonMixin class which can be reused for all three GitPython test +classes. The only thing we vary for these tests is the URL that we use. + +For pygit2 this is more complicated, since it supports A) both passphraseless +and passphrase-protected SSH keys, and B) both global and per-remote credential +parameters. So, for SSH tests we need to run each GitPython test case in 4 +different ways to cover pygit2: + +1. Passphraseless key, global credential options +2. Passphraseless key, per-repo credential options +3. Passphrase-protected key, global credential options +4. Passphrase-protected key, per-repo credential options + +For HTTP tests, we need to run each GitPython test case in 2 different ways to +cover pygit2 with authentication: + +1. Global credential options +2. Per-repo credential options + +For unauthenticated HTTP, we can just run a single case just like for a +GitPython test function, with the only change being to the git_pillar_provider +config option. + +The way we accomplish the extra test cases for pygit2 is not by writing more +test functions, but to keep the same test function names both in the GitPython +test classes and the pygit2 test classes, and just perform multiple pillar +compilations and asserts in each pygit2 test function. + + +For SSH tests, a system user is added and a temporary sshd instance is started +on a randomized port. The user and sshd server are torn down after the tests +are run. + +For HTTP tests, nginx + uWSGI + git-http-backend handles serving the repo. +However, there was a change in git 2.4.4 which causes a fetch to hang when +using uWSGI. This was worked around in uWSGI 2.0.13 by adding an additional +setting. However, Ubuntu 16.04 LTS ships with uWSGI 2.0.12 in their official +repos, so to work around this we pip install a newer uWSGI (with CGI support +baked in) within a virtualenv the test suite creates, and then uses that uwsgi +binary to start the uWSGI daemon. More info on the git issue and the uWSGI +workaround can be found in the below two links: + +https://github.com/git/git/commit/6bc0cb5 +https://github.com/unbit/uwsgi/commit/ac1e354 ''' + # Import Python libs from __future__ import absolute_import -import os -import tempfile -import tornado.web - -from salt.utils.gitfs import GITPYTHON_MINVER, PYGIT2_MINVER +import random +import string # Import Salt Testing libs +from tests.support.gitfs import ( + USERNAME, + PASSWORD, + GitPillarSSHTestBase, + GitPillarHTTPTestBase, +) from tests.support.helpers import ( destructiveTest, - http_basic_auth, - skip_if_not_root, - Webserver, + requires_system_grains, + skip_if_not_root ) -from tests.support.git_pillar import GitPillarSSHTestBase, GitPillarHTTPTestBase -from tests.support.paths import TMP -from tests.support.unit import skipIf from tests.support.mock import NO_MOCK, NO_MOCK_REASON +from tests.support.unit import skipIf # Import Salt libs import salt.utils +from salt.utils.gitfs import GITPYTHON_MINVER, PYGIT2_MINVER from salt.utils.versions import LooseVersion +from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES as VIRTUALENV_NAMES +from salt.ext.six.moves import range # pylint: disable=redefined-builtin +# Check for requisite components try: import git HAS_GITPYTHON = \ @@ -40,8 +106,15 @@ try: except ImportError: HAS_PYGIT2 = False -USERNAME = 'gitpillaruser' -PASSWORD = 'saltrules' +HAS_SSHD = bool(salt.utils.which('sshd')) +HAS_NGINX = bool(salt.utils.which('nginx')) +HAS_VIRTUALENV = bool(salt.utils.which_bin(VIRTUALENV_NAMES)) + + +def _rand_key_name(length): + return 'id_rsa_{0}'.format( + ''.join(random.choice(string.ascii_letters) for _ in range(length)) + ) class GitPythonMixin(object): @@ -49,7 +122,7 @@ class GitPythonMixin(object): GitPython doesn't support anything fancy in terms of authentication options, so all of the tests for GitPython can be re-used via this mixin. ''' - def test_git_pillar_single_source(self): + def test_single_source(self): ''' Test using a single ext_pillar repo ''' @@ -70,7 +143,7 @@ class GitPythonMixin(object): 'nested_dict': {'master': True}}} ) - def test_git_pillar_multiple_sources_master_dev_no_merge_lists(self): + def test_multiple_sources_master_dev_no_merge_lists(self): ''' Test using two ext_pillar dirs. Since all git_pillar repos are merged into a single dictionary, ordering matters. @@ -98,7 +171,7 @@ class GitPythonMixin(object): 'nested_dict': {'master': True, 'dev': True}}} ) - def test_git_pillar_multiple_sources_dev_master_no_merge_lists(self): + def test_multiple_sources_dev_master_no_merge_lists(self): ''' Test using two ext_pillar dirs. Since all git_pillar repos are merged into a single dictionary, ordering matters. @@ -126,7 +199,7 @@ class GitPythonMixin(object): 'nested_dict': {'master': True, 'dev': True}}} ) - def test_git_pillar_multiple_sources_master_dev_merge_lists(self): + def test_multiple_sources_master_dev_merge_lists(self): ''' Test using two ext_pillar dirs. Since all git_pillar repos are merged into a single dictionary, ordering matters. @@ -154,7 +227,7 @@ class GitPythonMixin(object): 'nested_dict': {'master': True, 'dev': True}}} ) - def test_git_pillar_multiple_sources_dev_master_merge_lists(self): + def test_multiple_sources_dev_master_merge_lists(self): ''' Test using two ext_pillar dirs. Since all git_pillar repos are merged into a single dictionary, ordering matters. @@ -182,7 +255,7 @@ class GitPythonMixin(object): 'nested_dict': {'master': True, 'dev': True}}} ) - def test_git_pillar_multiple_sources_with_pillarenv(self): + def test_multiple_sources_with_pillarenv(self): ''' Test using pillarenv to restrict results to those from a single branch ''' @@ -205,7 +278,7 @@ class GitPythonMixin(object): 'nested_dict': {'master': True}}} ) - def test_git_pillar_includes_enabled(self): + def test_includes_enabled(self): ''' Test with git_pillar_includes enabled. The top_only branch references an SLS file from the master branch, so we should see the key from that @@ -231,7 +304,7 @@ class GitPythonMixin(object): 'included_pillar': True} ) - def test_git_pillar_includes_disabled(self): + def test_includes_disabled(self): ''' Test with git_pillar_includes enabled. The top_only branch references an SLS file from the master branch, but since includes are disabled it @@ -263,58 +336,44 @@ class GitPythonMixin(object): @destructiveTest -@skipIf(not salt.utils.which('sshd'), 'sshd not present') -@skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) +@skipIf(NO_MOCK, NO_MOCK_REASON) @skipIf(salt.utils.is_windows(), 'minion is windows') @skip_if_not_root -@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) +@skipIf(not HAS_SSHD, 'sshd not present') class TestGitPythonSSH(GitPillarSSHTestBase, GitPythonMixin): ''' Test git_pillar with GitPython using SSH authentication - - NOTE: Any tests added to this test class should have equivalent tests (if - possible) in the TestPygit2SSH class. Also, bear in mind that the pygit2 - versions of these tests need to be more complex in that they need to test - both with passphraseless and passphrase-protecteed keys, both with global - and per-remote configuration. So for every time we run a GitPython test, we - need to run that same test four different ways for pygit2. This is because - GitPython's ability to use git-over-SSH is limited to passphraseless keys. - So, unlike pygit2, we don't need to test global or per-repo credential - config params since GitPython doesn't use them. ''' - sshd_config_dir = tempfile.mkdtemp(dir=TMP) + id_rsa_nopass = _rand_key_name(8) + id_rsa_withpass = _rand_key_name(8) username = USERNAME passphrase = PASSWORD -@skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) +@skipIf(NO_MOCK, NO_MOCK_REASON) @skipIf(salt.utils.is_windows(), 'minion is windows') @skip_if_not_root -@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) +@skipIf(not HAS_NGINX, 'nginx not present') +@skipIf(not HAS_VIRTUALENV, 'virtualenv not present') class TestGitPythonHTTP(GitPillarHTTPTestBase, GitPythonMixin): ''' Test git_pillar with GitPython using unauthenticated HTTP ''' - root_dir = tempfile.mkdtemp(dir=TMP) + pass -@skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) +@skipIf(NO_MOCK, NO_MOCK_REASON) @skipIf(salt.utils.is_windows(), 'minion is windows') @skip_if_not_root -@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_GITPYTHON, 'GitPython >= {0} required'.format(GITPYTHON_MINVER)) +@skipIf(not HAS_NGINX, 'nginx not present') +@skipIf(not HAS_VIRTUALENV, 'virtualenv not present') class TestGitPythonAuthenticatedHTTP(TestGitPythonHTTP, GitPythonMixin): ''' - Since GitPython doesn't support passing credentials, we can test - authenticated GitPython by encoding the username:password pair into the - repository's URL. The configuration will otherwise remain the same, so we - can reuse all of the tests from TestGitPythonHTTP. - - The same cannot be done for pygit2 however, since using authentication - requires that specific params are set in the YAML config, so the YAML we - use to drive the tests will be significantly different for authenticated - repositories. + Test git_pillar with GitPython using authenticated HTTP ''' - root_dir = tempfile.mkdtemp(dir=TMP) username = USERNAME password = PASSWORD @@ -324,7 +383,8 @@ class TestGitPythonAuthenticatedHTTP(TestGitPythonHTTP, GitPythonMixin): Create start the webserver ''' super(TestGitPythonAuthenticatedHTTP, cls).setUpClass() - # Override the URL set up in the parent class + # Override the URL set up in the parent class to encode the + # username/password into it. cls.url = 'http://{username}:{password}@127.0.0.1:{port}/repo.git'.format( username=cls.username, password=cls.password, @@ -335,11 +395,11 @@ class TestGitPythonAuthenticatedHTTP(TestGitPythonHTTP, GitPythonMixin): @destructiveTest -@skipIf(not salt.utils.which('sshd'), 'sshd not present') -@skipIf(not HAS_PYGIT2, 'pygit2 >= {0} required'.format(PYGIT2_MINVER)) +@skipIf(NO_MOCK, NO_MOCK_REASON) @skipIf(salt.utils.is_windows(), 'minion is windows') @skip_if_not_root -@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYGIT2, 'pygit2 >= {0} required'.format(PYGIT2_MINVER)) +@skipIf(not HAS_SSHD, 'sshd not present') class TestPygit2SSH(GitPillarSSHTestBase): ''' Test git_pillar with pygit2 using SSH authentication @@ -347,11 +407,13 @@ class TestPygit2SSH(GitPillarSSHTestBase): NOTE: Any tests added to this test class should have equivalent tests (if possible) in the TestGitPythonSSH class. ''' + id_rsa_nopass = _rand_key_name(8) + id_rsa_withpass = _rand_key_name(8) username = USERNAME passphrase = PASSWORD - sshd_config_dir = tempfile.mkdtemp(dir=TMP) - def test_git_pillar_single_source(self): + @requires_system_grains + def test_single_source(self, grains): ''' Test using a single ext_pillar repo ''' @@ -389,6 +451,10 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) + if grains['os_family'] == 'Debian': + # passphrase-protected currently does not work here + return + # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 @@ -417,7 +483,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - def test_git_pillar_multiple_sources_master_dev_no_merge_lists(self): + @requires_system_grains + def test_multiple_sources_master_dev_no_merge_lists(self, grains): ''' Test using two ext_pillar dirs. Since all git_pillar repos are merged into a single dictionary, ordering matters. @@ -434,7 +501,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): 'nested_dict': {'master': True, 'dev': True}} } - # passphraseless key, global credential options + # Test with passphraseless key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_nopass} @@ -449,7 +516,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphraseless key, per-repo credential options + # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 cachedir: {cachedir} @@ -466,7 +533,11 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphrase-protected key, global credential options + if grains['os_family'] == 'Debian': + # passphrase-protected currently does not work here + return + + # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_withpass} @@ -482,7 +553,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphrase-protected key, per-repo credential options + # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 cachedir: {cachedir} @@ -501,7 +572,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - def test_git_pillar_multiple_sources_dev_master_no_merge_lists(self): + @requires_system_grains + def test_multiple_sources_dev_master_no_merge_lists(self, grains): ''' Test using two ext_pillar dirs. Since all git_pillar repos are merged into a single dictionary, ordering matters. @@ -518,7 +590,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): 'nested_dict': {'master': True, 'dev': True}} } - # passphraseless key, global credential options + # Test with passphraseless key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_nopass} @@ -533,7 +605,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphraseless key, per-repo credential options + # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 cachedir: {cachedir} @@ -550,7 +622,11 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphrase-protected key, global credential options + if grains['os_family'] == 'Debian': + # passphrase-protected currently does not work here + return + + # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_withpass} @@ -566,7 +642,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphrase-protected key, per-repo credential options + # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 cachedir: {cachedir} @@ -585,7 +661,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - def test_git_pillar_multiple_sources_master_dev_merge_lists(self): + @requires_system_grains + def test_multiple_sources_master_dev_merge_lists(self, grains): ''' Test using two ext_pillar dirs. Since all git_pillar repos are merged into a single dictionary, ordering matters. @@ -602,7 +679,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): 'nested_dict': {'master': True, 'dev': True}} } - # passphraseless key, global credential options + # Test with passphraseless key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_nopass} @@ -617,7 +694,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphraseless key, per-repo credential options + # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 cachedir: {cachedir} @@ -634,7 +711,11 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphrase-protected key, global credential options + if grains['os_family'] == 'Debian': + # passphrase-protected currently does not work here + return + + # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_withpass} @@ -650,7 +731,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphrase-protected key, per-repo credential options + # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 cachedir: {cachedir} @@ -669,7 +750,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - def test_git_pillar_multiple_sources_dev_master_merge_lists(self): + @requires_system_grains + def test_multiple_sources_dev_master_merge_lists(self, grains): ''' Test using two ext_pillar dirs. Since all git_pillar repos are merged into a single dictionary, ordering matters. @@ -686,7 +768,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): 'nested_dict': {'master': True, 'dev': True}} } - # passphraseless key, global credential options + # Test with passphraseless key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_nopass} @@ -701,7 +783,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphraseless key, per-repo credential options + # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 cachedir: {cachedir} @@ -718,7 +800,11 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphrase-protected key, global credential options + if grains['os_family'] == 'Debian': + # passphrase-protected currently does not work here + return + + # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_withpass} @@ -734,7 +820,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphrase-protected key, per-repo credential options + # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 cachedir: {cachedir} @@ -753,7 +839,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - def test_git_pillar_multiple_sources_with_pillarenv(self): + @requires_system_grains + def test_multiple_sources_with_pillarenv(self, grains): ''' Test using pillarenv to restrict results to those from a single branch ''' @@ -797,6 +884,10 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) + if grains['os_family'] == 'Debian': + # passphrase-protected currently does not work here + return + # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 @@ -832,7 +923,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - def test_git_pillar_includes_enabled(self): + @requires_system_grains + def test_includes_enabled(self, grains): ''' Test with git_pillar_includes enabled. The top_only branch references an SLS file from the master branch, so we should see the @@ -847,7 +939,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): 'included_pillar': True } - # passphraseless key, global credential options + # Test with passphraseless key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_nopass} @@ -862,7 +954,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphraseless key, per-repo credential options + # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 cachedir: {cachedir} @@ -879,7 +971,11 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphrase-protected key, global credential options + if grains['os_family'] == 'Debian': + # passphrase-protected currently does not work here + return + + # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_pubkey: {pubkey_withpass} @@ -895,7 +991,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphrase-protected key, per-repo credential options + # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 cachedir: {cachedir} @@ -914,7 +1010,8 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - def test_git_pillar_includes_disabled(self): + @requires_system_grains + def test_includes_disabled(self, grains): ''' Test with git_pillar_includes enabled. The top_only branch references an SLS file from the master branch, but since includes are disabled it @@ -932,7 +1029,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): "available on the salt master"] } - # passphraseless key, global credential options + # Test with passphraseless key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_includes: False @@ -948,7 +1045,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphraseless key, per-repo credential options + # Test with passphraseless key and per-repo credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_includes: False @@ -966,7 +1063,11 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphrase-protected key, global credential options + if grains['os_family'] == 'Debian': + # passphrase-protected currently does not work here + return + + # Test with passphrase-protected key and global credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_includes: False @@ -983,7 +1084,7 @@ class TestPygit2SSH(GitPillarSSHTestBase): ''') self.assertEqual(ret, expected) - # passphrase-protected key, per-repo credential options + # Test with passphrase-protected key and per-repo credential options ret = self.get_pillar('''\ git_pillar_provider: pygit2 git_pillar_includes: False @@ -1002,3 +1103,651 @@ class TestPygit2SSH(GitPillarSSHTestBase): - env: base ''') self.assertEqual(ret, expected) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(salt.utils.is_windows(), 'minion is windows') +@skip_if_not_root +@skipIf(not HAS_PYGIT2, 'pygit2 >= {0} required'.format(PYGIT2_MINVER)) +@skipIf(not HAS_NGINX, 'nginx not present') +@skipIf(not HAS_VIRTUALENV, 'virtualenv not present') +class TestPygit2HTTP(GitPillarHTTPTestBase): + ''' + Test git_pillar with pygit2 using SSH authentication + ''' + def test_single_source(self): + ''' + Test using a single ext_pillar repo + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}} + } + + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + ''') + self.assertEqual(ret, expected) + + def test_multiple_sources_master_dev_no_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the master branch followed by dev, and with + pillar_merge_lists disabled. + ''' + expected = { + 'branch': 'dev', + 'mylist': ['dev'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['dev'], + 'nested_dict': {'master': True, 'dev': True}} + } + + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual(ret, expected) + + def test_multiple_sources_dev_master_no_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the dev branch followed by master, and with + pillar_merge_lists disabled. + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True, 'dev': True}} + } + + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - dev {url} + - master {url} + ''') + self.assertEqual(ret, expected) + + def test_multiple_sources_master_dev_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the master branch followed by dev, and with + pillar_merge_lists enabled. + ''' + expected = { + 'branch': 'dev', + 'mylist': ['master', 'dev'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['master', 'dev'], + 'nested_dict': {'master': True, 'dev': True}} + } + + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual(ret, expected) + + def test_multiple_sources_dev_master_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the dev branch followed by master, and with + pillar_merge_lists enabled. + ''' + expected = { + 'branch': 'master', + 'mylist': ['dev', 'master'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['dev', 'master'], + 'nested_dict': {'master': True, 'dev': True}} + } + + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - dev {url} + - master {url} + ''') + self.assertEqual(ret, expected) + + def test_multiple_sources_with_pillarenv(self): + ''' + Test using pillarenv to restrict results to those from a single branch + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}} + } + + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillarenv: base + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual(ret, expected) + + def test_includes_enabled(self): + ''' + Test with git_pillar_includes enabled. The top_only branch references + an SLS file from the master branch, so we should see the + "included_pillar" key from that SLS file in the compiled pillar data. + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}}, + 'included_pillar': True + } + + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + - top_only {url}: + - env: base + ''') + self.assertEqual(ret, expected) + + def test_includes_disabled(self): + ''' + Test with git_pillar_includes enabled. The top_only branch references + an SLS file from the master branch, but since includes are disabled it + will not find the SLS file and the "included_pillar" key should not be + present in the compiled pillar data. We should instead see an error + message in the compiled data. + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}}, + '_errors': ["Specified SLS 'bar' in environment 'base' is not " + "available on the salt master"] + } + + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_includes: False + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + - top_only {url}: + - env: base + ''') + self.assertEqual(ret, expected) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(salt.utils.is_windows(), 'minion is windows') +@skip_if_not_root +@skipIf(not HAS_PYGIT2, 'pygit2 >= {0} required'.format(PYGIT2_MINVER)) +@skipIf(not HAS_NGINX, 'nginx not present') +@skipIf(not HAS_VIRTUALENV, 'virtualenv not present') +class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): + ''' + Test git_pillar with pygit2 using SSH authentication + + NOTE: Any tests added to this test class should have equivalent tests (if + possible) in the TestGitPythonSSH class. + ''' + user = USERNAME + password = PASSWORD + + def test_single_source(self): + ''' + Test using a single ext_pillar repo + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}} + } + + # Test with global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_user: {user} + git_pillar_password: {password} + git_pillar_insecure_auth: True + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + ''') + self.assertEqual(ret, expected) + + # Test with per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url}: + - user: {user} + - password: {password} + - insecure_auth: True + ''') + self.assertEqual(ret, expected) + + def test_multiple_sources_master_dev_no_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the master branch followed by dev, and with + pillar_merge_lists disabled. + ''' + expected = { + 'branch': 'dev', + 'mylist': ['dev'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['dev'], + 'nested_dict': {'master': True, 'dev': True}} + } + + # Test with global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_user: {user} + git_pillar_password: {password} + git_pillar_insecure_auth: True + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual(ret, expected) + + # Test with per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - master {url}: + - user: {user} + - password: {password} + - insecure_auth: True + - dev {url}: + - user: {user} + - password: {password} + - insecure_auth: True + ''') + self.assertEqual(ret, expected) + + def test_multiple_sources_dev_master_no_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the dev branch followed by master, and with + pillar_merge_lists disabled. + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True, 'dev': True}} + } + + # Test with global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_user: {user} + git_pillar_password: {password} + git_pillar_insecure_auth: True + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - dev {url} + - master {url} + ''') + self.assertEqual(ret, expected) + + # Test with per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: False + ext_pillar: + - git: + - dev {url}: + - user: {user} + - password: {password} + - insecure_auth: True + - master {url}: + - user: {user} + - password: {password} + - insecure_auth: True + ''') + self.assertEqual(ret, expected) + + def test_multiple_sources_master_dev_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the master branch followed by dev, and with + pillar_merge_lists enabled. + ''' + expected = { + 'branch': 'dev', + 'mylist': ['master', 'dev'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['master', 'dev'], + 'nested_dict': {'master': True, 'dev': True}} + } + + # Test with global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_user: {user} + git_pillar_password: {password} + git_pillar_insecure_auth: True + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual(ret, expected) + + # Test with per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - master {url}: + - user: {user} + - password: {password} + - insecure_auth: True + - dev {url}: + - user: {user} + - password: {password} + - insecure_auth: True + ''') + self.assertEqual(ret, expected) + + def test_multiple_sources_dev_master_merge_lists(self): + ''' + Test using two ext_pillar dirs. Since all git_pillar repos are merged + into a single dictionary, ordering matters. + + This tests with the dev branch followed by master, and with + pillar_merge_lists enabled. + ''' + expected = { + 'branch': 'master', + 'mylist': ['dev', 'master'], + 'mydict': {'master': True, + 'dev': True, + 'nested_list': ['dev', 'master'], + 'nested_dict': {'master': True, 'dev': True}} + } + + # Test with global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_user: {user} + git_pillar_password: {password} + git_pillar_insecure_auth: True + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - dev {url} + - master {url} + ''') + self.assertEqual(ret, expected) + + # Test with per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillar_merge_lists: True + ext_pillar: + - git: + - dev {url}: + - user: {user} + - password: {password} + - insecure_auth: True + - master {url}: + - user: {user} + - password: {password} + - insecure_auth: True + ''') + self.assertEqual(ret, expected) + + def test_multiple_sources_with_pillarenv(self): + ''' + Test using pillarenv to restrict results to those from a single branch + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}} + } + + # Test with global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_user: {user} + git_pillar_password: {password} + git_pillar_insecure_auth: True + cachedir: {cachedir} + extension_modules: {extmods} + pillarenv: base + ext_pillar: + - git: + - master {url} + - dev {url} + ''') + self.assertEqual(ret, expected) + + # Test with per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + pillarenv: base + ext_pillar: + - git: + - master {url}: + - user: {user} + - password: {password} + - insecure_auth: True + - dev {url}: + - user: {user} + - password: {password} + - insecure_auth: True + ''') + self.assertEqual(ret, expected) + + def test_includes_enabled(self): + ''' + Test with git_pillar_includes enabled. The top_only branch references + an SLS file from the master branch, so we should see the + "included_pillar" key from that SLS file in the compiled pillar data. + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}}, + 'included_pillar': True + } + + # Test with global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_user: {user} + git_pillar_password: {password} + git_pillar_insecure_auth: True + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + - top_only {url}: + - env: base + ''') + self.assertEqual(ret, expected) + + # Test with per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url}: + - user: {user} + - password: {password} + - insecure_auth: True + - top_only {url}: + - user: {user} + - password: {password} + - insecure_auth: True + - env: base + ''') + self.assertEqual(ret, expected) + + def test_includes_disabled(self): + ''' + Test with git_pillar_includes enabled. The top_only branch references + an SLS file from the master branch, but since includes are disabled it + will not find the SLS file and the "included_pillar" key should not be + present in the compiled pillar data. We should instead see an error + message in the compiled data. + ''' + expected = { + 'branch': 'master', + 'mylist': ['master'], + 'mydict': {'master': True, + 'nested_list': ['master'], + 'nested_dict': {'master': True}}, + '_errors': ["Specified SLS 'bar' in environment 'base' is not " + "available on the salt master"] + } + + # Test with global credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_includes: False + git_pillar_user: {user} + git_pillar_password: {password} + git_pillar_insecure_auth: True + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url} + - top_only {url}: + - env: base + ''') + self.assertEqual(ret, expected) + + # Test with per-repo credential options + ret = self.get_pillar('''\ + git_pillar_provider: pygit2 + git_pillar_includes: False + cachedir: {cachedir} + extension_modules: {extmods} + ext_pillar: + - git: + - master {url}: + - user: {user} + - password: {password} + - insecure_auth: True + - top_only {url}: + - user: {user} + - password: {password} + - insecure_auth: True + - env: base + ''') + self.assertEqual(ret, expected) diff --git a/tests/support/git_pillar.py b/tests/support/gitfs.py similarity index 93% rename from tests/support/git_pillar.py rename to tests/support/gitfs.py index 68bf44b07d..1e9a1d6271 100644 --- a/tests/support/git_pillar.py +++ b/tests/support/gitfs.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ''' -Base classes for git_pillar integration tests +Base classes for gitfs/git_pillar integration tests ''' # Import python libs @@ -10,10 +10,8 @@ import errno import logging import os import psutil -import random import shutil import signal -import string import tempfile import textwrap import time @@ -28,16 +26,17 @@ from salt.ext.six.moves import range # pylint: disable=redefined-builtin # Import Salt Testing libs from tests.support.case import ModuleCase from tests.support.mixins import LoaderModuleMockMixin, SaltReturnAssertsMixin -from tests.support.paths import FILES, TMP +from tests.support.paths import TMP from tests.support.helpers import ( get_unused_localhost_port, requires_system_grains, - Webserver, ) from tests.support.mock import patch log = logging.getLogger(__name__) +USERNAME = 'gitpillaruser' +PASSWORD = 'saltrules' _OPTS = { '__role': 'minion', @@ -73,12 +72,6 @@ PROC_TIMEOUT = 10 NOTSET = object() -def _rand_key_name(length): - return 'id_rsa_{0}'.format( - ''.join(random.choice(string.ascii_letters) for _ in range(length)) - ) - - class ProcessManager(object): ''' Functions used both to set up self-contained SSH/HTTP servers for testing @@ -119,13 +112,10 @@ class ProcessManager(object): 'Failed fo find %s process after %d seconds', name, self.wait ) - else: - raise Exception( - 'Unable to find {0} process running from temp config file ' - '{1} using psutil. Check to see if an instance of nginx from ' - 'an earlier aborted run of these tests is running, if so then ' - 'manually kill it and re-run test(s).'.format(name, search) - ) + raise Exception( + 'Unable to find {0} process running from temp config file {1} ' + 'using psutil'.format(name, search) + ) class SSHDMixin(ModuleCase, ProcessManager, SaltReturnAssertsMixin): @@ -136,6 +126,7 @@ class SSHDMixin(ModuleCase, ProcessManager, SaltReturnAssertsMixin): @classmethod def prep_server(cls): + cls.sshd_config_dir = tempfile.mkdtemp(dir=TMP) cls.sshd_config = os.path.join(cls.sshd_config_dir, 'sshd_config') cls.sshd_port = get_unused_localhost_port() cls.url = 'ssh://{username}@127.0.0.1:{port}/~/repo.git'.format( @@ -202,6 +193,11 @@ class WebserverMixin(ModuleCase, ProcessManager, SaltReturnAssertsMixin): cls.uwsgi_port = get_unused_localhost_port() cls.url = 'http://127.0.0.1:{port}/repo.git'.format(port=cls.nginx_port) cls.ext_opts = {'url': cls.url} + # Add auth params if present (if so this will trigger the spawned + # server to turn on HTTP basic auth). + for credential_param in ('user', 'password'): + if hasattr(cls, credential_param): + cls.ext_opts[credential_param] = getattr(cls, credential_param) @requires_system_grains def spawn_server(self, grains): @@ -283,6 +279,9 @@ class GitFSTestBase(GitTestBase, LoaderModuleMockMixin): } } + def make_repo(self, root_dir, user='root'): + raise NotImplementedError() + class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin): ''' @@ -302,7 +301,7 @@ class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin): Run git_pillar with the specified configuration ''' cachedir = tempfile.mkdtemp(dir=TMP) - self.addCleanup(shutil.rmtree, cachedir, ignore_errors=True) + #self.addCleanup(shutil.rmtree, cachedir, ignore_errors=True) ext_pillar_opts = yaml.safe_load( ext_pillar_conf.format( cachedir=cachedir, @@ -440,15 +439,9 @@ class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin): class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin): ''' - Base class for GitPython and Pygit2 SSH tests. Redefine sshd_config_dir in - a subclass using tempfile.mkdtemp() + Base class for GitPython and Pygit2 SSH tests ''' - sshd_config_dir = None - # Creates random key names to (hopefully) ensure we're not overwriting an - # existing key in /root/.ssh. Even though these are destructive tests, we - # don't want to mess with something as important as ssh. - id_rsa_nopass = _rand_key_name(8) - id_rsa_withpass = _rand_key_name(8) + id_rsa_nopass = id_rsa_withpass = None @classmethod def tearDownClass(cls): @@ -460,7 +453,11 @@ class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin): if dirname is not None: shutil.rmtree(dirname, ignore_errors=True) ssh_dir = os.path.expanduser('~/.ssh') - for filename in (cls.id_rsa_nopass, cls.id_rsa_withpass, cls.case.git_ssh): + for filename in (cls.id_rsa_nopass, + cls.id_rsa_nopass + '.pub', + cls.id_rsa_withpass, + cls.id_rsa_withpass + '.pub', + cls.case.git_ssh): try: os.remove(os.path.join(ssh_dir, filename)) except OSError as exc: @@ -549,6 +546,6 @@ class GitPillarHTTPTestBase(GitPillarTestBase, WebserverMixin): search=self.uwsgi_conf) if self.nginx_proc is None and self.uwsgi_proc is None: - self.spawn_server() + self.spawn_server() # pylint: disable=E1120 self.make_repo(self.repo_dir) From f7a6f3537075b965bf27114ac17554f2619050bf Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 27 Apr 2017 18:32:45 -0600 Subject: [PATCH 11/19] allow dynamic loading of cloud drivers --- salt/modules/saltutil.py | 36 ++++++++++++++++++++++++++++++++++++ salt/runners/saltutil.py | 26 ++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index 01789c67a2..e3fbd92465 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -579,6 +579,41 @@ def sync_output(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blackl sync_outputters = salt.utils.alias_function(sync_output, 'sync_outputters') +def sync_clouds(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist=None): + ''' + .. versionadded:: Nitrogen + + Sync utility modules from ``salt://_cloud`` to the minion + + saltenv : base + The fileserver environment from which to sync. To sync from more than + one environment, pass a comma-separated list. + + refresh : True + If ``True``, refresh the available execution modules on the minion. + This refresh will be performed even if no new utility modules are + synced. Set to ``False`` to prevent this refresh. + + extmod_whitelist : None + comma-seperated list of modules to sync + + extmod_blacklist : None + comma-seperated list of modules to blacklist based on type + + CLI Examples: + + .. code-block:: bash + + salt '*' saltutil.sync_clouds + salt '*' saltutil.sync_clouds saltenv=dev + salt '*' saltutil.sync_clouds saltenv=base,dev + ''' + ret = _sync('clouds', saltenv, extmod_whitelist, extmod_blacklist) + if refresh: + refresh_modules() + return ret + + def sync_utils(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist=None): ''' .. versionadded:: 2014.7.0 @@ -763,6 +798,7 @@ def sync_all(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist ''' log.debug('Syncing all') ret = {} + ret['clouds'] = sync_clouds(saltenv, False, extmod_whitelist, extmod_blacklist) ret['beacons'] = sync_beacons(saltenv, False, extmod_whitelist, extmod_blacklist) ret['modules'] = sync_modules(saltenv, False, extmod_whitelist, extmod_blacklist) ret['states'] = sync_states(saltenv, False, extmod_whitelist, extmod_blacklist) diff --git a/salt/runners/saltutil.py b/salt/runners/saltutil.py index 5b8bac878e..efc8fdb272 100644 --- a/salt/runners/saltutil.py +++ b/salt/runners/saltutil.py @@ -443,3 +443,29 @@ def sync_cache(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): ''' return salt.utils.extmods.sync(__opts__, 'cache', saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist)[0] + + +def sync_clouds(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): + ''' + .. versionadded:: Nitrogen + + Sync utils modules from ``salt://_cloud`` to the master + + saltenv : base + The fileserver environment from which to sync. To sync from more than + one environment, pass a comma-separated list. + + extmod_whitelist : None + comma-seperated list of modules to sync + + extmod_blacklist : None + comma-seperated list of modules to blacklist based on type + + CLI Example: + + .. code-block:: bash + + salt-run saltutil.sync_cloud + ''' + return salt.utils.extmods.sync(__opts__, 'cloud', saltenv=saltenv, extmod_whitelist=extmod_whitelist, + extmod_blacklist=extmod_blacklist)[0] From 3a26d5d42d74f563f04e9e9b5dc7b4d3e2743b0d Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Fri, 28 Apr 2017 09:12:05 -0600 Subject: [PATCH 12/19] add cloud to saltutil sync test --- tests/integration/modules/test_saltutil.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/modules/test_saltutil.py b/tests/integration/modules/test_saltutil.py index c9a1bd9062..3c5661ee44 100644 --- a/tests/integration/modules/test_saltutil.py +++ b/tests/integration/modules/test_saltutil.py @@ -73,6 +73,7 @@ class SaltUtilSyncModuleTest(ModuleCase): Test syncing all ModuleCase ''' expected_return = {'engines': [], + 'clouds': [], 'grains': [], 'beacons': [], 'utils': [], @@ -95,6 +96,7 @@ class SaltUtilSyncModuleTest(ModuleCase): Test syncing all ModuleCase with whitelist ''' expected_return = {'engines': [], + 'clouds': [], 'grains': [], 'beacons': [], 'utils': [], @@ -114,6 +116,7 @@ class SaltUtilSyncModuleTest(ModuleCase): Test syncing all ModuleCase with blacklist ''' expected_return = {'engines': [], + 'clouds': [], 'grains': [], 'beacons': [], 'utils': [], @@ -135,6 +138,7 @@ class SaltUtilSyncModuleTest(ModuleCase): Test syncing all ModuleCase with whitelist and blacklist ''' expected_return = {'engines': [], + 'clouds': [], 'grains': [], 'beacons': [], 'utils': [], From eb889173b02200d8f437ce70841a4c99aff035d6 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Tue, 6 Dec 2016 16:39:15 -0700 Subject: [PATCH 13/19] Revert "fix augeas module so shlex doesn't strip quotes" --- salt/modules/augeas_cfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/augeas_cfg.py b/salt/modules/augeas_cfg.py index 401a11eb1b..6b1f1e7b1b 100644 --- a/salt/modules/augeas_cfg.py +++ b/salt/modules/augeas_cfg.py @@ -199,7 +199,7 @@ def execute(context=None, lens=None, commands=(), load_path=None): method = METHOD_MAP[cmd] nargs = arg_map[method] - parts = salt.utils.shlex_split(arg, posix=False) + parts = salt.utils.shlex_split(arg) if len(parts) not in nargs: err = '{0} takes {1} args: {2}'.format(method, nargs, parts) From 3e9394862f6a9e6dfe12ad7c52a40433652324bd Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Thu, 27 Apr 2017 16:31:42 -0400 Subject: [PATCH 14/19] allow vmware to query deploy arg from opts --- salt/cloud/clouds/vmware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/cloud/clouds/vmware.py b/salt/cloud/clouds/vmware.py index 713fba655a..839e2562f2 100644 --- a/salt/cloud/clouds/vmware.py +++ b/salt/cloud/clouds/vmware.py @@ -2375,7 +2375,7 @@ def create(vm_): 'private_key', vm_, __opts__, search_global=False, default=None ) deploy = config.get_cloud_config_value( - 'deploy', vm_, __opts__, search_global=False, default=True + 'deploy', vm_, __opts__, search_global=True, default=True ) wait_for_ip_timeout = config.get_cloud_config_value( 'wait_for_ip_timeout', vm_, __opts__, default=20 * 60 From 402be207b26f9807fb889f4282a47824b6c3b695 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 28 Apr 2017 11:42:52 -0500 Subject: [PATCH 15/19] Skip Pygit2 tests until EPEL fixes pygit2/libgit2 version mismatch We can revert this commit once they get the updated pygit2 pushed to EPEL stable. --- tests/integration/pillar/test_git_pillar.py | 18 ++++++++++++++++++ tests/support/gitfs.py | 6 ++++++ 2 files changed, 24 insertions(+) diff --git a/tests/integration/pillar/test_git_pillar.py b/tests/integration/pillar/test_git_pillar.py index 4e6b65d51c..b1024abd20 100644 --- a/tests/integration/pillar/test_git_pillar.py +++ b/tests/integration/pillar/test_git_pillar.py @@ -412,6 +412,12 @@ class TestPygit2SSH(GitPillarSSHTestBase): username = USERNAME passphrase = PASSWORD + def setUp(self): + super(TestPygit2SSH, self).setUp() + if self.is_el7(): # pylint: disable=E1120 + self.skipTest( + 'skipped until EPEL7 fixes pygit2/libgit2 version mismatch') + @requires_system_grains def test_single_source(self, grains): ''' @@ -1115,6 +1121,12 @@ class TestPygit2HTTP(GitPillarHTTPTestBase): ''' Test git_pillar with pygit2 using SSH authentication ''' + def setUp(self): + super(TestPygit2HTTP, self).setUp() + if self.is_el7(): # pylint: disable=E1120 + self.skipTest( + 'skipped until EPEL7 fixes pygit2/libgit2 version mismatch') + def test_single_source(self): ''' Test using a single ext_pillar repo @@ -1352,6 +1364,12 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): user = USERNAME password = PASSWORD + def setUp(self): + super(TestPygit2AuthenticatedHTTP, self).setUp() + if self.is_el7(): # pylint: disable=E1120 + self.skipTest( + 'skipped until EPEL7 fixes pygit2/libgit2 version mismatch') + def test_single_source(self): ''' Test using a single ext_pillar repo diff --git a/tests/support/gitfs.py b/tests/support/gitfs.py index 1e9a1d6271..2fdc772a36 100644 --- a/tests/support/gitfs.py +++ b/tests/support/gitfs.py @@ -241,6 +241,12 @@ class GitTestBase(ModuleCase): git_opts = '-c user.name="Foo Bar" -c user.email=foo@bar.com' ext_opts = {} + # We need to temporarily skip pygit2 tests on EL7 until the EPEL packager + # updates pygit2 to bring it up-to-date with libgit2. + @requires_system_grains + def is_el7(self, grains): + return grains['os_family'] == 'RedHat' and grains['osmajorrelease'] == 7 + @classmethod def setUpClass(cls): cls.prep_server() From 523316f0d7a60e48848dd04722a5a73821165672 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 28 Apr 2017 17:19:15 -0600 Subject: [PATCH 16/19] Update Python and Deps --- pkg/windows/modules/get-settings.psm1 | 4 +-- pkg/windows/req_base.txt | 38 +++++++++++++-------------- pkg/windows/req_pip.txt | 2 +- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pkg/windows/modules/get-settings.psm1 b/pkg/windows/modules/get-settings.psm1 index bff1a50050..61df5ee3e8 100644 --- a/pkg/windows/modules/get-settings.psm1 +++ b/pkg/windows/modules/get-settings.psm1 @@ -60,7 +60,7 @@ Function Get-Settings { # Filenames for 64 bit Windows $64bitPrograms = @{ "PyCrypto2" = "pycrypto-2.6.1-cp27-none-win_amd64.whl" - "Python2" = "python-2.7.12.amd64.msi" + "Python2" = "python-2.7.13.amd64.msi" "PyYAML2" = "PyYAML-3.11.win-amd64-py2.7.exe" "Python3" = "python-3.5.3-amd64.exe" "PyWin323" = "pywin32-220.1-cp35-cp35m-win_amd64.whl" @@ -70,7 +70,7 @@ Function Get-Settings { # Filenames for 32 bit Windows $32bitPrograms = @{ "PyCrypto2" = "pycrypto-2.6.1-cp27-none-win32.whl" - "Python2" = "python-2.7.12.msi" + "Python2" = "python-2.7.13.msi" "PyYAML2" = "PyYAML-3.11.win32-py2.7.exe" "Python3" = "python-3.5.3.exe" "PyWin323" = "pywin32-220.1-cp35-cp35m-win32.whl" diff --git a/pkg/windows/req_base.txt b/pkg/windows/req_base.txt index 909ebad4f2..dc2f8e0da1 100644 --- a/pkg/windows/req_base.txt +++ b/pkg/windows/req_base.txt @@ -1,36 +1,36 @@ -backports-abc==0.4 +backports-abc==0.5 backports.ssl-match-hostname==3.5.0.1 certifi cffi==1.10.0 -CherryPy==7.1.0 +CherryPy==10.2.1 cryptography==1.8.1 enum34==1.1.6 -futures==3.0.5 +futures==3.1.1 gitdb==0.6.4 -GitPython==2.0.8 +GitPython==2.1.3 idna==2.5 -ioflo==1.5.5 +ioflo==1.6.7 ioloop==0.1a0 ipaddress==1.0.18 -Jinja2==2.9.4 -libnacl==1.4.5 -Mako==1.0.4 -MarkupSafe==0.23 +Jinja2==2.9.6 +libnacl==1.5.0 +Mako==1.0.6 +MarkupSafe==1.0 msgpack-python==0.4.8 -psutil==4.3.0 -pyasn1==0.1.9 +psutil==5.2.2 +pyasn1==0.2.3 pycparser==2.17 pycurl==7.43.0 -PyMySQL==0.7.6 -pyOpenSSL==16.2.0 -python-dateutil==2.5.3 -python-gnupg==0.3.8 -pyzmq==16.0.1 -requests==2.10.0 +PyMySQL==0.7.11 +pyOpenSSL==17.0.0 +python-dateutil==2.6.0 +python-gnupg==0.4.0 +pyzmq==16.0.2 +requests==2.13.0 singledispatch==3.4.0.3 six==1.10.0 smmap==0.9.0 timelib==0.2.4 -tornado==4.4.1 -wheel==0.29.0 +tornado==4.5.1 +wheel==0.30.0a0 WMI==1.4.9 diff --git a/pkg/windows/req_pip.txt b/pkg/windows/req_pip.txt index 4bb77f1241..4f9618b489 100644 --- a/pkg/windows/req_pip.txt +++ b/pkg/windows/req_pip.txt @@ -1,2 +1,2 @@ pip==9.0.1 -setuptools==34.3.1 \ No newline at end of file +setuptools==35.0.2 \ No newline at end of file From 92fd24c107fbd07dd53953f4d64cf209cf207209 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 1 May 2017 12:20:39 -0600 Subject: [PATCH 17/19] only template pillar_tree files if specified This should only be templated if the user specifies that it should be templated, otherwise we get a similar problem that is fixed here https://github.com/saltstack/salt/pull/40464 Fixes #40978 --- salt/pillar/file_tree.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/salt/pillar/file_tree.py b/salt/pillar/file_tree.py index 53b45740ba..7ae7a05bf4 100644 --- a/salt/pillar/file_tree.py +++ b/salt/pillar/file_tree.py @@ -56,10 +56,12 @@ intended to be used to deploy a file using ``contents_pillar`` with a files would not affected by the ``keep_newline`` configuration. However, this module does not actually distinguish between binary and text files. -.. versionchanged:: develop +.. versionchanged:: Nitrogen Templating/rendering has been added. You can now specify a default render pipeline and a black- and whitelist of (dis)allowed renderers. + ``template`` must be set to ``True`` for templating to happen. + .. code-block:: yaml ext_pillar: @@ -71,6 +73,7 @@ intended to be used to deploy a file using ``contents_pillar`` with a renderer_whitelist: - jinja - yaml + template: True Assigning Pillar Data to Individual Hosts ----------------------------------------- @@ -220,7 +223,8 @@ def _construct_pillar(top_dir, keep_newline=False, render_default=None, renderer_blacklist=None, - renderer_whitelist=None): + renderer_whitelist=None, + template=False): ''' Construct pillar from file tree. ''' @@ -272,11 +276,13 @@ def _construct_pillar(top_dir, file_path, exc.strerror) else: - data = salt.template.compile_template_str(template=contents, - renderers=renderers, - default=render_default, - blacklist=renderer_blacklist, - whitelist=renderer_whitelist) + data = contents + if template is True: + data = salt.template.compile_template_str(template=contents, + renderers=renderers, + default=render_default, + blacklist=renderer_blacklist, + whitelist=renderer_whitelist) if salt.utils.stringio.is_readable(data): pillar_node[file_name] = data.getvalue() else: @@ -293,7 +299,8 @@ def ext_pillar(minion_id, keep_newline=False, render_default=None, renderer_blacklist=None, - renderer_whitelist=None): + renderer_whitelist=None, + template=False): ''' Compile pillar data for the specified minion ID ''' @@ -341,7 +348,8 @@ def ext_pillar(minion_id, keep_newline, render_default, renderer_blacklist, - renderer_whitelist) + renderer_whitelist, + template) ) else: if debug is True: @@ -367,7 +375,8 @@ def ext_pillar(minion_id, keep_newline, render_default, renderer_blacklist, - renderer_whitelist) + renderer_whitelist, + template) return salt.utils.dictupdate.merge(ngroup_pillar, host_pillar, strategy='recurse') From 39157b40ed7291190176b8600286f4e5de828225 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 1 May 2017 17:13:42 -0500 Subject: [PATCH 18/19] Fix some git_pillar integration test failures For some reason these were not failing when I opened #40777, but now that the PR is merged are failing for PR builds (as well as locally in my test env). This fixes those failures. --- tests/support/gitfs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/support/gitfs.py b/tests/support/gitfs.py index 2fdc772a36..33df691a53 100644 --- a/tests/support/gitfs.py +++ b/tests/support/gitfs.py @@ -448,6 +448,7 @@ class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin): Base class for GitPython and Pygit2 SSH tests ''' id_rsa_nopass = id_rsa_withpass = None + git_ssh = '/tmp/git_ssh' @classmethod def tearDownClass(cls): @@ -463,7 +464,7 @@ class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin): cls.id_rsa_nopass + '.pub', cls.id_rsa_withpass, cls.id_rsa_withpass + '.pub', - cls.case.git_ssh): + cls.git_ssh): try: os.remove(os.path.join(ssh_dir, filename)) except OSError as exc: @@ -478,7 +479,6 @@ class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin): self.sshd_proc = self.find_proc(name='sshd', search=self.sshd_config) self.sshd_bin = salt.utils.which('sshd') - self.git_ssh = '/tmp/git_ssh' if self.sshd_proc is None: self.spawn_server() From 6e05417411cec66b1a79d8adbf143c3bb866e229 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 2 May 2017 09:00:44 -0600 Subject: [PATCH 19/19] Replace Nitrogen versionadded with Oxygen for develop function --- salt/runners/saltutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/runners/saltutil.py b/salt/runners/saltutil.py index 2cdb65dc4f..6bdb83d498 100644 --- a/salt/runners/saltutil.py +++ b/salt/runners/saltutil.py @@ -448,7 +448,7 @@ def sync_cache(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): def sync_fileserver(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): ''' - .. versionadded:: Nitrogen + .. versionadded:: Oxygen Sync utils modules from ``salt://_fileserver`` to the master