From 1da9f340dcf054a0bbe2c48cc75d99670483561e Mon Sep 17 00:00:00 2001 From: matt LLVW Date: Fri, 21 Sep 2018 16:08:19 +0200 Subject: [PATCH 01/50] Fix test opts in append, prepend states/file.py Fixes #49604 --- salt/states/file.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/salt/states/file.py b/salt/states/file.py index 70d808f6c0..76f217f027 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -4905,7 +4905,9 @@ def append(name, check_res, check_msg = _check_file(name) if not check_res: # Try to create the file - touch(name, makedirs=makedirs) + touch_ret = touch(name, makedirs=makedirs) + if __opts__['test']: + return touch_ret retry_res, retry_msg = _check_file(name) if not retry_res: return _error(ret, check_msg) @@ -5186,7 +5188,9 @@ def prepend(name, check_res, check_msg = _check_file(name) if not check_res: # Try to create the file - touch(name, makedirs=makedirs) + touch_ret = touch(name, makedirs=makedirs) + if __opts__['test']: + return touch_ret retry_res, retry_msg = _check_file(name) if not retry_res: return _error(ret, check_msg) From cae88f2a252f168cdb7ed975977312ee735328d0 Mon Sep 17 00:00:00 2001 From: William Giokas <1007380@gmail.com> Date: Tue, 25 Sep 2018 10:09:20 -0600 Subject: [PATCH 02/50] Move all pipelines to be fully scripted This allows us to properly clean out the workspace before a job run. --- .ci/kitchen-centos7-py2 | 134 ++++++++++++++++++------------------ .ci/kitchen-centos7-py3 | 134 ++++++++++++++++++------------------ .ci/kitchen-ubuntu1604-py2 | 134 ++++++++++++++++++------------------ .ci/kitchen-ubuntu1604-py3 | 134 ++++++++++++++++++------------------ .ci/kitchen-windows2016-py2 | 134 ++++++++++++++++++------------------ .ci/kitchen-windows2016-py3 | 134 ++++++++++++++++++------------------ 6 files changed, 408 insertions(+), 396 deletions(-) diff --git a/.ci/kitchen-centos7-py2 b/.ci/kitchen-centos7-py2 index 5cc9984cf0..7ff9d274b3 100644 --- a/.ci/kitchen-centos7-py2 +++ b/.ci/kitchen-centos7-py2 @@ -1,73 +1,75 @@ -pipeline { - agent { label 'kitchen-slave' } - options { - timestamps() - ansiColor('xterm') - } - environment { - SALT_KITCHEN_PLATFORMS = "/var/jenkins/workspace/platforms.yml" - SALT_KITCHEN_DRIVER = "/var/jenkins/workspace/driver.yml" - PATH = "/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin" - RBENV_VERSION = "2.4.2" - TEST_SUITE = "py2" - TEST_PLATFORM = "centos-7" - PY_COLORS = 1 - } - stages { - stage('github-pending') { - steps { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } - stage('setup') { - steps { - sh 'bundle install --with ec2 windows --without opennebula docker' - } - } - stage('run kitchen') { - steps { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' +node('kitchen-slave') { + timestamps { + ansiColor('xterm') { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + 'TEST_SUITE=py2', + 'TEST_PLATFORM=centos-7', + 'PY_COLORS=1', + { + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + try { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" } - }} - } - post { - always { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + stage('setup-bundle') { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + try { + stage('run kitchen') { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } } - }} - archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' - archiveArtifacts artifacts: 'artifacts/logs/minion' - archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + finally { + stage('cleanup kitchen') { + script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + }} + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } + finally { + try { + junit 'artifacts/xml-unittests-output/*.xml' + } + finally { + cleanWs notFailBuild: true + if (currentBuild.result == 'SUCCESS') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + else { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } } } } } - post { - always { - junit 'artifacts/xml-unittests-output/*.xml' - cleanWs() - } - success { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - failure { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } } diff --git a/.ci/kitchen-centos7-py3 b/.ci/kitchen-centos7-py3 index 120bd3bee9..92518be767 100644 --- a/.ci/kitchen-centos7-py3 +++ b/.ci/kitchen-centos7-py3 @@ -1,73 +1,75 @@ -pipeline { - agent { label 'kitchen-slave' } - options { - timestamps() - ansiColor('xterm') - } - environment { - SALT_KITCHEN_PLATFORMS = "/var/jenkins/workspace/platforms.yml" - SALT_KITCHEN_DRIVER = "/var/jenkins/workspace/driver.yml" - PATH = "/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin" - RBENV_VERSION = "2.4.2" - TEST_SUITE = "py3" - TEST_PLATFORM = "centos-7" - PY_COLORS = 1 - } - stages { - stage('github-pending') { - steps { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } - stage('setup') { - steps { - sh 'bundle install --with ec2 windows --without opennebula docker' - } - } - stage('run kitchen') { - steps { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' +node('kitchen-slave') { + timestamps { + ansiColor('xterm') { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + 'TEST_SUITE=py3', + 'TEST_PLATFORM=centos-7', + 'PY_COLORS=1', + { + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + try { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" } - }} - } - post { - always { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + stage('setup-bundle') { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + try { + stage('run kitchen') { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } } - }} - archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' - archiveArtifacts artifacts: 'artifacts/logs/minion' - archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + finally { + stage('cleanup kitchen') { + script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + }} + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } + finally { + try { + junit 'artifacts/xml-unittests-output/*.xml' + } + finally { + cleanWs notFailBuild: true + if (currentBuild.result == 'SUCCESS') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + else { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } } } } } - post { - always { - junit 'artifacts/xml-unittests-output/*.xml' - cleanWs() - } - success { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - failure { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } } diff --git a/.ci/kitchen-ubuntu1604-py2 b/.ci/kitchen-ubuntu1604-py2 index f4c8277cec..7fe554f266 100644 --- a/.ci/kitchen-ubuntu1604-py2 +++ b/.ci/kitchen-ubuntu1604-py2 @@ -1,73 +1,75 @@ -pipeline { - agent { label 'kitchen-slave' } - options { - timestamps() - ansiColor('xterm') - } - environment { - SALT_KITCHEN_PLATFORMS = "/var/jenkins/workspace/platforms.yml" - SALT_KITCHEN_DRIVER = "/var/jenkins/workspace/driver.yml" - PATH = "/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin" - RBENV_VERSION = "2.4.2" - TEST_SUITE = "py2" - TEST_PLATFORM = "ubuntu-1604" - PY_COLORS = 1 - } - stages { - stage('github-pending') { - steps { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } - stage('setup') { - steps { - sh 'bundle install --with ec2 windows --without opennebula docker' - } - } - stage('run kitchen') { - steps { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' +node('kitchen-slave') { + timestamps { + ansiColor('xterm') { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + 'TEST_SUITE=py2', + 'TEST_PLATFORM=ubuntu-1604', + 'PY_COLORS=1', + { + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + try { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" } - }} - } - post { - always { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + stage('setup-bundle') { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + try { + stage('run kitchen') { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } } - }} - archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' - archiveArtifacts artifacts: 'artifacts/logs/minion' - archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + finally { + stage('cleanup kitchen') { + script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + }} + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } + finally { + try { + junit 'artifacts/xml-unittests-output/*.xml' + } + finally { + cleanWs notFailBuild: true + if (currentBuild.result == 'SUCCESS') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + else { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } } } } } - post { - always { - junit 'artifacts/xml-unittests-output/*.xml' - cleanWs() - } - success { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - failure { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } } diff --git a/.ci/kitchen-ubuntu1604-py3 b/.ci/kitchen-ubuntu1604-py3 index c26f466efa..2aff3cf89d 100644 --- a/.ci/kitchen-ubuntu1604-py3 +++ b/.ci/kitchen-ubuntu1604-py3 @@ -1,73 +1,75 @@ -pipeline { - agent { label 'kitchen-slave' } - options { - timestamps() - ansiColor('xterm') - } - environment { - SALT_KITCHEN_PLATFORMS = "/var/jenkins/workspace/platforms.yml" - SALT_KITCHEN_DRIVER = "/var/jenkins/workspace/driver.yml" - PATH = "/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin" - RBENV_VERSION = "2.4.2" - TEST_SUITE = "py3" - TEST_PLATFORM = "ubuntu-1604" - PY_COLORS = 1 - } - stages { - stage('github-pending') { - steps { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } - stage('setup') { - steps { - sh 'bundle install --with ec2 windows --without opennebula docker' - } - } - stage('run kitchen') { - steps { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' +node('kitchen-slave') { + timestamps { + ansiColor('xterm') { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + 'TEST_SUITE=py3', + 'TEST_PLATFORM=ubuntu-1604', + 'PY_COLORS=1', + { + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + try { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" } - }} - } - post { - always { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + stage('setup-bundle') { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + try { + stage('run kitchen') { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } } - }} - archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' - archiveArtifacts artifacts: 'artifacts/logs/minion' - archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + finally { + stage('cleanup kitchen') { + script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + }} + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } + finally { + try { + junit 'artifacts/xml-unittests-output/*.xml' + } + finally { + cleanWs notFailBuild: true + if (currentBuild.result == 'SUCCESS') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + else { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } } } } } - post { - always { - junit 'artifacts/xml-unittests-output/*.xml' - cleanWs() - } - success { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - failure { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } } diff --git a/.ci/kitchen-windows2016-py2 b/.ci/kitchen-windows2016-py2 index d0e7320975..c8feba8ad4 100644 --- a/.ci/kitchen-windows2016-py2 +++ b/.ci/kitchen-windows2016-py2 @@ -1,73 +1,75 @@ -pipeline { - agent { label 'kitchen-slave' } - options { - timestamps() - ansiColor('xterm') - } - environment { - SALT_KITCHEN_PLATFORMS = "/var/jenkins/workspace/platforms.yml" - SALT_KITCHEN_DRIVER = "/var/jenkins/workspace/driver.yml" - PATH = "/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin" - RBENV_VERSION = "2.4.2" - TEST_SUITE = "py2" - TEST_PLATFORM = "windows-2016" - PY_COLORS = 1 - } - stages { - stage('github-pending') { - steps { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } - stage('setup') { - steps { - sh 'bundle install --with ec2 windows --without opennebula docker' - } - } - stage('run kitchen') { - steps { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' +node('kitchen-slave') { + timestamps { + ansiColor('xterm') { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + 'TEST_SUITE=py2', + 'TEST_PLATFORM=windows-2016', + 'PY_COLORS=1', + { + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + try { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" } - }} - } - post { - always { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + stage('setup-bundle') { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + try { + stage('run kitchen') { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } } - }} - archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' - archiveArtifacts artifacts: 'artifacts/logs/minion' - archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + finally { + stage('cleanup kitchen') { + script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + }} + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } + finally { + try { + junit 'artifacts/xml-unittests-output/*.xml' + } + finally { + cleanWs notFailBuild: true + if (currentBuild.result == 'SUCCESS') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + else { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } } } } } - post { - always { - junit 'artifacts/xml-unittests-output/*.xml' - cleanWs() - } - success { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - failure { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } } diff --git a/.ci/kitchen-windows2016-py3 b/.ci/kitchen-windows2016-py3 index 11c3661447..6307d190c9 100644 --- a/.ci/kitchen-windows2016-py3 +++ b/.ci/kitchen-windows2016-py3 @@ -1,73 +1,75 @@ -pipeline { - agent { label 'kitchen-slave' } - options { - timestamps() - ansiColor('xterm') - } - environment { - SALT_KITCHEN_PLATFORMS = "/var/jenkins/workspace/platforms.yml" - SALT_KITCHEN_DRIVER = "/var/jenkins/workspace/driver.yml" - PATH = "/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin" - RBENV_VERSION = "2.4.2" - TEST_SUITE = "py3" - TEST_PLATFORM = "windows-2016" - PY_COLORS = 1 - } - stages { - stage('github-pending') { - steps { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } - stage('setup') { - steps { - sh 'bundle install --with ec2 windows --without opennebula docker' - } - } - stage('run kitchen') { - steps { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' +node('kitchen-slave') { + timestamps { + ansiColor('xterm') { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + 'TEST_SUITE=py3', + 'TEST_PLATFORM=windows-2016', + 'PY_COLORS=1', + { + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm + } + try { + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" } - }} - } - post { - always { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + stage('setup-bundle') { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + try { + stage('run kitchen') { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } + } } - }} - archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' - archiveArtifacts artifacts: 'artifacts/logs/minion' - archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + finally { + stage('cleanup kitchen') { + script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + }} + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } + finally { + try { + junit 'artifacts/xml-unittests-output/*.xml' + } + finally { + cleanWs notFailBuild: true + if (currentBuild.result == 'SUCCESS') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + else { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + } } } } } - post { - always { - junit 'artifacts/xml-unittests-output/*.xml' - cleanWs() - } - success { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - failure { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } } From df3791c61bafeabc3a67f7408ac57c7e432f0dd8 Mon Sep 17 00:00:00 2001 From: Matt Phillips Date: Mon, 24 Sep 2018 13:31:27 -0400 Subject: [PATCH 03/50] state.orch: generate jid if missing via salt-run cli a jid is generated in RunnerClient via gen_async_pub and passed on via orchestration_jid if the func is orchestration, but in other callsites this jid generation doesn't happen (ie, the netapi client, since it directly invokes AsyncMixins.cmd_sync). There seems to be no obvious way to correct this at the netapi level so we just default to a new jid if orch_jid is missing. there may be other calling contexts where this could have been occurring. --- salt/runners/state.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/runners/state.py b/salt/runners/state.py index 430cb518f5..2e3d70508c 100644 --- a/salt/runners/state.py +++ b/salt/runners/state.py @@ -10,6 +10,7 @@ import logging import salt.loader import salt.utils.event import salt.utils.functools +import salt.utils.jid from salt.exceptions import SaltInvocationError LOGGER = logging.getLogger(__name__) @@ -110,6 +111,8 @@ def orchestrate(mods, pillarenv = __opts__['pillarenv'] if saltenv is None and 'saltenv' in __opts__: saltenv = __opts__['saltenv'] + if orchestration_jid is None: + orchestration_jid = salt.utils.jid.gen_jid(__opts__) running = minion.functions['state.sls']( mods, From 201697d55c8c41275c53b550ce03ebb505a1235f Mon Sep 17 00:00:00 2001 From: Wesley Whetstone Date: Wed, 26 Sep 2018 10:30:56 -0700 Subject: [PATCH 04/50] fixing an issue where sentry logger would fail to get tags from grains and some other lint fixes for this file --- salt/log/handlers/sentry_mod.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/salt/log/handlers/sentry_mod.py b/salt/log/handlers/sentry_mod.py index 7e82a8795b..51decb9315 100644 --- a/salt/log/handlers/sentry_mod.py +++ b/salt/log/handlers/sentry_mod.py @@ -112,13 +112,16 @@ __virtualname__ = 'sentry' def __virtual__(): if HAS_RAVEN is True: - __grains__ = salt.loader.grains(__opts__) - __salt__ = salt.loader.minion_mods(__opts__) return __virtualname__ return False def setup_handlers(): + ''' + sets up the sentry handler + ''' + __grains__ = salt.loader.grains(__opts__) + __salt__ = salt.loader.minion_mods(__opts__) if 'sentry_handler' not in __opts__: log.debug('No \'sentry_handler\' key was found in the configuration') return False @@ -132,7 +135,9 @@ def setup_handlers(): transport_registry = TransportRegistry(default_transports) url = urlparse(dsn) if not transport_registry.supported_scheme(url.scheme): - raise ValueError('Unsupported Sentry DSN scheme: {0}'.format(url.scheme)) + raise ValueError( + 'Unsupported Sentry DSN scheme: %s', url.scheme + ) except ValueError as exc: log.info( 'Raven failed to parse the configuration provided DSN: %s', exc @@ -201,7 +206,11 @@ def setup_handlers(): context_dict = {} if context is not None: for tag in context: - tag_value = __salt__['grains.get'](tag) + try: + tag_value = __grains__[tag] + except KeyError: + log.debug('Sentry tag \'%s\' not found in grains.', tag) + continue if len(tag_value) > 0: context_dict[tag] = tag_value if len(context_dict) > 0: @@ -215,4 +224,7 @@ def setup_handlers(): def get_config_value(name, default=None): + ''' + returns a configuration option for the sentry_handler + ''' return __opts__['sentry_handler'].get(name, default) From 5b16996e20abf9d0276c30ac1a910155f697a9c8 Mon Sep 17 00:00:00 2001 From: William Giokas <1007380@gmail.com> Date: Wed, 26 Sep 2018 12:12:33 -0600 Subject: [PATCH 05/50] Fix the new pipelines --- .ci/kitchen-centos7-py2 | 2 +- .ci/kitchen-centos7-py3 | 2 +- .ci/kitchen-ubuntu1604-py2 | 2 +- .ci/kitchen-ubuntu1604-py3 | 2 +- .ci/kitchen-windows2016-py2 | 2 +- .ci/kitchen-windows2016-py3 | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/kitchen-centos7-py2 b/.ci/kitchen-centos7-py2 index 7ff9d274b3..c04ad78aed 100644 --- a/.ci/kitchen-centos7-py2 +++ b/.ci/kitchen-centos7-py2 @@ -8,7 +8,7 @@ node('kitchen-slave') { 'RBENV_VERSION=2.4.2', 'TEST_SUITE=py2', 'TEST_PLATFORM=centos-7', - 'PY_COLORS=1', + 'PY_COLORS=1',]) { stage('checkout-scm') { cleanWs notFailBuild: true diff --git a/.ci/kitchen-centos7-py3 b/.ci/kitchen-centos7-py3 index 92518be767..25fd8b8f54 100644 --- a/.ci/kitchen-centos7-py3 +++ b/.ci/kitchen-centos7-py3 @@ -8,7 +8,7 @@ node('kitchen-slave') { 'RBENV_VERSION=2.4.2', 'TEST_SUITE=py3', 'TEST_PLATFORM=centos-7', - 'PY_COLORS=1', + 'PY_COLORS=1',]) { stage('checkout-scm') { cleanWs notFailBuild: true diff --git a/.ci/kitchen-ubuntu1604-py2 b/.ci/kitchen-ubuntu1604-py2 index 7fe554f266..94ca29cc36 100644 --- a/.ci/kitchen-ubuntu1604-py2 +++ b/.ci/kitchen-ubuntu1604-py2 @@ -8,7 +8,7 @@ node('kitchen-slave') { 'RBENV_VERSION=2.4.2', 'TEST_SUITE=py2', 'TEST_PLATFORM=ubuntu-1604', - 'PY_COLORS=1', + 'PY_COLORS=1',]) { stage('checkout-scm') { cleanWs notFailBuild: true diff --git a/.ci/kitchen-ubuntu1604-py3 b/.ci/kitchen-ubuntu1604-py3 index 2aff3cf89d..6a7d167162 100644 --- a/.ci/kitchen-ubuntu1604-py3 +++ b/.ci/kitchen-ubuntu1604-py3 @@ -8,7 +8,7 @@ node('kitchen-slave') { 'RBENV_VERSION=2.4.2', 'TEST_SUITE=py3', 'TEST_PLATFORM=ubuntu-1604', - 'PY_COLORS=1', + 'PY_COLORS=1',]) { stage('checkout-scm') { cleanWs notFailBuild: true diff --git a/.ci/kitchen-windows2016-py2 b/.ci/kitchen-windows2016-py2 index c8feba8ad4..9ef81550d9 100644 --- a/.ci/kitchen-windows2016-py2 +++ b/.ci/kitchen-windows2016-py2 @@ -8,7 +8,7 @@ node('kitchen-slave') { 'RBENV_VERSION=2.4.2', 'TEST_SUITE=py2', 'TEST_PLATFORM=windows-2016', - 'PY_COLORS=1', + 'PY_COLORS=1',]) { stage('checkout-scm') { cleanWs notFailBuild: true diff --git a/.ci/kitchen-windows2016-py3 b/.ci/kitchen-windows2016-py3 index 6307d190c9..fcd90f7086 100644 --- a/.ci/kitchen-windows2016-py3 +++ b/.ci/kitchen-windows2016-py3 @@ -8,7 +8,7 @@ node('kitchen-slave') { 'RBENV_VERSION=2.4.2', 'TEST_SUITE=py3', 'TEST_PLATFORM=windows-2016', - 'PY_COLORS=1', + 'PY_COLORS=1',]) { stage('checkout-scm') { cleanWs notFailBuild: true From ac0f8000d8102247cb1014c83426a46c6e29a76b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 26 Sep 2018 12:44:20 -0500 Subject: [PATCH 06/50] Fix 3 bugs in subdict matching 1. Non-ascii was causing a UnicodeDecodeError when matching 2. Matching a dict value after a wildcard (i.e. the key is the wildcard) did not work. This is because it was matching the pattern against the key, instead of the value as it should have. 3. It was not possible to use a wildcard for the first level of matching. --- salt/utils/data.py | 65 ++++++++++++++++++++++++----------- tests/unit/utils/test_data.py | 30 ++++++++++++++++ 2 files changed, 75 insertions(+), 20 deletions(-) diff --git a/salt/utils/data.py b/salt/utils/data.py index 35c77ecd49..3740d88d34 100644 --- a/salt/utils/data.py +++ b/salt/utils/data.py @@ -455,13 +455,14 @@ def traverse_dict(data, key, default=None, delimiter=DEFAULT_TARGET_DELIM): data['foo']['bar']['baz'] if this value exists, and will otherwise return the dict in the default argument. ''' + ptr = data try: for each in key.split(delimiter): - data = data[each] + ptr = ptr[each] except (KeyError, IndexError, TypeError): # Encountered a non-indexable value in the middle of traversing return default - return data + return ptr @jinja_filter('traverse') @@ -476,16 +477,17 @@ def traverse_dict_and_list(data, key, default=None, delimiter=DEFAULT_TARGET_DEL {'foo':{'bar':['baz']}} , if data like {'foo':{'bar':{'0':'baz'}}} then return data['foo']['bar']['0'] ''' + ptr = data for each in key.split(delimiter): - if isinstance(data, list): + if isinstance(ptr, list): try: idx = int(each) except ValueError: embed_match = False # Index was not numeric, lets look at any embedded dicts - for embedded in (x for x in data if isinstance(x, dict)): + for embedded in (x for x in ptr if isinstance(x, dict)): try: - data = embedded[each] + ptr = embedded[each] embed_match = True break except KeyError: @@ -495,15 +497,15 @@ def traverse_dict_and_list(data, key, default=None, delimiter=DEFAULT_TARGET_DEL return default else: try: - data = data[idx] + ptr = ptr[idx] except IndexError: return default else: try: - data = data[each] + ptr = ptr[each] except (KeyError, TypeError): return default - return data + return ptr def subdict_match(data, @@ -519,16 +521,28 @@ def subdict_match(data, latter. ''' def _match(target, pattern, regex_match=False, exact_match=False): + try: + target = six.text_type(target).lower() + except UnicodeDecodeError: + # We're on PY2 and target is a str type with non-ascii chars. Coax + # it into a unicode type. + target = salt.utils.stringutils.to_unicode(target).lower() + try: + pattern = six.text_type(pattern).lower() + except UnicodeDecodeError: + # We're on PY2 and pattern is a str type with non-ascii chars. Coax + # it into a unicode type. + pattern = salt.utils.stringutils.to_unicode(pattern).lower() + if regex_match: try: - return re.match(pattern.lower(), six.text_type(target).lower()) + return re.match(pattern, target) except Exception: log.error('Invalid regex \'%s\' in match', pattern) return False - elif exact_match: - return six.text_type(target).lower() == pattern.lower() else: - return fnmatch.fnmatch(six.text_type(target).lower(), pattern.lower()) + return target == pattern if exact_match \ + else fnmatch.fnmatch(target, pattern) def _dict_match(target, pattern, regex_match=False, exact_match=False): wildcard = pattern.startswith('*:') @@ -548,11 +562,6 @@ def subdict_match(data, return True if wildcard: for key in target: - if _match(key, - pattern, - regex_match=regex_match, - exact_match=exact_match): - return True if isinstance(target[key], dict): if _dict_match(target[key], pattern, @@ -566,15 +575,31 @@ def subdict_match(data, regex_match=regex_match, exact_match=exact_match): return True + elif _match(target[key], + pattern, + regex_match=regex_match, + exact_match=exact_match): + return True + return False + + splits = expr.split(delimiter) + num_splits = len(splits) + if num_splits == 1: + # Delimiter not present, this can't possibly be a match return False for idx in range(1, expr.count(delimiter) + 1): - splits = expr.split(delimiter) key = delimiter.join(splits[:idx]) - matchstr = delimiter.join(splits[idx:]) + if key == '*': + # We are matching on everything under the top level, so we need to + # treat the match as the entire data being passed in + matchstr = expr + match = data + else: + matchstr = delimiter.join(splits[idx:]) + match = traverse_dict_and_list(data, key, {}, delimiter=delimiter) log.debug("Attempting to match '%s' in '%s' using delimiter '%s'", matchstr, key, delimiter) - match = traverse_dict_and_list(data, key, {}, delimiter=delimiter) if match == {}: continue if isinstance(match, dict): diff --git a/tests/unit/utils/test_data.py b/tests/unit/utils/test_data.py index 78f4144f9f..030f22b202 100644 --- a/tests/unit/utils/test_data.py +++ b/tests/unit/utils/test_data.py @@ -144,6 +144,36 @@ class DataTestCase(TestCase): ) ) + def test_subdict_match_with_wildcards(self): + ''' + Tests subdict matching when wildcards are used in the expression + ''' + data = { + 'a': { + 'b': { + 'ç': 'd', + 'é': ['eff', 'gee', '8ch'], + 'ĩ': {'j': 'k'} + } + } + } + assert salt.utils.data.subdict_match(data, '*:*:*:*') + assert salt.utils.data.subdict_match(data, 'a:*:*:*') + assert salt.utils.data.subdict_match(data, 'a:b:*:*') + assert salt.utils.data.subdict_match(data, 'a:b:ç:*') + assert salt.utils.data.subdict_match(data, 'a:b:*:d') + assert salt.utils.data.subdict_match(data, 'a:*:ç:d') + assert salt.utils.data.subdict_match(data, '*:b:ç:d') + assert salt.utils.data.subdict_match(data, '*:*:ç:d') + assert salt.utils.data.subdict_match(data, '*:*:*:d') + assert salt.utils.data.subdict_match(data, 'a:*:*:d') + assert salt.utils.data.subdict_match(data, 'a:b:*:ef*') + assert salt.utils.data.subdict_match(data, 'a:b:*:g*') + assert salt.utils.data.subdict_match(data, 'a:b:*:j:*') + assert salt.utils.data.subdict_match(data, 'a:b:*:j:k') + assert salt.utils.data.subdict_match(data, 'a:b:*:*:k') + assert salt.utils.data.subdict_match(data, 'a:b:*:*:*') + def test_traverse_dict(self): test_two_level_dict = {'foo': {'bar': 'baz'}} From 6795472a3e0732ce44239d33572b5d1b0805c28f Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 26 Sep 2018 13:32:42 -0500 Subject: [PATCH 07/50] Add clarifying comment --- salt/utils/data.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/salt/utils/data.py b/salt/utils/data.py index 3740d88d34..c69ba900e2 100644 --- a/salt/utils/data.py +++ b/salt/utils/data.py @@ -521,17 +521,22 @@ def subdict_match(data, latter. ''' def _match(target, pattern, regex_match=False, exact_match=False): + # The reason for using six.text_type first and _then_ using + # to_unicode as a fallback is because we want to eventually have + # unicode types for comparison below. If either value is numeric then + # six.text_type will turn it into a unicode string. However, if the + # value is a PY2 str type with non-ascii chars, then the result will be + # a UnicodeDecodeError. In those cases, we simply use to_unicode to + # decode it to unicode. The reason we can't simply use to_unicode to + # begin with is that (by design) to_unicode will raise a TypeError if a + # non-string/bytestring/bytearray value is passed. try: target = six.text_type(target).lower() except UnicodeDecodeError: - # We're on PY2 and target is a str type with non-ascii chars. Coax - # it into a unicode type. target = salt.utils.stringutils.to_unicode(target).lower() try: pattern = six.text_type(pattern).lower() except UnicodeDecodeError: - # We're on PY2 and pattern is a str type with non-ascii chars. Coax - # it into a unicode type. pattern = salt.utils.stringutils.to_unicode(pattern).lower() if regex_match: From cf5c179fd9db99b5ebdd0036b06438608501d105 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 26 Sep 2018 14:47:46 -0500 Subject: [PATCH 08/50] fix test_managed_file_with_grains_data Closes saltstack/salt-jenkins#1121 --- tests/integration/files/conf/minion | 1 + tests/integration/states/test_file.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/files/conf/minion b/tests/integration/files/conf/minion index 371e9260d1..bb93151aa1 100644 --- a/tests/integration/files/conf/minion +++ b/tests/integration/files/conf/minion @@ -25,6 +25,7 @@ integration.test: True # Grains addons grains: test_grain: cheese + grain_path: /tmp/salt-tests-tmpdir/file-grain-test script: grail alot: many planets: diff --git a/tests/integration/states/test_file.py b/tests/integration/states/test_file.py index 8f618cbb50..b34f74cde3 100644 --- a/tests/integration/states/test_file.py +++ b/tests/integration/states/test_file.py @@ -355,7 +355,6 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): file. ''' grain_path = os.path.join(TMP, 'file-grain-test') - self.run_function('grains.set', ['grain_path', grain_path]) state_file = 'file-grainget' self.run_function('state.sls', [state_file]) From 5b02548e3b56b0bd7d15e3d3baf88e5d6d4e0dd7 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 26 Sep 2018 17:55:00 +0200 Subject: [PATCH 09/50] Add error logging --- salt/modules/zypper.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index 473f9d5b66..41aed80ac9 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -187,7 +187,15 @@ class _Zypper(object): :return: ''' - return self.exit_code not in self.SUCCESS_EXIT_CODES + if self.exit_code: + msg = self.SUCCESS_EXIT_CODES.get(self.exit_code) + if msg: + log.info(msg) + msg = self.WARNING_EXIT_CODES.get(self.exit_code) + if msg: + log.warning(msg) + + return self.exit_code not in self.SUCCESS_EXIT_CODES and self.exit_code not in self.WARNING_EXIT_CODES def _is_lock(self): ''' From 756ef777aa267803261cf6f2169a3712c16ea8eb Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 26 Sep 2018 17:54:53 +0200 Subject: [PATCH 10/50] Update error list for zypper --- salt/modules/zypper.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index 41aed80ac9..d901326754 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -74,7 +74,25 @@ class _Zypper(object): Allows serial zypper calls (first came, first won). ''' - SUCCESS_EXIT_CODES = [0, 100, 101, 102, 103] + SUCCESS_EXIT_CODES = { + 0: 'Successful run of zypper with no special info.', + 100: 'Patches are available for installation.', + 101: 'Security patches are available for installation.', + 102: 'Installation successful, reboot required.', + 103: 'Installation succesful, restart of the package manager itself required.', + } + + WARNING_EXIT_CODES = { + 6: 'No repositories are defined.', + 7: 'The ZYPP library is locked.', + 106: 'Some repository had to be disabled temporarily because it failed to refresh. ' + 'You should check your repository configuration (e.g. zypper ref -f).', + 107: 'Installation basically succeeded, but some of the packages %post install scripts returned an error. ' + 'These packages were successfully unpacked to disk and are registered in the rpm database, ' + 'but due to the failed install script they may not work as expected. The failed scripts output might ' + 'reveal what actually went wrong. Any scripts output is also logged to /var/log/zypp/history.' + } + LOCK_EXIT_CODE = 7 XML_DIRECTIVES = ['-x', '--xmlout'] ZYPPER_LOCK = '/var/run/zypp.pid' From 26461f969d1d633111b0ab6b99be29b3752db222 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 27 Sep 2018 15:12:29 -0400 Subject: [PATCH 11/50] Add helper import comments --- salt/grains/chronos.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/grains/chronos.py b/salt/grains/chronos.py index df8eca32e0..3b5add6895 100644 --- a/salt/grains/chronos.py +++ b/salt/grains/chronos.py @@ -5,9 +5,11 @@ Generate chronos proxy minion grains. .. versionadded:: 2015.8.2 ''' +# Import Python libs from __future__ import absolute_import, print_function, unicode_literals +# Import Salt libs import salt.utils.http import salt.utils.platform __proxyenabled__ = ['chronos'] From b4f1a72bbb8a5a2ec0e6c15605c3cb1601f651ab Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 28 Sep 2018 15:22:33 +0200 Subject: [PATCH 12/50] Add missing docstrings --- salt/_compat.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/_compat.py b/salt/_compat.py index 9b10646ace..ce41b4c7f0 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -78,11 +78,23 @@ def bytes_(s, encoding='latin-1', errors='strict'): if PY3: def ascii_native_(s): + ''' + Python 3 handler. + + :param s: + :return: + ''' if isinstance(s, text_type): s = s.encode('ascii') return str(s, 'ascii', 'strict') else: def ascii_native_(s): + ''' + Python 2 handler. + + :param s: + :return: + ''' if isinstance(s, text_type): s = s.encode('ascii') return str(s) From 0498e946f9b499a410e55e56e28be781428366ef Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 28 Sep 2018 15:34:05 +0200 Subject: [PATCH 13/50] Fix ipaddress imports --- salt/_compat.py | 12 ++++++++---- salt/cloud/clouds/saltify.py | 5 +---- salt/cloud/clouds/vagrant.py | 6 ++---- salt/ext/win_inet_pton.py | 2 +- salt/minion.py | 5 +---- salt/modules/ipset.py | 5 +---- salt/modules/network.py | 5 +---- salt/modules/vagrant.py | 6 +----- salt/utils/minions.py | 5 +---- tests/unit/grains/test_core.py | 5 +---- tests/unit/modules/test_network.py | 14 +++----------- 11 files changed, 21 insertions(+), 49 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index ce41b4c7f0..22fe19b6cb 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -145,7 +145,11 @@ def string_io(data=None): # cStringIO can't handle unicode except (UnicodeEncodeError, TypeError): return StringIO(data) -if PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress + +try: + if PY3: + import ipaddress + else: + import salt.ext.ipaddress as ipaddress +except ImportError: + ipaddress = None diff --git a/salt/cloud/clouds/saltify.py b/salt/cloud/clouds/saltify.py index c9cc281b42..e0e56349a0 100644 --- a/salt/cloud/clouds/saltify.py +++ b/salt/cloud/clouds/saltify.py @@ -27,10 +27,7 @@ import salt.utils.cloud import salt.config as config import salt.client import salt.ext.six as six -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress from salt.exceptions import SaltCloudException, SaltCloudSystemExit diff --git a/salt/cloud/clouds/vagrant.py b/salt/cloud/clouds/vagrant.py index a24170c78a..9d6032e179 100644 --- a/salt/cloud/clouds/vagrant.py +++ b/salt/cloud/clouds/vagrant.py @@ -26,10 +26,8 @@ import salt.utils import salt.config as config import salt.client import salt.ext.six as six -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress + from salt.exceptions import SaltCloudException, SaltCloudSystemExit, \ SaltInvocationError diff --git a/salt/ext/win_inet_pton.py b/salt/ext/win_inet_pton.py index 1204bede10..89aba14ce9 100644 --- a/salt/ext/win_inet_pton.py +++ b/salt/ext/win_inet_pton.py @@ -9,7 +9,7 @@ from __future__ import absolute_import import socket import ctypes import os -import ipaddress +from salt._compat import ipaddress import salt.ext.six as six diff --git a/salt/minion.py b/salt/minion.py index ad9a672362..273c1dcbd2 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -26,10 +26,7 @@ from binascii import crc32 # Import Salt Libs # pylint: disable=import-error,no-name-in-module,redefined-builtin from salt.ext import six -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress from salt.ext.six.moves import range from salt.utils.zeromq import zmq, ZMQDefaultLoop, install_zmq, ZMQ_VERSION_INFO diff --git a/salt/modules/ipset.py b/salt/modules/ipset.py index 7047e84c29..1a0fa0044d 100644 --- a/salt/modules/ipset.py +++ b/salt/modules/ipset.py @@ -13,10 +13,7 @@ from salt.ext.six.moves import map, range import salt.utils.path # Import third-party libs -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress # Set up logging log = logging.getLogger(__name__) diff --git a/salt/modules/network.py b/salt/modules/network.py index 92893572a6..60f586f6bc 100644 --- a/salt/modules/network.py +++ b/salt/modules/network.py @@ -26,10 +26,7 @@ from salt.exceptions import CommandExecutionError # Import 3rd-party libs from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,no-name-in-module,redefined-builtin -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress log = logging.getLogger(__name__) diff --git a/salt/modules/vagrant.py b/salt/modules/vagrant.py index 0592dede55..0f518c2602 100644 --- a/salt/modules/vagrant.py +++ b/salt/modules/vagrant.py @@ -39,11 +39,7 @@ import salt.utils.path import salt.utils.stringutils from salt.exceptions import CommandExecutionError, SaltInvocationError import salt.ext.six as six - -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress log = logging.getLogger(__name__) diff --git a/salt/utils/minions.py b/salt/utils/minions.py index 25c2a22ecd..2ad3a0ea69 100644 --- a/salt/utils/minions.py +++ b/salt/utils/minions.py @@ -26,10 +26,7 @@ import salt.cache from salt.ext import six # Import 3rd-party libs -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress HAS_RANGE = False try: import seco.range # pylint: disable=import-error diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py index 56771f6fab..5a4f7b764f 100644 --- a/tests/unit/grains/test_core.py +++ b/tests/unit/grains/test_core.py @@ -33,10 +33,7 @@ import salt.grains.core as core # Import 3rd-party libs from salt.ext import six -if six.PY3: - import ipaddress -else: - import salt.ext.ipaddress as ipaddress +from salt._compat import ipaddress log = logging.getLogger(__name__) diff --git a/tests/unit/modules/test_network.py b/tests/unit/modules/test_network.py index d616d8b2ba..4cc51b0a28 100644 --- a/tests/unit/modules/test_network.py +++ b/tests/unit/modules/test_network.py @@ -25,15 +25,7 @@ import salt.utils.network import salt.utils.path import salt.modules.network as network from salt.exceptions import CommandExecutionError -if six.PY2: - import salt.ext.ipaddress as ipaddress - HAS_IPADDRESS = True -else: - try: - import ipaddress - HAS_IPADDRESS = True - except ImportError: - HAS_IPADDRESS = False +from salt._compat import ipaddress @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -276,7 +268,7 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): self.assertDictEqual(network.connect('host', 'port'), {'comment': ret, 'result': True}) - @skipIf(HAS_IPADDRESS is False, 'unable to import \'ipaddress\'') + @skipIf(bool(ipaddress), 'unable to import \'ipaddress\'') def test_is_private(self): ''' Test for Check if the given IP address is a private address @@ -288,7 +280,7 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): return_value=True): self.assertTrue(network.is_private('::1')) - @skipIf(HAS_IPADDRESS is False, 'unable to import \'ipaddress\'') + @skipIf(bool(ipaddress), 'unable to import \'ipaddress\'') def test_is_loopback(self): ''' Test for Check if the given IP address is a loopback address From 561c99e6716cfaf87b7984822c245ac8f89bcfd2 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 28 Sep 2018 15:34:59 +0200 Subject: [PATCH 14/50] Remove unused import --- salt/cloud/clouds/vagrant.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/cloud/clouds/vagrant.py b/salt/cloud/clouds/vagrant.py index 9d6032e179..364c3ccaeb 100644 --- a/salt/cloud/clouds/vagrant.py +++ b/salt/cloud/clouds/vagrant.py @@ -25,7 +25,6 @@ import tempfile import salt.utils import salt.config as config import salt.client -import salt.ext.six as six from salt._compat import ipaddress from salt.exceptions import SaltCloudException, SaltCloudSystemExit, \ From bbeec4d60c035ef791902383c8b7e011b72d5382 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 28 Sep 2018 15:35:51 +0200 Subject: [PATCH 15/50] Fix ipaddress import --- salt/cloud/clouds/vagrant.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/cloud/clouds/vagrant.py b/salt/cloud/clouds/vagrant.py index 364c3ccaeb..0fe410eb91 100644 --- a/salt/cloud/clouds/vagrant.py +++ b/salt/cloud/clouds/vagrant.py @@ -26,9 +26,7 @@ import salt.utils import salt.config as config import salt.client from salt._compat import ipaddress - -from salt.exceptions import SaltCloudException, SaltCloudSystemExit, \ - SaltInvocationError +from salt.exceptions import SaltCloudException, SaltCloudSystemExit, SaltInvocationError # Get logging started log = logging.getLogger(__name__) From 5e970eaad1b1c713e2bb0d38f406e6ce2a17b1c5 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 28 Sep 2018 15:37:00 +0200 Subject: [PATCH 16/50] Fix unicode imports in compat --- salt/_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/_compat.py b/salt/_compat.py index 22fe19b6cb..a0fa5e7b82 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -5,7 +5,7 @@ Salt compatibility code # pylint: disable=import-error,unused-import,invalid-name # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals, print_function import sys import types From 1ae94f11a6b177fc7119736963f59a77a77c7d5f Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 18 Sep 2018 17:52:32 +0200 Subject: [PATCH 17/50] Override standard IPv6Address class --- salt/_compat.py | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/salt/_compat.py b/salt/_compat.py index a0fa5e7b82..1a09a9835c 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -10,6 +10,7 @@ import sys import types # Import 3rd-party libs +from salt.exceptions import SaltException from salt.ext.six import binary_type, string_types, text_type from salt.ext.six.moves import cStringIO, StringIO @@ -153,3 +154,64 @@ try: import salt.ext.ipaddress as ipaddress except ImportError: ipaddress = None + + +class IPv6AddressScoped(ipaddress.IPv6Address): + ''' + Represent and manipulate single IPv6 Addresses. + Scope-aware version + ''' + def __init__(self, address): + ''' + Instantiate a new IPv6 address object. Scope is moved to an attribute 'scope'. + + Args: + address: A string or integer representing the IP + + Additionally, an integer can be passed, so + IPv6Address('2001:db8::') == IPv6Address(42540766411282592856903984951653826560) + or, more generally + IPv6Address(int(IPv6Address('2001:db8::'))) == IPv6Address('2001:db8::') + + Raises: + AddressValueError: If address isn't a valid IPv6 address. + + :param address: + ''' + if '%' in address: + buff = address.split('%') + if len(buff) != 2: + raise SaltException('Invalid IPv6 address: "{}"'.format(address)) + address, self.__scope = buff + else: + self.__scope = None + + ipaddress._BaseAddress.__init__(self, address) + ipaddress._BaseV6.__init__(self, address) + + # Efficient constructor from integer. + if isinstance(address, int): + self._check_int_address(address) + self._ip = address + return + + if isinstance(address, bytes): + self._check_packed_address(address, 16) + self._ip = ipaddress._int_from_bytes(address, 'big') + return + + addr_str = str(address) + self._ip = self._ip_int_from_string(addr_str) + + @property + def scope(self): + ''' + Return scope of IPv6 address. + + :return: + ''' + return self.__scope + + +if ipaddress: + ipaddress.IPv6Address = IPv6AddressScoped From 49b32b76a2f5fbb43067d5a0697d545f11eaddf5 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 18 Sep 2018 23:27:56 +0200 Subject: [PATCH 18/50] Check version via object --- salt/_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/_compat.py b/salt/_compat.py index 1a09a9835c..fa08e19419 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -36,7 +36,7 @@ except Exception: # True if we are running on Python 3. -PY3 = sys.version_info[0] == 3 +PY3 = sys.version_info.major == 3 if PY3: From eeab9f2f8e7cb3e4c2ea8052572a4955c0dbc2fa Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 18 Sep 2018 23:28:25 +0200 Subject: [PATCH 19/50] Isolate Py2 and Py3 mode --- salt/_compat.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index fa08e19419..ac20b42268 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -11,7 +11,7 @@ import types # Import 3rd-party libs from salt.exceptions import SaltException -from salt.ext.six import binary_type, string_types, text_type +from salt.ext.six import binary_type, string_types, text_type, integer_types from salt.ext.six.moves import cStringIO, StringIO HAS_XML = True @@ -178,7 +178,7 @@ class IPv6AddressScoped(ipaddress.IPv6Address): :param address: ''' - if '%' in address: + if isinstance(address, string_types) and '%' in address: buff = address.split('%') if len(buff) != 2: raise SaltException('Invalid IPv6 address: "{}"'.format(address)) @@ -186,11 +186,12 @@ class IPv6AddressScoped(ipaddress.IPv6Address): else: self.__scope = None - ipaddress._BaseAddress.__init__(self, address) - ipaddress._BaseV6.__init__(self, address) + if sys.version_info.major == 2: + ipaddress._BaseAddress.__init__(self, address) + ipaddress._BaseV6.__init__(self, address) # Efficient constructor from integer. - if isinstance(address, int): + if isinstance(address, integer_types): self._check_int_address(address) self._ip = address return From cab5f62772b12261d4e62de33d33435801fc1630 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 18 Sep 2018 23:39:49 +0200 Subject: [PATCH 20/50] Add logging --- salt/_compat.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/_compat.py b/salt/_compat.py index ac20b42268..d81499f2a1 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -8,12 +8,15 @@ Salt compatibility code from __future__ import absolute_import, unicode_literals, print_function import sys import types +import logging # Import 3rd-party libs from salt.exceptions import SaltException from salt.ext.six import binary_type, string_types, text_type, integer_types from salt.ext.six.moves import cStringIO, StringIO +log = logging.getLogger(__name__) + HAS_XML = True try: # Python >2.5 From b40e1e1dded3721fee076596e5e12a7e7971e6c8 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 18 Sep 2018 23:40:27 +0200 Subject: [PATCH 21/50] Add debugging to the ip_address method (py2 and py3) --- salt/_compat.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/salt/_compat.py b/salt/_compat.py index d81499f2a1..c8c619d6d7 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -217,5 +217,40 @@ class IPv6AddressScoped(ipaddress.IPv6Address): return self.__scope +def ip_address(address): + """Take an IP string/int and return an object of the correct type. + + Args: + address: A string or integer, the IP address. Either IPv4 or + IPv6 addresses may be supplied; integers less than 2**32 will + be considered to be IPv4 by default. + + Returns: + An IPv4Address or IPv6Address object. + + Raises: + ValueError: if the *address* passed isn't either a v4 or a v6 + address + + """ + try: + return ipaddress.IPv4Address(address) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError): + log.debug('Error while parsing IPv4 address: %s', address, exc_info=True) + + try: + return IPv6AddressScoped(address) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError): + log.debug('Error while parsing IPv6 address: %s', address, exc_info=True) + + if isinstance(address, bytes): + raise ipaddress.AddressValueError('{} does not appear to be an IPv4 or IPv6 address. ' + 'Did you pass in a bytes (str in Python 2) instead ' + 'of a unicode object?'.format(repr(address))) + + raise ValueError('{} does not appear to be an IPv4 or IPv6 address'.format(repr(address))) + + if ipaddress: ipaddress.IPv6Address = IPv6AddressScoped + ipaddress.ip_address = ip_address From 7a6ea3b2263f349efe078b41ace479bab2f8ffba Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 18 Sep 2018 23:41:01 +0200 Subject: [PATCH 22/50] Remove multiple returns and add check for address syntax --- salt/_compat.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index c8c619d6d7..87f76ebb05 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -197,15 +197,14 @@ class IPv6AddressScoped(ipaddress.IPv6Address): if isinstance(address, integer_types): self._check_int_address(address) self._ip = address - return - - if isinstance(address, bytes): + elif isinstance(address, bytes): self._check_packed_address(address, 16) self._ip = ipaddress._int_from_bytes(address, 'big') - return - - addr_str = str(address) - self._ip = self._ip_int_from_string(addr_str) + else: + address = str(address) + if '/' in address: + raise ipaddress.AddressValueError("Unexpected '/' in {}".format(address)) + self._ip = self._ip_int_from_string(address) @property def scope(self): From 8e62633fd58e8f844f829c006d4d30d876fb5085 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 18 Sep 2018 23:45:30 +0200 Subject: [PATCH 23/50] Remove unnecessary variable for import detection --- salt/_compat.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index 87f76ebb05..651a885f02 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -17,7 +17,6 @@ from salt.ext.six.moves import cStringIO, StringIO log = logging.getLogger(__name__) -HAS_XML = True try: # Python >2.5 import xml.etree.cElementTree as ElementTree @@ -35,7 +34,6 @@ except Exception: import elementtree.ElementTree as ElementTree except Exception: ElementTree = None - HAS_XML = False # True if we are running on Python 3. @@ -49,7 +47,7 @@ else: import exceptions -if HAS_XML: +if ElementTree is not None: if not hasattr(ElementTree, 'ParseError'): class ParseError(Exception): ''' From deb0b4e0ed1bafd7d8868c089803289ba834a61a Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 18 Sep 2018 23:45:48 +0200 Subject: [PATCH 24/50] Remove duplicated code --- salt/_compat.py | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index 651a885f02..ecc55acf12 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -78,28 +78,21 @@ def bytes_(s, encoding='latin-1', errors='strict'): return s -if PY3: - def ascii_native_(s): - ''' - Python 3 handler. +def ascii_native_(s): + ''' + Python 2/3 handler. - :param s: - :return: - ''' - if isinstance(s, text_type): - s = s.encode('ascii') - return str(s, 'ascii', 'strict') -else: - def ascii_native_(s): - ''' - Python 2 handler. + :param s: + :return: + ''' + if isinstance(s, text_type): + s = s.encode('ascii') + if PY3: + s = str(s, 'ascii', 'strict') + else: + s = str(s) + return s - :param s: - :return: - ''' - if isinstance(s, text_type): - s = s.encode('ascii') - return str(s) ascii_native_.__doc__ = ''' Python 3: If ``s`` is an instance of ``text_type``, return From 3fde850640bee232648ac8e363ed9f79cc805a9c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 18 Sep 2018 23:55:42 +0200 Subject: [PATCH 25/50] Remove unnecessary operator --- salt/_compat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/_compat.py b/salt/_compat.py index ecc55acf12..d328b35f9e 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -53,7 +53,6 @@ if ElementTree is not None: ''' older versions of ElementTree do not have ParseError ''' - pass ElementTree.ParseError = ParseError From d05999e28da4fe13a532c5829284af405528689b Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 18 Sep 2018 23:56:04 +0200 Subject: [PATCH 26/50] Remove multiple returns --- salt/_compat.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index d328b35f9e..9f9d7fd434 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -62,9 +62,7 @@ def text_(s, encoding='latin-1', errors='strict'): If ``s`` is an instance of ``binary_type``, return ``s.decode(encoding, errors)``, otherwise return ``s`` ''' - if isinstance(s, binary_type): - return s.decode(encoding, errors) - return s + return s.decode(encoding, errors) if isinstance(s, binary_type) else s def bytes_(s, encoding='latin-1', errors='strict'): @@ -72,9 +70,7 @@ def bytes_(s, encoding='latin-1', errors='strict'): If ``s`` is an instance of ``text_type``, return ``s.encode(encoding, errors)``, otherwise return ``s`` ''' - if isinstance(s, text_type): - return s.encode(encoding, errors) - return s + return s.encode(encoding, errors) if isinstance(s, text_type) else s def ascii_native_(s): From 40d0e6bed2c67421e0c63ce02de0bc56105b4571 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 18 Sep 2018 23:56:33 +0200 Subject: [PATCH 27/50] Use ternary operator instead --- salt/_compat.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index 9f9d7fd434..fee9e92c90 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -82,11 +82,8 @@ def ascii_native_(s): ''' if isinstance(s, text_type): s = s.encode('ascii') - if PY3: - s = str(s, 'ascii', 'strict') - else: - s = str(s) - return s + + return str(s, 'ascii', 'strict') if PY3 else s ascii_native_.__doc__ = ''' From 77f1c446387736fdb4cad26aa5e7aa493925c41d Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 18 Sep 2018 23:56:51 +0200 Subject: [PATCH 28/50] Remove duplicated code --- salt/_compat.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index fee9e92c90..5b208e63f3 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -95,24 +95,17 @@ Python 2: If ``s`` is an instance of ``text_type``, return ''' -if PY3: - def native_(s, encoding='latin-1', errors='strict'): - ''' - If ``s`` is an instance of ``text_type``, return - ``s``, otherwise return ``str(s, encoding, errors)`` - ''' - if isinstance(s, text_type): - return s - return str(s, encoding, errors) -else: - def native_(s, encoding='latin-1', errors='strict'): - ''' - If ``s`` is an instance of ``text_type``, return - ``s.encode(encoding, errors)``, otherwise return ``str(s)`` - ''' - if isinstance(s, text_type): - return s.encode(encoding, errors) - return str(s) +def native_(s, encoding='latin-1', errors='strict'): + ''' + If ``s`` is an instance of ``text_type``, return + ``s``, otherwise return ``str(s, encoding, errors)`` + ''' + if PY3: + out = s if isinstance(s, text_type) else str(s, encoding, errors) + else: + out = s.encode(encoding, errors) if isinstance(s, text_type) else str(s) + + return out native_.__doc__ = ''' Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise From 693e4d4fd2a223f758ce680a39bb6319c22266cf Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 18 Sep 2018 23:58:24 +0200 Subject: [PATCH 29/50] Move docstrings to their native places --- salt/_compat.py | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index 5b208e63f3..5fc593aec1 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -75,10 +75,11 @@ def bytes_(s, encoding='latin-1', errors='strict'): def ascii_native_(s): ''' - Python 2/3 handler. + Python 3: If ``s`` is an instance of ``text_type``, return + ``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')`` - :param s: - :return: + Python 2: If ``s`` is an instance of ``text_type``, return + ``s.encode('ascii')``, otherwise return ``str(s)`` ''' if isinstance(s, text_type): s = s.encode('ascii') @@ -86,19 +87,13 @@ def ascii_native_(s): return str(s, 'ascii', 'strict') if PY3 else s -ascii_native_.__doc__ = ''' -Python 3: If ``s`` is an instance of ``text_type``, return -``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')`` - -Python 2: If ``s`` is an instance of ``text_type``, return -``s.encode('ascii')``, otherwise return ``str(s)`` -''' - - def native_(s, encoding='latin-1', errors='strict'): ''' - If ``s`` is an instance of ``text_type``, return - ``s``, otherwise return ``str(s, encoding, errors)`` + Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise + return ``str(s, encoding, errors)`` + + Python 2: If ``s`` is an instance of ``text_type``, return + ``s.encode(encoding, errors)``, otherwise return ``str(s)`` ''' if PY3: out = s if isinstance(s, text_type) else str(s, encoding, errors) @@ -107,14 +102,6 @@ def native_(s, encoding='latin-1', errors='strict'): return out -native_.__doc__ = ''' -Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise -return ``str(s, encoding, errors)`` - -Python 2: If ``s`` is an instance of ``text_type``, return -``s.encode(encoding, errors)``, otherwise return ``str(s)`` -''' - def string_io(data=None): # cStringIO can't handle unicode ''' From 8bb54380b1efb7fd910548516459e571ce32a548 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 19 Sep 2018 00:04:03 +0200 Subject: [PATCH 30/50] Add real exception message --- salt/_compat.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index 5fc593aec1..e283219c11 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -197,13 +197,15 @@ def ip_address(address): """ try: return ipaddress.IPv4Address(address) - except (ipaddress.AddressValueError, ipaddress.NetmaskValueError): - log.debug('Error while parsing IPv4 address: %s', address, exc_info=True) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: + log.debug('Error while parsing IPv4 address: %s', address) + log.debug(err) try: return IPv6AddressScoped(address) - except (ipaddress.AddressValueError, ipaddress.NetmaskValueError): - log.debug('Error while parsing IPv6 address: %s', address, exc_info=True) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: + log.debug('Error while parsing IPv6 address: %s', address) + log.debug(err) if isinstance(address, bytes): raise ipaddress.AddressValueError('{} does not appear to be an IPv4 or IPv6 address. ' From f5644bf617679ca7ca4161a11fb4715567443e83 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 19 Sep 2018 00:04:22 +0200 Subject: [PATCH 31/50] Add logging to the ip_interface --- salt/_compat.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/salt/_compat.py b/salt/_compat.py index e283219c11..dff576b473 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -215,6 +215,43 @@ def ip_address(address): raise ValueError('{} does not appear to be an IPv4 or IPv6 address'.format(repr(address))) +def ip_interface(address): + """Take an IP string/int and return an object of the correct type. + + Args: + address: A string or integer, the IP address. Either IPv4 or + IPv6 addresses may be supplied; integers less than 2**32 will + be considered to be IPv4 by default. + + Returns: + An IPv4Interface or IPv6Interface object. + + Raises: + ValueError: if the string passed isn't either a v4 or a v6 + address. + + Notes: + The IPv?Interface classes describe an Address on a particular + Network, so they're basically a combination of both the Address + and Network classes. + + """ + try: + return ipaddress.IPv4Interface(address) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: + log.debug('Error while getting IPv4 interface for address %s', address) + log.debug(err) + + try: + return ipaddress.IPv6Interface(address) + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: + log.debug('Error while getting IPv6 interface for address %s', address) + log.debug(err) + + raise ValueError('{} does not appear to be an IPv4 or IPv6 interface'.format(address)) + + if ipaddress: ipaddress.IPv6Address = IPv6AddressScoped ipaddress.ip_address = ip_address + ipaddress.ip_interface = ip_interface From 66ec29ca750036acadd98b0ac4885a7f13dac031 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 19 Sep 2018 00:32:40 +0200 Subject: [PATCH 32/50] Add scope on str --- salt/_compat.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/_compat.py b/salt/_compat.py index dff576b473..c483b67ae4 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -178,6 +178,10 @@ class IPv6AddressScoped(ipaddress.IPv6Address): ''' return self.__scope + def __str__(self): + return text_type(self._string_from_ip_int(self._ip) + + ('%' + self.scope if self.scope is not None else '')) + def ip_address(address): """Take an IP string/int and return an object of the correct type. From 379ead4c655dd10564d3ba92145e8d726a5ced67 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 19 Sep 2018 00:45:15 +0200 Subject: [PATCH 33/50] Lintfix: mute not called constructors --- salt/_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/_compat.py b/salt/_compat.py index c483b67ae4..1027796596 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -2,7 +2,7 @@ ''' Salt compatibility code ''' -# pylint: disable=import-error,unused-import,invalid-name +# pylint: disable=import-error,unused-import,invalid-name,W0231,W0233 # Import python libs from __future__ import absolute_import, unicode_literals, print_function From cefb16b5c2676e6c940031e1ce093061cfac705d Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 19 Sep 2018 15:15:42 +0200 Subject: [PATCH 34/50] Add extra detection for hexadecimal packed bytes on Python2. This cannot be detected with type comparison, because bytes == str and at the same time bytes != str if compatibility is not around --- salt/_compat.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/salt/_compat.py b/salt/_compat.py index 1027796596..cad7f92514 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -160,7 +160,8 @@ class IPv6AddressScoped(ipaddress.IPv6Address): if isinstance(address, integer_types): self._check_int_address(address) self._ip = address - elif isinstance(address, bytes): + elif (sys.version_info.major == 3 and isinstance(address, bytes) # Python-3 compatibility messes up + or sys.version_info.major == 2 and self._is_packed_binary(address)): # bytes and str types for Python 2. self._check_packed_address(address, 16) self._ip = ipaddress._int_from_bytes(address, 'big') else: @@ -169,6 +170,22 @@ class IPv6AddressScoped(ipaddress.IPv6Address): raise ipaddress.AddressValueError("Unexpected '/' in {}".format(address)) self._ip = self._ip_int_from_string(address) + def _is_packed_binary(self, data): + ''' + Check if data is hexadecimal packed + + :param data: + :return: + ''' + packed = False + if len(data) == 16 and ':' not in data: + try: + packed = bool(int(str(bytearray(data)).encode('hex'), 16)) + except ValueError: + pass + + return packed + @property def scope(self): ''' From 7f2208b2d04263c7db3cacb5f040eab70cb94690 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 19 Sep 2018 15:16:41 +0200 Subject: [PATCH 35/50] Fix py2 case where the same class cannot initialise itself on Python2 via super. --- salt/_compat.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/salt/_compat.py b/salt/_compat.py index cad7f92514..06eaf4f8b8 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -200,6 +200,25 @@ class IPv6AddressScoped(ipaddress.IPv6Address): ('%' + self.scope if self.scope is not None else '')) +class IPv6InterfaceScoped(ipaddress.IPv6Interface): + ''' + Update + ''' + def __init__(self, address): + if isinstance(address, (bytes, int)): + IPv6AddressScoped.__init__(self, address) + self.network = ipaddress.IPv6Network(self._ip) + self._prefixlen = self._max_prefixlen + return + + addr = ipaddress._split_optional_netmask(address) + IPv6AddressScoped.__init__(self, addr[0]) + self.network = ipaddress.IPv6Network(address, strict=False) + self.netmask = self.network.netmask + self._prefixlen = self.network._prefixlen + self.hostmask = self.network.hostmask + + def ip_address(address): """Take an IP string/int and return an object of the correct type. @@ -274,5 +293,7 @@ def ip_interface(address): if ipaddress: ipaddress.IPv6Address = IPv6AddressScoped + if sys.version_info.major == 2: + ipaddress.IPv6Interface = IPv6AddressScoped ipaddress.ip_address = ip_address ipaddress.ip_interface = ip_interface From 12038eaacfb5f5ac9eb83f0796130b7c7ac36822 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 19 Sep 2018 15:26:16 +0200 Subject: [PATCH 36/50] Simplify checking clause --- salt/_compat.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index 06eaf4f8b8..b8e762e430 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -144,6 +144,10 @@ class IPv6AddressScoped(ipaddress.IPv6Address): :param address: ''' + if not hasattr(self, '_is_packed_binary'): + # This method (below) won't be around for Python 3 and we need check this differently anyway + self._is_packed_binary = lambda p: isinstance(p, bytes) + if isinstance(address, string_types) and '%' in address: buff = address.split('%') if len(buff) != 2: @@ -160,8 +164,7 @@ class IPv6AddressScoped(ipaddress.IPv6Address): if isinstance(address, integer_types): self._check_int_address(address) self._ip = address - elif (sys.version_info.major == 3 and isinstance(address, bytes) # Python-3 compatibility messes up - or sys.version_info.major == 2 and self._is_packed_binary(address)): # bytes and str types for Python 2. + elif self._is_packed_binary(address): self._check_packed_address(address, 16) self._ip = ipaddress._int_from_bytes(address, 'big') else: From 54233cae128dcb218ef6c679ac0053ae1ebcbdca Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 19 Sep 2018 15:30:20 +0200 Subject: [PATCH 37/50] Do not use introspection for method swap --- salt/_compat.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index b8e762e430..e6f3721b7f 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -144,10 +144,6 @@ class IPv6AddressScoped(ipaddress.IPv6Address): :param address: ''' - if not hasattr(self, '_is_packed_binary'): - # This method (below) won't be around for Python 3 and we need check this differently anyway - self._is_packed_binary = lambda p: isinstance(p, bytes) - if isinstance(address, string_types) and '%' in address: buff = address.split('%') if len(buff) != 2: @@ -159,6 +155,10 @@ class IPv6AddressScoped(ipaddress.IPv6Address): if sys.version_info.major == 2: ipaddress._BaseAddress.__init__(self, address) ipaddress._BaseV6.__init__(self, address) + else: + # This method (below) won't be around for Python 3 + # and this type is needed to be checked differently + self._is_packed_binary = lambda p: isinstance(p, bytes) # Efficient constructor from integer. if isinstance(address, integer_types): From 1c02a312c5273ef50a8c0f845584dfca58e36924 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 19 Sep 2018 15:43:18 +0200 Subject: [PATCH 38/50] Fix wrong type swap --- salt/_compat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index e6f3721b7f..fc87202a63 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -203,7 +203,7 @@ class IPv6AddressScoped(ipaddress.IPv6Address): ('%' + self.scope if self.scope is not None else '')) -class IPv6InterfaceScoped(ipaddress.IPv6Interface): +class IPv6InterfaceScoped(ipaddress.IPv6Interface, IPv6AddressScoped): ''' Update ''' @@ -297,6 +297,6 @@ def ip_interface(address): if ipaddress: ipaddress.IPv6Address = IPv6AddressScoped if sys.version_info.major == 2: - ipaddress.IPv6Interface = IPv6AddressScoped + ipaddress.IPv6Interface = IPv6InterfaceScoped ipaddress.ip_address = ip_address ipaddress.ip_interface = ip_interface From f52ea0be7c1f414dd51c398a61d3e2b485c55355 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 19 Sep 2018 16:54:47 +0200 Subject: [PATCH 39/50] Add Py3.4 old implementation's fix --- salt/_compat.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/_compat.py b/salt/_compat.py index fc87202a63..c15cb84d80 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -160,6 +160,11 @@ class IPv6AddressScoped(ipaddress.IPv6Address): # and this type is needed to be checked differently self._is_packed_binary = lambda p: isinstance(p, bytes) + # Python 3.4 fix. Versions higher are simply not affected + # https://github.com/python/cpython/blob/3.4/Lib/ipaddress.py#L543-L544 + self._version = 6 + self._max_prefixlen = ipaddress.IPV6LENGTH + # Efficient constructor from integer. if isinstance(address, integer_types): self._check_int_address(address) From 5932a0d152e74efb571c03d42353e62662fa98c6 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 21 Sep 2018 11:30:46 +0200 Subject: [PATCH 40/50] Lintfix --- salt/_compat.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index c15cb84d80..65b4f3dcf3 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -156,10 +156,6 @@ class IPv6AddressScoped(ipaddress.IPv6Address): ipaddress._BaseAddress.__init__(self, address) ipaddress._BaseV6.__init__(self, address) else: - # This method (below) won't be around for Python 3 - # and this type is needed to be checked differently - self._is_packed_binary = lambda p: isinstance(p, bytes) - # Python 3.4 fix. Versions higher are simply not affected # https://github.com/python/cpython/blob/3.4/Lib/ipaddress.py#L543-L544 self._version = 6 @@ -185,6 +181,9 @@ class IPv6AddressScoped(ipaddress.IPv6Address): :param data: :return: ''' + if sys.version_info.major > 2: + return isinstance(data, bytes) + packed = False if len(data) == 16 and ':' not in data: try: From 0a86112f24e1d9908f5767858dba76725458ea5d Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 21 Sep 2018 11:34:41 +0200 Subject: [PATCH 41/50] Lintfix refactor: remove duplicate returns as not needed --- salt/_compat.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index 65b4f3dcf3..38806fd6e2 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -181,15 +181,15 @@ class IPv6AddressScoped(ipaddress.IPv6Address): :param data: :return: ''' - if sys.version_info.major > 2: - return isinstance(data, bytes) - packed = False - if len(data) == 16 and ':' not in data: - try: - packed = bool(int(str(bytearray(data)).encode('hex'), 16)) - except ValueError: - pass + if sys.version_info.major > 2: + packed = isinstance(data, bytes) + else: + if len(data) == 16 and ':' not in data: + try: + packed = bool(int(str(bytearray(data)).encode('hex'), 16)) + except ValueError: + pass return packed From c314e1c675e782391883b7e72c454b162c78585e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 24 Sep 2018 11:46:38 +0200 Subject: [PATCH 42/50] Revert method remapping with pylint updates --- salt/_compat.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/salt/_compat.py b/salt/_compat.py index 38806fd6e2..0576210afc 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -144,6 +144,13 @@ class IPv6AddressScoped(ipaddress.IPv6Address): :param address: ''' + # pylint: disable-all + if not hasattr(self, '_is_packed_binary'): + # This method (below) won't be around for some Python 3 versions + # and we need check this differently anyway + self._is_packed_binary = lambda p: isinstance(p, bytes) + # pylint: enable-all + if isinstance(address, string_types) and '%' in address: buff = address.split('%') if len(buff) != 2: @@ -182,14 +189,11 @@ class IPv6AddressScoped(ipaddress.IPv6Address): :return: ''' packed = False - if sys.version_info.major > 2: - packed = isinstance(data, bytes) - else: - if len(data) == 16 and ':' not in data: - try: - packed = bool(int(str(bytearray(data)).encode('hex'), 16)) - except ValueError: - pass + if len(data) == 16 and ':' not in data: + try: + packed = bool(int(str(bytearray(data)).encode('hex'), 16)) + except ValueError: + pass return packed From 3ce265e7cc02511c157d6fc38b928ad6d898d46c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 19 Sep 2018 00:32:22 +0200 Subject: [PATCH 43/50] Remove unnecessary manipulation with IPv6 scope outside of the IPv6Address object instance --- salt/utils/dns.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/salt/utils/dns.py b/salt/utils/dns.py index db08bcb7ac..40011016fd 100644 --- a/salt/utils/dns.py +++ b/salt/utils/dns.py @@ -1029,18 +1029,13 @@ def parse_resolv(src='/etc/resolv.conf'): try: (directive, arg) = (line[0].lower(), line[1:]) # Drop everything after # or ; (comments) - arg = list(itertools.takewhile( - lambda x: x[0] not in ('#', ';'), arg)) - + arg = list(itertools.takewhile(lambda x: x[0] not in ('#', ';'), arg)) if directive == 'nameserver': - # Split the scope (interface) if it is present - addr, scope = arg[0].split('%', 1) if '%' in arg[0] else (arg[0], '') + addr = arg[0] try: ip_addr = ipaddress.ip_address(addr) version = ip_addr.version - # Rejoin scope after address validation - if scope: - ip_addr = '%'.join((str(ip_addr), scope)) + ip_addr = str(ip_addr) if ip_addr not in nameservers: nameservers.append(ip_addr) if version == 4 and ip_addr not in ip4_nameservers: From 615b09aea1811815e40a3afbc09870c43f4e9f3d Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 19 Sep 2018 00:43:28 +0200 Subject: [PATCH 44/50] Lintfix: W0611 --- tests/unit/modules/test_network.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/modules/test_network.py b/tests/unit/modules/test_network.py index 4cc51b0a28..bcc1022693 100644 --- a/tests/unit/modules/test_network.py +++ b/tests/unit/modules/test_network.py @@ -20,7 +20,6 @@ from tests.support.mock import ( ) # Import Salt Libs -from salt.ext import six import salt.utils.network import salt.utils.path import salt.modules.network as network From facb34a6cc2ec8ac2cf5696d1354f668e482392f Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Wed, 19 Sep 2018 15:05:13 +0200 Subject: [PATCH 45/50] Add additional check --- tests/unit/utils/test_jinja.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/utils/test_jinja.py b/tests/unit/utils/test_jinja.py index 4686739a1d..a6f252dcb2 100644 --- a/tests/unit/utils/test_jinja.py +++ b/tests/unit/utils/test_jinja.py @@ -978,6 +978,10 @@ class TestCustomExtensions(TestCase): ''' Test the `ipaddr` Jinja filter. ''' + rendered = render_jinja_tmpl("{{ '::' | ipaddr }}", + dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)) + self.assertEqual(rendered, '::') + rendered = render_jinja_tmpl("{{ '192.168.0.1' | ipaddr }}", dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)) self.assertEqual(rendered, '192.168.0.1') From bb37ba05785b535e2eddab14138f388488b0d0f3 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 28 Sep 2018 15:49:05 +0200 Subject: [PATCH 46/50] Add IPv6 scoped test --- tests/unit/utils/test_jinja.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/utils/test_jinja.py b/tests/unit/utils/test_jinja.py index a6f252dcb2..4c4c83bebe 100644 --- a/tests/unit/utils/test_jinja.py +++ b/tests/unit/utils/test_jinja.py @@ -966,6 +966,10 @@ class TestCustomExtensions(TestCase): dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)) self.assertEqual(rendered, 'False') + rendered = render_jinja_tmpl("{{ 'fe80::20d:b9ff:fe01:ea8%eth0' | is_ipv6 }}", + dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)) + self.assertEqual(rendered, 'True') + rendered = render_jinja_tmpl("{{ 'FE80::' | is_ipv6 }}", dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)) self.assertEqual(rendered, 'True') From c14f26f351ad19451e4c6c2504c8f4284b154a58 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 28 Sep 2018 16:31:22 +0200 Subject: [PATCH 47/50] Reverse skipping tests: if no ipaddress --- tests/unit/modules/test_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/modules/test_network.py b/tests/unit/modules/test_network.py index bcc1022693..ea3026299d 100644 --- a/tests/unit/modules/test_network.py +++ b/tests/unit/modules/test_network.py @@ -267,7 +267,7 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): self.assertDictEqual(network.connect('host', 'port'), {'comment': ret, 'result': True}) - @skipIf(bool(ipaddress), 'unable to import \'ipaddress\'') + @skipIf(not bool(ipaddress), 'unable to import \'ipaddress\'') def test_is_private(self): ''' Test for Check if the given IP address is a private address @@ -279,7 +279,7 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): return_value=True): self.assertTrue(network.is_private('::1')) - @skipIf(bool(ipaddress), 'unable to import \'ipaddress\'') + @skipIf(not bool(ipaddress), 'unable to import \'ipaddress\'') def test_is_loopback(self): ''' Test for Check if the given IP address is a loopback address From 855ffe60fc38c160428230f1f25cd1e4faf584f0 Mon Sep 17 00:00:00 2001 From: Brett Benassi Date: Fri, 28 Sep 2018 12:10:55 -0600 Subject: [PATCH 48/50] Adding timeout to all pipelines so that the build aborts --- .ci/docs | 1 + .ci/kitchen-centos7-py2 | 18 ++++++++++-------- .ci/kitchen-centos7-py3 | 16 +++++++++------- .ci/kitchen-ubuntu1604-py2 | 12 +++++++----- .ci/kitchen-ubuntu1604-py3 | 14 ++++++++------ .ci/kitchen-windows2016-py2 | 12 +++++++----- .ci/kitchen-windows2016-py3 | 14 ++++++++------ .ci/lint | 1 + 8 files changed, 51 insertions(+), 37 deletions(-) diff --git a/.ci/docs b/.ci/docs index 24f6f469e3..8409d9df04 100644 --- a/.ci/docs +++ b/.ci/docs @@ -3,6 +3,7 @@ pipeline { options { timestamps() ansiColor('xterm') + timeout(time: 6, unit: 'HOURS') } environment { PYENV_ROOT = "/usr/local/pyenv" diff --git a/.ci/kitchen-centos7-py2 b/.ci/kitchen-centos7-py2 index c04ad78aed..555423a349 100644 --- a/.ci/kitchen-centos7-py2 +++ b/.ci/kitchen-centos7-py2 @@ -1,4 +1,5 @@ -node('kitchen-slave') { +timeout(time: 6, unit: 'HOURS') { + node('kitchen-slave') { timestamps { ansiColor('xterm') { withEnv([ @@ -66,10 +67,11 @@ node('kitchen-slave') { description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", status: 'FAILURE', context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } - } - } - } - } -} + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/.ci/kitchen-centos7-py3 b/.ci/kitchen-centos7-py3 index 25fd8b8f54..6c561843da 100644 --- a/.ci/kitchen-centos7-py3 +++ b/.ci/kitchen-centos7-py3 @@ -1,4 +1,5 @@ -node('kitchen-slave') { +timeout(time: 6, unit: 'HOURS') { + node('kitchen-slave') { timestamps { ansiColor('xterm') { withEnv([ @@ -67,9 +68,10 @@ node('kitchen-slave') { status: 'FAILURE', context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" } - } - } - } - } - } -} + } + } + } + } + } + } +} \ No newline at end of file diff --git a/.ci/kitchen-ubuntu1604-py2 b/.ci/kitchen-ubuntu1604-py2 index 94ca29cc36..ebbeb91858 100644 --- a/.ci/kitchen-ubuntu1604-py2 +++ b/.ci/kitchen-ubuntu1604-py2 @@ -1,4 +1,5 @@ -node('kitchen-slave') { +timeout(time: 6, unit: 'HOURS') { + node('kitchen-slave') { timestamps { ansiColor('xterm') { withEnv([ @@ -69,7 +70,8 @@ node('kitchen-slave') { } } } - } - } - } -} + } + } + } + } +} \ No newline at end of file diff --git a/.ci/kitchen-ubuntu1604-py3 b/.ci/kitchen-ubuntu1604-py3 index 6a7d167162..ff86c7bc4b 100644 --- a/.ci/kitchen-ubuntu1604-py3 +++ b/.ci/kitchen-ubuntu1604-py3 @@ -1,4 +1,5 @@ -node('kitchen-slave') { +timeout(time: 6, unit: 'HOURS') { + node('kitchen-slave') { timestamps { ansiColor('xterm') { withEnv([ @@ -67,9 +68,10 @@ node('kitchen-slave') { status: 'FAILURE', context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" } - } - } - } - } - } + } + } + } + } + } + } } diff --git a/.ci/kitchen-windows2016-py2 b/.ci/kitchen-windows2016-py2 index 9ef81550d9..40dd989d82 100644 --- a/.ci/kitchen-windows2016-py2 +++ b/.ci/kitchen-windows2016-py2 @@ -1,4 +1,5 @@ -node('kitchen-slave') { +timeout(time: 6, unit: 'HOURS') { + node('kitchen-slave') { timestamps { ansiColor('xterm') { withEnv([ @@ -68,8 +69,9 @@ node('kitchen-slave') { context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" } } - } - } - } - } + } + } + } + } + } } diff --git a/.ci/kitchen-windows2016-py3 b/.ci/kitchen-windows2016-py3 index fcd90f7086..7a05f22f6e 100644 --- a/.ci/kitchen-windows2016-py3 +++ b/.ci/kitchen-windows2016-py3 @@ -1,4 +1,5 @@ -node('kitchen-slave') { +timeout(time: 6, unit: 'HOURS') { + node('kitchen-slave') { timestamps { ansiColor('xterm') { withEnv([ @@ -68,8 +69,9 @@ node('kitchen-slave') { context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" } } - } - } - } - } -} + } + } + } + } + } +} \ No newline at end of file diff --git a/.ci/lint b/.ci/lint index 885174b0bf..3a2414b5c2 100644 --- a/.ci/lint +++ b/.ci/lint @@ -3,6 +3,7 @@ pipeline { options { timestamps() ansiColor('xterm') + timeout(time: 6, unit: 'HOURS') } environment { PYENV_ROOT = "/usr/local/pyenv" From f6d967931c504303c99ac1d9d80d21b304478b77 Mon Sep 17 00:00:00 2001 From: Brett Benassi Date: Fri, 28 Sep 2018 13:31:58 -0600 Subject: [PATCH 49/50] Updating syntax to be better and with new lines --- .ci/docs | 6 +- .ci/kitchen-centos7-py2 | 140 ++++++++++++++++++------------------ .ci/kitchen-centos7-py3 | 140 ++++++++++++++++++------------------ .ci/kitchen-ubuntu1604-py2 | 140 ++++++++++++++++++------------------ .ci/kitchen-ubuntu1604-py3 | 138 +++++++++++++++++------------------ .ci/kitchen-windows2016-py2 | 138 +++++++++++++++++------------------ .ci/kitchen-windows2016-py3 | 140 ++++++++++++++++++------------------ 7 files changed, 428 insertions(+), 414 deletions(-) diff --git a/.ci/docs b/.ci/docs index 8409d9df04..a8ce2aada0 100644 --- a/.ci/docs +++ b/.ci/docs @@ -1,9 +1,11 @@ pipeline { - agent { label 'docs' } + agent { + label 'docs' + } options { timestamps() ansiColor('xterm') - timeout(time: 6, unit: 'HOURS') + timeout(time: 2, unit: 'HOURS') } environment { PYENV_ROOT = "/usr/local/pyenv" diff --git a/.ci/kitchen-centos7-py2 b/.ci/kitchen-centos7-py2 index 555423a349..305de52c04 100644 --- a/.ci/kitchen-centos7-py2 +++ b/.ci/kitchen-centos7-py2 @@ -1,77 +1,79 @@ timeout(time: 6, unit: 'HOURS') { - node('kitchen-slave') { - timestamps { - ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py2', - 'TEST_PLATFORM=centos-7', - 'PY_COLORS=1',]) - { - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - stage('setup-bundle') { - sh 'bundle install --with ec2 windows --without opennebula docker' + node('kitchen-slave') { + timestamps { + ansiColor('xterm') { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + 'TEST_SUITE=py2', + 'TEST_PLATFORM=centos-7', + 'PY_COLORS=1', + ]) { + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm } try { - stage('run kitchen') { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + stage('setup-bundle') { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + try { + stage('run kitchen') { + withCredentials([ + [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] + ]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } } } + } finally { + stage('cleanup kitchen') { + script { + withCredentials([ + [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] + ]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + } + } + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } finally { + try { + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + cleanWs notFailBuild: true + if (currentBuild.result == 'SUCCESS') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } else { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } } - finally { - stage('cleanup kitchen') { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - }} - archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' - archiveArtifacts artifacts: 'artifacts/logs/minion' - archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' - } - } } - finally { - try { - junit 'artifacts/xml-unittests-output/*.xml' - } - finally { - cleanWs notFailBuild: true - if (currentBuild.result == 'SUCCESS') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } - } - } - } - } - } -} \ No newline at end of file + } + } + } +} diff --git a/.ci/kitchen-centos7-py3 b/.ci/kitchen-centos7-py3 index 6c561843da..0a2d392004 100644 --- a/.ci/kitchen-centos7-py3 +++ b/.ci/kitchen-centos7-py3 @@ -1,77 +1,79 @@ timeout(time: 6, unit: 'HOURS') { - node('kitchen-slave') { - timestamps { - ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py3', - 'TEST_PLATFORM=centos-7', - 'PY_COLORS=1',]) - { - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - stage('setup-bundle') { - sh 'bundle install --with ec2 windows --without opennebula docker' + node('kitchen-slave') { + timestamps { + ansiColor('xterm') { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + 'TEST_SUITE=py3', + 'TEST_PLATFORM=centos-7', + 'PY_COLORS=1', + ]) { + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm } try { - stage('run kitchen') { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + stage('setup-bundle') { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + try { + stage('run kitchen') { + withCredentials([ + [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] + ]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } } } + } finally { + stage('cleanup kitchen') { + script { + withCredentials([ + [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] + ]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + } + } + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } finally { + try { + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + cleanWs notFailBuild: true + if (currentBuild.result == 'SUCCESS') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } else { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } } - finally { - stage('cleanup kitchen') { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - }} - archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' - archiveArtifacts artifacts: 'artifacts/logs/minion' - archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' - } - } } - finally { - try { - junit 'artifacts/xml-unittests-output/*.xml' - } - finally { - cleanWs notFailBuild: true - if (currentBuild.result == 'SUCCESS') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } - } - } - } - } - } -} \ No newline at end of file + } + } + } +} diff --git a/.ci/kitchen-ubuntu1604-py2 b/.ci/kitchen-ubuntu1604-py2 index ebbeb91858..005209f111 100644 --- a/.ci/kitchen-ubuntu1604-py2 +++ b/.ci/kitchen-ubuntu1604-py2 @@ -1,77 +1,79 @@ timeout(time: 6, unit: 'HOURS') { - node('kitchen-slave') { - timestamps { - ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py2', - 'TEST_PLATFORM=ubuntu-1604', - 'PY_COLORS=1',]) - { - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - stage('setup-bundle') { - sh 'bundle install --with ec2 windows --without opennebula docker' + node('kitchen-slave') { + timestamps { + ansiColor('xterm') { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + 'TEST_SUITE=py2', + 'TEST_PLATFORM=ubuntu-1604', + 'PY_COLORS=1', + ]) { + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm } try { - stage('run kitchen') { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + stage('setup-bundle') { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + try { + stage('run kitchen') { + withCredentials([ + [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] + ]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } } } + } finally { + stage('cleanup kitchen') { + script { + withCredentials([ + [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] + ]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + } + } + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } finally { + try { + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + cleanWs notFailBuild: true + if (currentBuild.result == 'SUCCESS') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } else { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } } - finally { - stage('cleanup kitchen') { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - }} - archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' - archiveArtifacts artifacts: 'artifacts/logs/minion' - archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' - } - } } - finally { - try { - junit 'artifacts/xml-unittests-output/*.xml' - } - finally { - cleanWs notFailBuild: true - if (currentBuild.result == 'SUCCESS') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } - } - } - } - } - } -} \ No newline at end of file + } + } + } +} diff --git a/.ci/kitchen-ubuntu1604-py3 b/.ci/kitchen-ubuntu1604-py3 index ff86c7bc4b..5b51c035be 100644 --- a/.ci/kitchen-ubuntu1604-py3 +++ b/.ci/kitchen-ubuntu1604-py3 @@ -1,77 +1,79 @@ timeout(time: 6, unit: 'HOURS') { - node('kitchen-slave') { - timestamps { - ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py3', - 'TEST_PLATFORM=ubuntu-1604', - 'PY_COLORS=1',]) - { - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - stage('setup-bundle') { - sh 'bundle install --with ec2 windows --without opennebula docker' + node('kitchen-slave') { + timestamps { + ansiColor('xterm') { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + 'TEST_SUITE=py3', + 'TEST_PLATFORM=ubuntu-1604', + 'PY_COLORS=1', + ]) { + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm } try { - stage('run kitchen') { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + stage('setup-bundle') { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + try { + stage('run kitchen') { + withCredentials([ + [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] + ]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } } } + } finally { + stage('cleanup kitchen') { + script { + withCredentials([ + [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] + ]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + } + } + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } finally { + try { + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + cleanWs notFailBuild: true + if (currentBuild.result == 'SUCCESS') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } else { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } } - finally { - stage('cleanup kitchen') { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - }} - archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' - archiveArtifacts artifacts: 'artifacts/logs/minion' - archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' - } - } } - finally { - try { - junit 'artifacts/xml-unittests-output/*.xml' - } - finally { - cleanWs notFailBuild: true - if (currentBuild.result == 'SUCCESS') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } - } - } - } - } - } + } + } + } } diff --git a/.ci/kitchen-windows2016-py2 b/.ci/kitchen-windows2016-py2 index 40dd989d82..cd24731a27 100644 --- a/.ci/kitchen-windows2016-py2 +++ b/.ci/kitchen-windows2016-py2 @@ -1,77 +1,79 @@ timeout(time: 6, unit: 'HOURS') { - node('kitchen-slave') { - timestamps { - ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py2', - 'TEST_PLATFORM=windows-2016', - 'PY_COLORS=1',]) - { - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - stage('setup-bundle') { - sh 'bundle install --with ec2 windows --without opennebula docker' + node('kitchen-slave') { + timestamps { + ansiColor('xterm') { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + 'TEST_SUITE=py2', + 'TEST_PLATFORM=windows-2016', + 'PY_COLORS=1', + ]) { + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm } try { - stage('run kitchen') { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + stage('setup-bundle') { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + try { + stage('run kitchen') { + withCredentials([ + [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] + ]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } } } + } finally { + stage('cleanup kitchen') { + script { + withCredentials([ + [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] + ]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + } + } + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } finally { + try { + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + cleanWs notFailBuild: true + if (currentBuild.result == 'SUCCESS') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } else { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } } - finally { - stage('cleanup kitchen') { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - }} - archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' - archiveArtifacts artifacts: 'artifacts/logs/minion' - archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' - } - } } - finally { - try { - junit 'artifacts/xml-unittests-output/*.xml' - } - finally { - cleanWs notFailBuild: true - if (currentBuild.result == 'SUCCESS') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } - } - } - } - } - } + } + } + } } diff --git a/.ci/kitchen-windows2016-py3 b/.ci/kitchen-windows2016-py3 index 7a05f22f6e..15a867bff0 100644 --- a/.ci/kitchen-windows2016-py3 +++ b/.ci/kitchen-windows2016-py3 @@ -1,77 +1,79 @@ timeout(time: 6, unit: 'HOURS') { - node('kitchen-slave') { - timestamps { - ansiColor('xterm') { - withEnv([ - 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', - 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', - 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', - 'RBENV_VERSION=2.4.2', - 'TEST_SUITE=py3', - 'TEST_PLATFORM=windows-2016', - 'PY_COLORS=1',]) - { - stage('checkout-scm') { - cleanWs notFailBuild: true - checkout scm - } - try { - stage('github-pending') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", - status: 'PENDING', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - stage('setup-bundle') { - sh 'bundle install --with ec2 windows --without opennebula docker' + node('kitchen-slave') { + timestamps { + ansiColor('xterm') { + withEnv([ + 'SALT_KITCHEN_PLATFORMS=/var/jenkins/workspace/platforms.yml', + 'SALT_KITCHEN_DRIVER=/var/jenkins/workspace/driver.yml', + 'PATH=/usr/local/rbenv/shims/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin', + 'RBENV_VERSION=2.4.2', + 'TEST_SUITE=py3', + 'TEST_PLATFORM=windows-2016', + 'PY_COLORS=1', + ]) { + stage('checkout-scm') { + cleanWs notFailBuild: true + checkout scm } try { - stage('run kitchen') { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' - sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + stage('github-pending') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "running ${TEST_SUITE}-${TEST_PLATFORM}...", + status: 'PENDING', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } + stage('setup-bundle') { + sh 'bundle install --with ec2 windows --without opennebula docker' + } + try { + stage('run kitchen') { + withCredentials([ + [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] + ]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM || bundle exec kitchen converge $TEST_SUITE-$TEST_PLATFORM' + sh 'bundle exec kitchen verify $TEST_SUITE-$TEST_PLATFORM' + } } } + } finally { + stage('cleanup kitchen') { + script { + withCredentials([ + [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'] + ]) { + sshagent(credentials: ['jenkins-testing-ssh-key']) { + sh 'ssh-add ~/.ssh/jenkins-testing.pem' + sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' + } + } + } + archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' + archiveArtifacts artifacts: 'artifacts/logs/minion' + archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' + } + } + } finally { + try { + junit 'artifacts/xml-unittests-output/*.xml' + } finally { + cleanWs notFailBuild: true + if (currentBuild.result == 'SUCCESS') { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", + status: 'SUCCESS', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } else { + githubNotify credentialsId: 'test-jenkins-credentials', + description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", + status: 'FAILURE', + context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" + } } } - finally { - stage('cleanup kitchen') { - script { withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sshagent(credentials: ['jenkins-testing-ssh-key']) { - sh 'ssh-add ~/.ssh/jenkins-testing.pem' - sh 'bundle exec kitchen destroy $TEST_SUITE-$TEST_PLATFORM' - } - }} - archiveArtifacts artifacts: 'artifacts/xml-unittests-output/*.xml' - archiveArtifacts artifacts: 'artifacts/logs/minion' - archiveArtifacts artifacts: 'artifacts/logs/salt-runtests.log' - } - } } - finally { - try { - junit 'artifacts/xml-unittests-output/*.xml' - } - finally { - cleanWs notFailBuild: true - if (currentBuild.result == 'SUCCESS') { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", - status: 'SUCCESS', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - else { - githubNotify credentialsId: 'test-jenkins-credentials', - description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has failed", - status: 'FAILURE', - context: "jenkins/pr/${TEST_SUITE}-${TEST_PLATFORM}" - } - } - } - } - } - } - } -} \ No newline at end of file + } + } + } +} From b25ac9f773992c661a29643fe23b8c9ae2cf0b8d Mon Sep 17 00:00:00 2001 From: Brett Benassi Date: Fri, 28 Sep 2018 15:42:31 -0600 Subject: [PATCH 50/50] Adding some changes for the status --- .ci/kitchen-centos7-py2 | 3 ++- .ci/kitchen-centos7-py3 | 3 ++- .ci/kitchen-ubuntu1604-py2 | 3 ++- .ci/kitchen-ubuntu1604-py3 | 3 ++- .ci/kitchen-windows2016-py2 | 3 ++- .ci/kitchen-windows2016-py3 | 3 ++- .ci/lint | 2 +- 7 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.ci/kitchen-centos7-py2 b/.ci/kitchen-centos7-py2 index 305de52c04..44fbf8b87f 100644 --- a/.ci/kitchen-centos7-py2 +++ b/.ci/kitchen-centos7-py2 @@ -59,7 +59,8 @@ timeout(time: 6, unit: 'HOURS') { junit 'artifacts/xml-unittests-output/*.xml' } finally { cleanWs notFailBuild: true - if (currentBuild.result == 'SUCCESS') { + def currentResult = currentBuild.result ?: 'SUCCESS' + if (currentResult == 'SUCCESS') { githubNotify credentialsId: 'test-jenkins-credentials', description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", status: 'SUCCESS', diff --git a/.ci/kitchen-centos7-py3 b/.ci/kitchen-centos7-py3 index 0a2d392004..de5e344a97 100644 --- a/.ci/kitchen-centos7-py3 +++ b/.ci/kitchen-centos7-py3 @@ -59,7 +59,8 @@ timeout(time: 6, unit: 'HOURS') { junit 'artifacts/xml-unittests-output/*.xml' } finally { cleanWs notFailBuild: true - if (currentBuild.result == 'SUCCESS') { + def currentResult = currentBuild.result ?: 'SUCCESS' + if (currentResult == 'SUCCESS') { githubNotify credentialsId: 'test-jenkins-credentials', description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", status: 'SUCCESS', diff --git a/.ci/kitchen-ubuntu1604-py2 b/.ci/kitchen-ubuntu1604-py2 index 005209f111..3ba95ebf5d 100644 --- a/.ci/kitchen-ubuntu1604-py2 +++ b/.ci/kitchen-ubuntu1604-py2 @@ -59,7 +59,8 @@ timeout(time: 6, unit: 'HOURS') { junit 'artifacts/xml-unittests-output/*.xml' } finally { cleanWs notFailBuild: true - if (currentBuild.result == 'SUCCESS') { + def currentResult = currentBuild.result ?: 'SUCCESS' + if ( currentResult == 'SUCCESS') { githubNotify credentialsId: 'test-jenkins-credentials', description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", status: 'SUCCESS', diff --git a/.ci/kitchen-ubuntu1604-py3 b/.ci/kitchen-ubuntu1604-py3 index 5b51c035be..84c127de9b 100644 --- a/.ci/kitchen-ubuntu1604-py3 +++ b/.ci/kitchen-ubuntu1604-py3 @@ -59,7 +59,8 @@ timeout(time: 6, unit: 'HOURS') { junit 'artifacts/xml-unittests-output/*.xml' } finally { cleanWs notFailBuild: true - if (currentBuild.result == 'SUCCESS') { + def currentResult = currentBuild.result ?: 'SUCCESS' + if (currentResult == 'SUCCESS') { githubNotify credentialsId: 'test-jenkins-credentials', description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", status: 'SUCCESS', diff --git a/.ci/kitchen-windows2016-py2 b/.ci/kitchen-windows2016-py2 index cd24731a27..886fa4cbd9 100644 --- a/.ci/kitchen-windows2016-py2 +++ b/.ci/kitchen-windows2016-py2 @@ -59,7 +59,8 @@ timeout(time: 6, unit: 'HOURS') { junit 'artifacts/xml-unittests-output/*.xml' } finally { cleanWs notFailBuild: true - if (currentBuild.result == 'SUCCESS') { + def currentResult = currentBuild.result ?: 'SUCCESS' + if (currentResult == 'SUCCESS') { githubNotify credentialsId: 'test-jenkins-credentials', description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", status: 'SUCCESS', diff --git a/.ci/kitchen-windows2016-py3 b/.ci/kitchen-windows2016-py3 index 15a867bff0..a85b57b399 100644 --- a/.ci/kitchen-windows2016-py3 +++ b/.ci/kitchen-windows2016-py3 @@ -59,7 +59,8 @@ timeout(time: 6, unit: 'HOURS') { junit 'artifacts/xml-unittests-output/*.xml' } finally { cleanWs notFailBuild: true - if (currentBuild.result == 'SUCCESS') { + def currentResult = currentBuild.result ?: 'SUCCESS' + if (currentResult == 'SUCCESS') { githubNotify credentialsId: 'test-jenkins-credentials', description: "The ${TEST_SUITE}-${TEST_PLATFORM} job has passed", status: 'SUCCESS', diff --git a/.ci/lint b/.ci/lint index 3a2414b5c2..28cea02139 100644 --- a/.ci/lint +++ b/.ci/lint @@ -3,7 +3,7 @@ pipeline { options { timestamps() ansiColor('xterm') - timeout(time: 6, unit: 'HOURS') + timeout(time: 1, unit: 'HOURS') } environment { PYENV_ROOT = "/usr/local/pyenv"