Merge branch '2018.3' into fix_51566

pkg/windows/build_env_2.ps1

@@ -22,22 +22,10 @@
 
 # Load parameters
 param(
-    [switch]$Silent
+    [switch]$Silent,
+    [switch]$NoPipDependencies
 )
 
-Write-Output "================================================================="
-Write-Output ""
-Write-Output "               Development Environment Installation"
-Write-Output ""
-Write-Output "               - Installs All Salt Dependencies"
-Write-Output "               - Detects 32/64 bit Architectures"
-Write-Output ""
-Write-Output "               To run silently add -Silent"
-Write-Output "               eg: dev_env.ps1 -Silent"
-Write-Output ""
-Write-Output "================================================================="
-Write-Output ""
-
 #==============================================================================
 # Get the Directory of actual script
 #==============================================================================
@@ -49,6 +37,22 @@ $script_path = $script_path.DirectoryName
 #==============================================================================
 $script_name = $MyInvocation.MyCommand.Name
 
+Write-Output "================================================================="
+Write-Output ""
+Write-Output "               Development Environment Installation"
+Write-Output ""
+Write-Output "               - Installs All Salt Dependencies"
+Write-Output "               - Detects 32/64 bit Architectures"
+Write-Output ""
+Write-Output "               To run silently add -Silent"
+Write-Output "               eg: ${script_name} -Silent"
+Write-Output ""
+Write-Output "               To run skip installing pip dependencies add -NoPipDependencies"
+Write-Output "               eg: ${script_name} -NoPipDependencies"
+Write-Output ""
+Write-Output "================================================================="
+Write-Output ""
+
 #==============================================================================
 # Import Modules
 #==============================================================================
@@ -211,25 +215,46 @@ if ( ! [bool]$Env:SALT_PIP_LOCAL_CACHE) {
 }
 
 #==============================================================================
-# Install pypi resources using pip
+# Install windows specific pypi resources using pip
 #    caching depends on environment variable SALT_REQ_LOCAL_CACHE
 #==============================================================================
 Write-Output " ----------------------------------------------------------------"
-Write-Output " - $script_name :: Installing pypi resources using pip . . ."
+Write-Output " - $script_name :: Installing windows specific pypi resources using pip . . ."
 Write-Output " ----------------------------------------------------------------"
 if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) {
-    Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req.txt" "pip install"
+    Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req_win.txt" "pip install"
 } else {
     if ( (Get-ChildItem $Env:SALT_REQ_LOCAL_CACHE | Measure-Object).Count -eq 0 ) {
         # folder empty
-        Write-Output "    pip download from req.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE"
+        Write-Output "    pip download from req_win.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE"
-        Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip download"
+        Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_win.txt" "pip download"
     }
     Write-Output "    reading from local pip cache $Env:SALT_REQ_LOCAL_CACHE"
     Write-Output "    If a (new) resource is missing, please delete all files in this cache, go online and repeat"
-  Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip install"
+  Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_win.txt" "pip install"
 }
 
+#==============================================================================
+# Install pypi resources using pip
+#    caching depends on environment variable SALT_REQ_LOCAL_CACHE
+#==============================================================================
+If ($NoPipDependencies -eq $false) {
+  Write-Output " ----------------------------------------------------------------"
+  Write-Output " - $script_name :: Installing pypi resources using pip . . ."
+  Write-Output " ----------------------------------------------------------------"
+  if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) {
+      Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req.txt" "pip install"
+  } else {
+      if ( (Get-ChildItem $Env:SALT_REQ_LOCAL_CACHE | Measure-Object).Count -eq 0 ) {
+          # folder empty
+          Write-Output "    pip download from req.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE"
+          Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip download"
+      }
+      Write-Output "    reading from local pip cache $Env:SALT_REQ_LOCAL_CACHE"
+      Write-Output "    If a (new) resource is missing, please delete all files in this cache, go online and repeat"
+    Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python2Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip install"
+  }
+}
 #==============================================================================
 # Cleaning Up PyWin32
 #==============================================================================

pkg/windows/build_env_3.ps1

@@ -22,22 +22,10 @@
 
 # Load parameters
 param(
-    [switch]$Silent
+    [switch]$Silent,
+    [switch]$NoPipDependencies
 )
 
-Write-Output "================================================================="
-Write-Output ""
-Write-Output "               Development Environment Installation"
-Write-Output ""
-Write-Output "               - Installs All Salt Dependencies"
-Write-Output "               - Detects 32/64 bit Architectures"
-Write-Output ""
-Write-Output "               To run silently add -Silent"
-Write-Output "               eg: dev_env.ps1 -Silent"
-Write-Output ""
-Write-Output "================================================================="
-Write-Output ""
-
 #==============================================================================
 # Get the Directory of actual script
 #==============================================================================
@@ -49,6 +37,22 @@ $script_path = $script_path.DirectoryName
 #==============================================================================
 $script_name = $MyInvocation.MyCommand.Name
 
+Write-Output "================================================================="
+Write-Output ""
+Write-Output "               Development Environment Installation"
+Write-Output ""
+Write-Output "               - Installs All Salt Dependencies"
+Write-Output "               - Detects 32/64 bit Architectures"
+Write-Output ""
+Write-Output "               To run silently add -Silent"
+Write-Output "               eg: ${script_name} -Silent"
+Write-Output ""
+Write-Output "               To run skip installing pip dependencies add -NoPipDependencies"
+Write-Output "               eg: ${script_name} -NoPipDependencies"
+Write-Output ""
+Write-Output "================================================================="
+Write-Output ""
+
 #==============================================================================
 # Import Modules
 #==============================================================================
@@ -211,23 +215,45 @@ if ( ! [bool]$Env:SALT_PIP_LOCAL_CACHE) {
 }
 
 #==============================================================================
-# Install pypi resources using pip
+# Install windows specific pypi resources using pip
 #    caching depends on environment variable SALT_REQ_LOCAL_CACHE
 #==============================================================================
 Write-Output " ----------------------------------------------------------------"
-Write-Output " - $script_name :: Installing pypi resources using pip . . ."
+Write-Output " - $script_name :: Installing windows specific pypi resources using pip . . ."
 Write-Output " ----------------------------------------------------------------"
 if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) {
-    Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req.txt" "pip install"
+    Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req_win.txt" "pip install"
 } else {
     if ( (Get-ChildItem $Env:SALT_REQ_LOCAL_CACHE | Measure-Object).Count -eq 0 ) {
         # folder empty
-        Write-Output "    pip download from req.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE"
+        Write-Output "    pip download from req_win.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE"
-        Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip download"
+        Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_win.txt" "pip download"
     }
     Write-Output "    reading from local pip cache $Env:SALT_REQ_LOCAL_CACHE"
     Write-Output "    If a (new) resource is missing, please delete all files in this cache, go online and repeat"
-  Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip install"
+  Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_win.txt" "pip install"
+}
+
+#==============================================================================
+# Install pypi resources using pip
+#    caching depends on environment variable SALT_REQ_LOCAL_CACHE
+#==============================================================================
+If ($NoPipDependencies -eq $false) {
+  Write-Output " ----------------------------------------------------------------"
+  Write-Output " - $script_name :: Installing pypi resources using pip . . ."
+  Write-Output " ----------------------------------------------------------------"
+  if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) {
+      Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req.txt" "pip install"
+  } else {
+      if ( (Get-ChildItem $Env:SALT_REQ_LOCAL_CACHE | Measure-Object).Count -eq 0 ) {
+          # folder empty
+          Write-Output "    pip download from req.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE"
+          Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip download"
+      }
+      Write-Output "    reading from local pip cache $Env:SALT_REQ_LOCAL_CACHE"
+      Write-Output "    If a (new) resource is missing, please delete all files in this cache, go online and repeat"
+    Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip install"
+  }
 }
 
 #==============================================================================
@@ -262,11 +288,13 @@ Remove-Item "$($ini['Settings']['Scripts3Dir'])\pywin32_*" -Force -Recurse
 #==============================================================================
 # Fix PyCrypto
 #==============================================================================
-Write-Output " ----------------------------------------------------------------"
+If ($NoPipDependencies -eq $false) {
-Write-Output "   - $script_name :: Fixing PyCrypto . . ."
+  Write-Output " ----------------------------------------------------------------"
-Write-Output " ----------------------------------------------------------------"
+  Write-Output "   - $script_name :: Fixing PyCrypto . . ."
-$nt_file = "$($ini['Settings']['Python3Dir'])\Lib\site-packages\Crypto\Random\OSRNG\nt.py"
+  Write-Output " ----------------------------------------------------------------"
-(Get-Content $nt_file) | Foreach-Object {$_ -replace '^import winrandom$', 'from Crypto.Random.OSRNG import winrandom'} | Set-Content $nt_file
+  $nt_file = "$($ini['Settings']['Python3Dir'])\Lib\site-packages\Crypto\Random\OSRNG\nt.py"
+  (Get-Content $nt_file) | Foreach-Object {$_ -replace '^import winrandom$', 'from Crypto.Random.OSRNG import winrandom'} | Set-Content $nt_file
+}
 
 #==============================================================================
 # Copy DLLs to Python Directory

pkg/windows/req.txt

@@ -1,3 +1,4 @@
+-r req_win.txt
 backports-abc==0.5
 backports.ssl-match-hostname==3.5.0.1
 certifi
@@ -28,7 +29,6 @@ pyOpenSSL==17.5.0
 python-dateutil==2.6.1
 python-gnupg==0.4.1
 pythonnet==2.3.0
-pywin32==223
 PyYAML==3.12
 pyzmq==16.0.3
 requests==2.21.0
@@ -37,4 +37,3 @@ smmap==0.9.0
 timelib==0.2.4
 tornado==4.5.1
 wheel==0.30.0a0
-WMI==1.4.9

pkg/windows/req_win.txt

@@ -0,0 +1,2 @@
+pywin32==223
+WMI==1.4.9

salt/modules/gem.py

@@ -91,8 +91,12 @@ def install(gems,           # pylint: disable=C0103
         Doesn't play nice with multiple gems at once
     :param rdoc: boolean : False
         Generate RDoc documentation for the gem(s).
+        For rubygems > 3 this is interpreted as the --no-document arg and the
+        ri option will then be ignored
     :param ri: boolean : False
         Generate RI documentation for the gem(s).
+        For rubygems > 3 this is interpreted as the --no-document arg and the
+        rdoc option will then be ignored
     :param pre_releases: boolean : False
         Include pre-releases in the available versions
     :param proxy: string : None
@@ -119,12 +123,18 @@ def install(gems,           # pylint: disable=C0103
     options = []
     if version:
         options.extend(['--version', version])
-    if not rdoc:
+    if _has_rubygems_3(ruby=ruby, runas=runas, gem_bin=gem_bin):
-        options.append('--no-rdoc')
+        if not rdoc or not ri:
-    if not ri:
+            options.append('--no-document')
-        options.append('--no-ri')
+        if pre_releases:
-    if pre_releases:
+            options.append('--prerelease')
-        options.append('--pre')
+    else:
+        if not rdoc:
+            options.append('--no-rdoc')
+        if not ri:
+            options.append('--no-ri')
+        if pre_releases:
+            options.append('--pre')
     if proxy:
         options.extend(['-p', proxy])
     if source:
@@ -224,6 +234,45 @@ def update_system(version='', ruby=None, runas=None, gem_bin=None):
                 runas=runas)
 
 
+def version(ruby=None, runas=None, gem_bin=None):
+    '''
+    Print out the version of gem
+
+    :param gem_bin: string : None
+        Full path to ``gem`` binary to use.
+    :param ruby: string : None
+        If RVM or rbenv are installed, the ruby version and gemset to use.
+        Ignored if ``gem_bin`` is specified.
+    :param runas: string : None
+        The user to run gem as.
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt '*' gem.version
+    '''
+    cmd = ['--version']
+    stdout = _gem(cmd,
+                  ruby,
+                  gem_bin=gem_bin,
+                  runas=runas)
+    ret = {}
+    for line in salt.utils.itertools.split(stdout, '\n'):
+        match = re.match(r'[.0-9]+', line)
+        if match:
+            ret = line
+            break
+    return ret
+
+
+def _has_rubygems_3(ruby=None, runas=None, gem_bin=None):
+    match = re.match(r'^3\..*', version(ruby=ruby, runas=runas, gem_bin=gem_bin))
+    if match:
+        return True
+    return False
+
+
 def list_(prefix='', ruby=None, runas=None, gem_bin=None):
     '''
     List locally installed gems.

tests/unit/modules/test_gem.py

@@ -73,12 +73,29 @@ class TestGemModule(TestCase, LoaderModuleMockMixin):
                 runas=None
             )
 
+    def test_install_pre_rubygems_3(self):
+        mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
+        with patch.dict(gem.__salt__,
+                        {'rvm.is_installed': MagicMock(return_value=False),
+                         'rbenv.is_installed': MagicMock(return_value=False),
+                         'cmd.run_all': mock}),\
+                patch.object(
+                    gem, '_has_rubygems_3', MagicMock(return_value=True)):
+            gem.install('rails', pre_releases=True)
+            mock.assert_called_once_with(
+                ['gem', 'install', 'rails', '--no-document', '--prerelease'],
+                runas=None,
+                python_shell=False
+            )
+
     def test_install_pre(self):
         mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
         with patch.dict(gem.__salt__,
                         {'rvm.is_installed': MagicMock(return_value=False),
                          'rbenv.is_installed': MagicMock(return_value=False),
-                         'cmd.run_all': mock}):
+                         'cmd.run_all': mock}),\
+                patch.object(
+                    gem, '_has_rubygems_3', MagicMock(return_value=False)):
             gem.install('rails', pre_releases=True)
             mock.assert_called_once_with(
                 ['gem', 'install', 'rails', '--no-rdoc', '--no-ri', '--pre'],

tests/unit/utils/test_filebuffer.py

@@ -9,9 +9,11 @@
 
 # Import Python libs
 from __future__ import absolute_import, unicode_literals, print_function
+import os
 
 # Import Salt Testing libs
 from tests.support.unit import TestCase
+from tests.support.paths import BASE_FILES
 
 # Import salt libs
 from salt.utils.filebuffer import BufferedReader, InvalidFileMode
@@ -30,3 +32,22 @@ class TestFileBuffer(TestCase):
 
         with self.assertRaises(InvalidFileMode):
             BufferedReader('/tmp/foo', mode='wb')
+
+    def test_issue_51309(self):
+        '''
+        https://github.com/saltstack/salt/issues/51309
+        '''
+        file_name = os.path.join(BASE_FILES, 'grail', 'scene33')
+
+        def find_value(text):
+            stripped_text = text.strip()
+            try:
+                with BufferedReader(file_name) as breader:
+                    for chunk in breader:
+                        if stripped_text in chunk:
+                            return True
+                return False
+            except (IOError, OSError):
+                return False
+
+        self.assertTrue(find_value('We have the Holy Hand Grenade'))