From da2e6325666b018510cebb67c6ea191c526d30f6 Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Thu, 27 Sep 2018 14:34:50 +0200 Subject: [PATCH 1/9] parted: fix the ordering of list command The command `parted -m -s {} print` do not print the "Type" column, but after the "File System" column print the name of the partition. This point that the type of the partition cannot be extracted using parted with machine parseable output. (cherry picked from commit f23d618e116367b1a6ad414307e53c57b118a457) --- salt/modules/parted.py | 4 ++-- tests/unit/modules/test_parted.py | 32 +++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/salt/modules/parted.py b/salt/modules/parted.py index b5f50e1e68..8f6199e203 100644 --- a/salt/modules/parted.py +++ b/salt/modules/parted.py @@ -183,8 +183,8 @@ def list_(device, unit=None): 'start': cols[1], 'end': cols[2], 'size': cols[3], - 'type': cols[4], - 'file system': cols[5], + 'file system': cols[4], + 'name': cols[5], 'flags': cols[6]} else: raise CommandExecutionError( diff --git a/tests/unit/modules/test_parted.py b/tests/unit/modules/test_parted.py index 0bf9bd5305..b814d2e0d2 100644 --- a/tests/unit/modules/test_parted.py +++ b/tests/unit/modules/test_parted.py @@ -207,17 +207,17 @@ class PartedTestCase(TestCase, LoaderModuleMockMixin): 'end': '150MB', 'number': '1', 'start': '17.4kB', - 'file system': '', + 'file system': 'ext3', 'flags': 'boot', - 'type': 'ext3', + 'name': '', 'size': '150MB'}, '2': { 'end': '4000GB', 'number': '2', 'start': '3921GB', - 'file system': '', + 'file system': 'linux-swap(v1)', 'flags': '', - 'type': 'linux-swap(v1)', + 'name': '', 'size': '79.3GB' } } @@ -245,17 +245,17 @@ class PartedTestCase(TestCase, LoaderModuleMockMixin): 'end': '150MB', 'number': '1', 'start': '17.4kB', - 'file system': '', + 'file system': 'ext3', 'flags': 'boot', - 'type': 'ext3', + 'name': '', 'size': '150MB'}, '2': { 'end': '4000GB', 'number': '2', 'start': '3921GB', - 'file system': '', + 'file system': 'linux-swap(v1)', 'flags': '', - 'type': 'linux-swap(v1)', + 'name': '', 'size': '79.3GB' } } @@ -282,17 +282,17 @@ class PartedTestCase(TestCase, LoaderModuleMockMixin): 'end': '150MB', 'number': '1', 'start': '17.4kB', - 'file system': '', + 'file system': 'ext3', 'flags': 'boot', - 'type': 'ext3', + 'name': '', 'size': '150MB'}, '2': { 'end': '4000GB', 'number': '2', 'start': '3921GB', - 'file system': '', + 'file system': 'linux-swap(v1)', 'flags': '', - 'type': 'linux-swap(v1)', + 'name': '', 'size': '79.3GB' } } @@ -319,17 +319,17 @@ class PartedTestCase(TestCase, LoaderModuleMockMixin): 'end': '150MB', 'number': '1', 'start': '17.4kB', - 'file system': '', + 'file system': 'ext3', 'flags': 'boot', - 'type': 'ext3', + 'name': '', 'size': '150MB'}, '2': { 'end': '4000GB', 'number': '2', 'start': '3921GB', - 'file system': '', + 'file system': 'linux-swap(v1)', 'flags': '', - 'type': 'linux-swap(v1)', + 'name': '', 'size': '79.3GB' } } From d38622e7ed68193f3f91515a0d6a458f194d02ea Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Fri, 19 Oct 2018 11:43:34 +0200 Subject: [PATCH 2/9] Fix lowpkg.diff documentation and parameter name Package names are useless for lowpkg.diff, as to make a full comparison we need to access the RPM Fix #50097 (cherry picked from commit 5484791c0f953de2c2e6c5c65c3621b67e3b0f5d) --- salt/modules/rpm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/rpm.py b/salt/modules/rpm.py index 15738b2523..893ae4f817 100644 --- a/salt/modules/rpm.py +++ b/salt/modules/rpm.py @@ -425,13 +425,13 @@ def owner(*paths): @salt.utils.decorators.path.which('rpm2cpio') @salt.utils.decorators.path.which('cpio') @salt.utils.decorators.path.which('diff') -def diff(package, path): +def diff(package_path, path): ''' Return a formatted diff between current file and original in a package. NOTE: this function includes all files (configuration and not), but does not work on binary content. - :param package: The name of the package + :param package: Full pack of the RPM file :param path: Full path to the installed file :return: Difference or empty string. For binary files only a notification. @@ -439,13 +439,13 @@ def diff(package, path): .. code-block:: bash - salt '*' lowpkg.diff apache2 /etc/apache2/httpd.conf + salt '*' lowpkg.diff /path/to/apache2.rpm /etc/apache2/httpd.conf ''' cmd = "rpm2cpio {0} " \ "| cpio -i --quiet --to-stdout .{1} " \ "| diff -u --label 'A {1}' --from-file=- --label 'B {1}' {1}" - res = __salt__['cmd.shell'](cmd.format(package, path), + res = __salt__['cmd.shell'](cmd.format(package_path, path), output_loglevel='trace') if res and res.startswith('Binary file'): return 'File \'{0}\' is binary and its content has been ' \ From b5cff17f77d702b72c0193102eb6610929d0d377 Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Tue, 6 Nov 2018 15:54:31 +0100 Subject: [PATCH 3/9] service: SUSE is not based on sysvinit anymore (cherry picked from commit dd1ec387bb34c72db9a3241c4c345743a75439ef) --- salt/modules/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/modules/service.py b/salt/modules/service.py index 81090826e4..002b070c30 100644 --- a/salt/modules/service.py +++ b/salt/modules/service.py @@ -41,6 +41,7 @@ def __virtual__(): 'elementary OS', 'McAfee OS Server', 'Raspbian', + 'SUSE', )) if __grains__.get('os') in disable: return (False, 'Your OS is on the disabled list') From ab8a8b84b1ab7ef0f48d947b34d8f891e6c58ce0 Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Tue, 27 Nov 2018 10:21:51 +0100 Subject: [PATCH 4/9] file: update attributes for lsattr and chattr (cherry picked from commit 1be76ec2915a18f82a17198da3852fe5d6a66ced) --- salt/modules/file.py | 8 ++++---- salt/states/file.py | 2 +- salt/states/netconfig.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 2e4eaf11bb..f85103d18d 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -577,7 +577,7 @@ def lsattr(path): for line in result.splitlines(): if not line.startswith('lsattr: '): vals = line.split(None, 1) - results[vals[1]] = re.findall(r"[acdijstuADST]", vals[0]) + results[vals[1]] = re.findall(r"[aAcCdDeijPsStTu]", vals[0]) return results @@ -594,8 +594,8 @@ def chattr(*files, **kwargs): should be added or removed from files attributes - One or more of the following characters: ``acdijstuADST``, representing - attributes to add to/remove from files + One or more of the following characters: ``aAcCdDeijPsStTu``, + representing attributes to add to/remove from files version a version number to assign to the file(s) @@ -620,7 +620,7 @@ def chattr(*files, **kwargs): raise SaltInvocationError( "Need an operator: 'add' or 'remove' to modify attributes.") if attributes is None: - raise SaltInvocationError("Need attributes: [AacDdijsTtSu]") + raise SaltInvocationError("Need attributes: [aAcCdDeijPsStTu]") cmd = ['chattr'] diff --git a/salt/states/file.py b/salt/states/file.py index 60b78915d4..0322a76140 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -2116,7 +2116,7 @@ def managed(name, attrs The attributes to have on this file, e.g. ``a``, ``i``. The attributes can be any or a combination of the following characters: - ``acdijstuADST``. + ``aAcCdDeijPsStTu``. .. note:: This option is **not** supported on Windows. diff --git a/salt/states/netconfig.py b/salt/states/netconfig.py index b1f8ed65e4..535a96805d 100644 --- a/salt/states/netconfig.py +++ b/salt/states/netconfig.py @@ -303,7 +303,7 @@ def saved(name, attrs The attributes to have on this file, e.g. ``a``, ``i``. The attributes can be any or a combination of the following characters: - ``acdijstuADST``. + ``aAcCdDeijPsStTu``. .. note:: This option is **not** supported on Windows. From 9193106c50f7263f6367563c09c59af46361ed30 Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Tue, 8 Jan 2019 13:30:57 +0100 Subject: [PATCH 5/9] cmdmod: add sysfs into the chroot Some commands executed inside the chroot require access to the /sys information. For example, /sys/firmware is used to get to information from UEFI systems. This patch mount a sysfs file system inside the chroot. Also normalize the name for devtmpfs, replacing the old udev one. (cherry picked from commit 2b5d45ac9901881945d3ff552b32e78b1664662e) --- salt/modules/cmdmod.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 0bfaaf38e9..a10f07c157 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -3035,12 +3035,16 @@ def run_chroot(root, ''' __salt__['mount.mount']( os.path.join(root, 'dev'), - 'udev', + 'devtmpfs', fstype='devtmpfs') __salt__['mount.mount']( os.path.join(root, 'proc'), 'proc', fstype='proc') + __salt__['mount.mount']( + os.path.join(root, 'sys'), + 'sysfs', + fstype='sysfs') # Execute chroot routine sh_ = '/bin/sh' @@ -3092,6 +3096,7 @@ def run_chroot(root, log.error('Processes running in chroot could not be killed, ' 'filesystem will remain mounted') + __salt__['mount.umount'](os.path.join(root, 'sys')) __salt__['mount.umount'](os.path.join(root, 'proc')) __salt__['mount.umount'](os.path.join(root, 'dev')) if hide_output: From 66743d6502fa4ec9d6d08a5efaf2b663834b2489 Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Thu, 10 Jan 2019 16:24:05 +0100 Subject: [PATCH 6/9] mount: cache blkid information During the mount of a device a call to blkid is done, to get information of the current devices. blkid is a slow operation, to avoid that use an internal cache, that is specified in /etc/blkid configuration file, but a single blkid can still redo this cache file, making the second call equally slow. This patch cache the blkid information for _active_mountinfo in the __context__ dict, making the next mount action a fast operation. (cherry picked from commit c1ede1d97c2c4f4cf7ae7ca84fe19f6bbcaa27cb) --- salt/modules/mount.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/modules/mount.py b/salt/modules/mount.py index 3d12d3aab5..4ba370e5b3 100644 --- a/salt/modules/mount.py +++ b/salt/modules/mount.py @@ -63,7 +63,9 @@ def _active_mountinfo(ret): msg = 'File not readable {0}' raise CommandExecutionError(msg.format(filename)) - blkid_info = __salt__['disk.blkid']() + if 'disk.blkid' not in __context__: + __context__['disk.blkid'] = __salt__['disk.blkid']() + blkid_info = __context__['disk.blkid'] with salt.utils.files.fopen(filename) as ifile: for line in ifile: From 467daf2e4cfeaedf6f6ac447c0f826be71dcccd9 Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Mon, 12 Nov 2018 14:27:13 +0100 Subject: [PATCH 7/9] parted: support variable length output for print The command print from parted have a variable lenght output, depending on the units requested, or the kind of partition. This patch add the logic to support the full range of outputs that parted 3.1, and early versions, are generating. (cherry picked from commit 5adf859e3f1c78a34886c065d8540947a7718120) --- salt/modules/parted.py | 31 +++++++++++++++-------- tests/unit/modules/test_parted.py | 41 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/salt/modules/parted.py b/salt/modules/parted.py index 8f6199e203..79c7dcd1ad 100644 --- a/salt/modules/parted.py +++ b/salt/modules/parted.py @@ -155,7 +155,7 @@ def list_(device, unit=None): for line in out: if line in ('BYT;', 'CHS;', 'CYL;'): continue - cols = line.replace(';', '').split(':') + cols = line.rstrip(';').split(':') if mode == 'info': if 7 <= len(cols) <= 8: ret['info'] = { @@ -177,15 +177,26 @@ def list_(device, unit=None): raise CommandExecutionError( 'Problem encountered while parsing output from parted') else: - if len(cols) == 7: - ret['partitions'][cols[0]] = { - 'number': cols[0], - 'start': cols[1], - 'end': cols[2], - 'size': cols[3], - 'file system': cols[4], - 'name': cols[5], - 'flags': cols[6]} + # Parted (v3.1) have a variable field list in machine + # readable output: + # + # number:start:end:[size:]([file system:name:flags;]|[free;]) + # + # * If units are in CHS 'size' is not printed. + # * If is a logical partition with PED_PARTITION_FREESPACE + # set, the last three fields are replaced with the + # 'free' text. + # + fields = ['number', 'start', 'end'] + if unit != 'chs': + fields.append('size') + if cols[-1] == 'free': + # Drop the last element from the list + cols.pop() + else: + fields.extend(['file system', 'name', 'flags']) + if len(fields) == len(cols): + ret['partitions'][cols[0]] = dict(six.moves.zip(fields, cols)) else: raise CommandExecutionError( 'Problem encountered while parsing output from parted') diff --git a/tests/unit/modules/test_parted.py b/tests/unit/modules/test_parted.py index b814d2e0d2..ffb001f6bc 100644 --- a/tests/unit/modules/test_parted.py +++ b/tests/unit/modules/test_parted.py @@ -116,6 +116,12 @@ class PartedTestCase(TestCase, LoaderModuleMockMixin): '''1:17.4kB:150MB:150MB:ext3::boot;\n''' '''2:3921GB:4000GB:79.3GB:linux-swap(v1)::;\n''' ), + "valid chs": ( + '''CHS;\n''' + '''/dev/sda:3133,0,2:scsi:512:512:gpt:AMCC 9650SE-24M DISK:;\n''' + '''1:0,0,34:2431,134,43:ext3::boot;\n''' + '''2:2431,134,44:2492,80,42:linux-swap(v1)::;\n''' + ), "valid_legacy": ( '''BYT;\n''' '''/dev/sda:4000GB:scsi:512:512:gpt:AMCC 9650SE-24M DISK;\n''' @@ -262,6 +268,41 @@ class PartedTestCase(TestCase, LoaderModuleMockMixin): } self.assertEqual(output, expected) + def test_list__valid_unit_chs_valid_cmd_output(self): + with patch('salt.modules.parted._validate_device', MagicMock()): + self.cmdrun_stdout.return_value = self.parted_print_output('valid chs') + output = parted.list_('/dev/sda', unit='chs') + self.cmdrun_stdout.assert_called_once_with('parted -m -s /dev/sda unit chs print') + expected = { + 'info': { + 'logical sector': '512', + 'physical sector': '512', + 'interface': 'scsi', + 'model': 'AMCC 9650SE-24M DISK', + 'disk': '/dev/sda', + 'disk flags': '', + 'partition table': 'gpt', + 'size': '3133,0,2' + }, + 'partitions': { + '1': { + 'end': '2431,134,43', + 'number': '1', + 'start': '0,0,34', + 'file system': 'ext3', + 'flags': 'boot', + 'name': ''}, + '2': { + 'end': '2492,80,42', + 'number': '2', + 'start': '2431,134,44', + 'file system': 'linux-swap(v1)', + 'flags': '', + 'name': ''} + } + } + self.assertEqual(output, expected) + def test_list__valid_legacy_cmd_output(self): with patch('salt.modules.parted._validate_device', MagicMock()): self.cmdrun_stdout.return_value = self.parted_print_output('valid_legacy') From cc94dab469816e99124009450fa134f14cb54c9f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 1 Feb 2019 16:22:43 +0000 Subject: [PATCH 8/9] Fix the `ldap` pacakge name in requirements/tests.txt --- requirements/tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/tests.txt b/requirements/tests.txt index 6747e0a2ed..03dcfe82ad 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -23,7 +23,7 @@ pyvmomi setproctitle cherrypy>=3.2.2,<18.0.0; python_version < '3.5' and sys.platform != 'win32' and sys.platform != 'darwin' cherrypy>=3.2.2; python_version >= '3.5' and sys.platform != 'win32' and sys.platform != 'darwin' -ldap; sys.platform != 'win32' and sys.platform != 'darwin' +python-ldap; sys.platform != 'win32' and sys.platform != 'darwin' pyinotify; sys.platform != 'win32' and sys.platform != 'darwin' PyMySQL; sys.platform != 'win32' and sys.platform != 'darwin' jsonschema From 6f0d4076e1b72f252af3c718c64fee33e26d01e5 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 1 Feb 2019 16:36:25 +0000 Subject: [PATCH 9/9] Actually, remove python-ldap. It needs system deps. Have salt-jenkins do it. --- requirements/tests.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/tests.txt b/requirements/tests.txt index 03dcfe82ad..a80cddc34a 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -23,7 +23,6 @@ pyvmomi setproctitle cherrypy>=3.2.2,<18.0.0; python_version < '3.5' and sys.platform != 'win32' and sys.platform != 'darwin' cherrypy>=3.2.2; python_version >= '3.5' and sys.platform != 'win32' and sys.platform != 'darwin' -python-ldap; sys.platform != 'win32' and sys.platform != 'darwin' pyinotify; sys.platform != 'win32' and sys.platform != 'darwin' PyMySQL; sys.platform != 'win32' and sys.platform != 'darwin' jsonschema