From 464eef6fe16a6dd26db48db7e8be93b8d17ecfe7 Mon Sep 17 00:00:00 2001 From: JerzyX Drozdz Date: Thu, 1 Mar 2018 10:41:48 +0100 Subject: [PATCH 01/26] _get_line_indent renamed to _set_line_indent --- salt/modules/file.py | 18 +++++++++--------- salt/modules/win_file.py | 28 ++++++++++++++-------------- 2 files changed, 23 insertions(+), 23 deletions(-) mode change 100644 => 100755 salt/modules/file.py mode change 100644 => 100755 salt/modules/win_file.py diff --git a/salt/modules/file.py b/salt/modules/file.py old mode 100644 new mode 100755 index 63abea73ee..9ce957098c --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1745,7 +1745,7 @@ def _assert_occurrence(src, probe, target, amount=1): return occ -def _get_line_indent(src, line, indent): +def _set_line_indent(src, line, indent): ''' Indent the line with the source line. ''' @@ -1758,7 +1758,7 @@ def _get_line_indent(src, line, indent): break idt.append(c) - return ''.join(idt) + line.strip() + return ''.join(idt) + line.lstrip() def line(path, content=None, match=None, mode=None, location=None, @@ -1901,7 +1901,7 @@ def line(path, content=None, match=None, mode=None, location=None, elif mode == 'delete': body = os.linesep.join([line for line in body.split(os.linesep) if line.find(match) < 0]) elif mode == 'replace': - body = os.linesep.join([(_get_line_indent(file_line, content, indent) + body = os.linesep.join([(_set_line_indent(file_line, content, indent) if (file_line.find(match) > -1 and not file_line == content) else file_line) for file_line in body.split(os.linesep)]) elif mode == 'insert': @@ -1919,7 +1919,7 @@ def line(path, content=None, match=None, mode=None, location=None, if line.find(after) > -1: in_range = True elif line.find(before) > -1 and in_range: - out.append(_get_line_indent(line, content, indent)) + out.append(_set_line_indent(line, content, indent)) out.append(line) body = os.linesep.join(out) @@ -1930,7 +1930,7 @@ def line(path, content=None, match=None, mode=None, location=None, for idx in range(len(lines)): _line = lines[idx] if _line.find(before) > -1: - cnd = _get_line_indent(_line, content, indent) + cnd = _set_line_indent(_line, content, indent) if not idx or (idx and _starts_till(lines[idx - 1], cnd) < 0): # Job for replace instead out.append(cnd) out.append(_line) @@ -1942,7 +1942,7 @@ def line(path, content=None, match=None, mode=None, location=None, lines = body.split(os.linesep) for idx, _line in enumerate(lines): out.append(_line) - cnd = _get_line_indent(_line, content, indent) + cnd = _set_line_indent(_line, content, indent) # No duplicates or append, if "after" is the last line if (_line.find(after) > -1 and (lines[((idx + 1) < len(lines)) and idx + 1 or idx].strip() != cnd or @@ -1954,7 +1954,7 @@ def line(path, content=None, match=None, mode=None, location=None, if location == 'start': body = os.linesep.join((content, body)) elif location == 'end': - body = os.linesep.join((body, _get_line_indent(body[-1], content, indent) if body else content)) + body = os.linesep.join((body, _set_line_indent(body[-1], content, indent) if body else content)) elif mode == 'ensure': after = after and after.strip() @@ -1987,7 +1987,7 @@ def line(path, content=None, match=None, mode=None, location=None, for idx in range(len(body)): if body[idx].find(before) > -1: prev = (idx > 0 and idx or 1) - 1 - out.append(_get_line_indent(body[idx], content, indent)) + out.append(_set_line_indent(body[idx], content, indent)) if _starts_till(out[prev], content) > -1: del out[prev] out.append(body[idx]) @@ -2006,7 +2006,7 @@ def line(path, content=None, match=None, mode=None, location=None, next_line = idx + 1 < len(body) and body[idx + 1] or None if next_line is not None and _starts_till(next_line, content) > -1: skip = next_line - out.append(_get_line_indent(body[idx], content, indent)) + out.append(_set_line_indent(body[idx], content, indent)) body = os.linesep.join(out) else: diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py old mode 100644 new mode 100755 index 0789a5b339..37bff011b3 --- a/salt/modules/win_file.py +++ b/salt/modules/win_file.py @@ -46,20 +46,20 @@ import salt.utils.path import salt.utils.platform import salt.utils.user from salt.modules.file import (check_hash, # pylint: disable=W0611 - directory_exists, get_managed, - check_managed, check_managed_changes, source_list, - touch, append, contains, contains_regex, get_source_sum, - contains_glob, find, psed, get_sum, _get_bkroot, _mkstemp_copy, - get_hash, manage_file, file_exists, get_diff, line, list_backups, - __clean_tmp, check_file_meta, _binary_replace, - _splitlines_preserving_trailing_newline, restore_backup, - access, copy, readdir, read, rmdir, truncate, replace, delete_backup, - search, _get_flags, extract_hash, _error, _sed_esc, _psed, - RE_FLAG_TABLE, blockreplace, prepend, seek_read, seek_write, rename, - lstat, path_exists_glob, write, pardir, join, HASHES, HASHES_REVMAP, - comment, uncomment, _add_flags, comment_line, _regex_to_static, - _get_line_indent, apply_template_on_contents, dirname, basename, - list_backups_dir, _assert_occurrence, _starts_till) + directory_exists, get_managed, + check_managed, check_managed_changes, source_list, + touch, append, contains, contains_regex, get_source_sum, + contains_glob, find, psed, get_sum, _get_bkroot, _mkstemp_copy, + get_hash, manage_file, file_exists, get_diff, line, list_backups, + __clean_tmp, check_file_meta, _binary_replace, + _splitlines_preserving_trailing_newline, restore_backup, + access, copy, readdir, read, rmdir, truncate, replace, delete_backup, + search, _get_flags, extract_hash, _error, _sed_esc, _psed, + RE_FLAG_TABLE, blockreplace, prepend, seek_read, seek_write, rename, + lstat, path_exists_glob, write, pardir, join, HASHES, HASHES_REVMAP, + comment, uncomment, _add_flags, comment_line, _regex_to_static, + _set_line_indent, apply_template_on_contents, dirname, basename, + list_backups_dir, _assert_occurrence, _starts_till) from salt.modules.file import normpath as normpath_ from salt.utils.functools import namespaced_function as _namespaced_function From de668166f9c9669375311cb5d10c6441ecb96c11 Mon Sep 17 00:00:00 2001 From: JerzyX Drozdz Date: Thu, 1 Mar 2018 10:44:37 +0100 Subject: [PATCH 02/26] _regex_to_static refactored to work on lists --- salt/modules/file.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 9ce957098c..a797853c83 100755 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1718,11 +1718,12 @@ def _regex_to_static(src, regex): return None try: - src = re.search(regex, src, re.M) + compiled = re.compile(regex, re.DOTALL) + src = [res.group() for res in map(compiled.search, src) if res] except Exception as ex: raise CommandExecutionError("{0}: '{1}'".format(_get_error_message(ex), regex)) - return src and src.group().rstrip('\r') or regex + return src and src[0] or regex def _assert_occurrence(src, probe, target, amount=1): From aacbb8c0e2386a215e707c3d4fb70ceae7dd42e3 Mon Sep 17 00:00:00 2001 From: JerzyX Drozdz Date: Thu, 1 Mar 2018 11:58:52 +0100 Subject: [PATCH 03/26] line function refactored to work on list --- salt/modules/file.py | 56 +++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index a797853c83..9d778e6f22 100755 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1890,8 +1890,8 @@ def line(path, content=None, match=None, mode=None, location=None, match = content with salt.utils.files.fopen(path, mode='r') as fp_: - body = salt.utils.stringutils.to_unicode(fp_.read()) - body_before = hashlib.sha256(salt.utils.stringutils.to_bytes(body)).hexdigest() + body = [salt.utils.stringutils.to_unicode(line) for line in fp_.readlines()] + body_before = hashlib.sha256(salt.utils.stringutils.to_bytes(''.join(body))).hexdigest() after = _regex_to_static(body, after) before = _regex_to_static(body, before) match = _regex_to_static(body, match) @@ -1900,11 +1900,11 @@ def line(path, content=None, match=None, mode=None, location=None, log.warning('Cannot find text to {0}. File \'{1}\' is empty.'.format(mode, path)) body = '' elif mode == 'delete': - body = os.linesep.join([line for line in body.split(os.linesep) if line.find(match) < 0]) + body = [line for line in body if line.find(match) < 0] elif mode == 'replace': - body = os.linesep.join([(_set_line_indent(file_line, content, indent) - if (file_line.find(match) > -1 and not file_line == content) else file_line) - for file_line in body.split(os.linesep)]) + body = [(_set_line_indent(file_line, content, indent) + if (file_line.find(match) > -1 and not file_line == content) else file_line) + for file_line in body] elif mode == 'insert': if not location and not before and not after: raise CommandExecutionError('On insert must be defined either "location" or "before/after" conditions.') @@ -1914,48 +1914,45 @@ def line(path, content=None, match=None, mode=None, location=None, _assert_occurrence(body, before, 'before') _assert_occurrence(body, after, 'after') out = [] - lines = body.split(os.linesep) in_range = False - for line in lines: + for line in body: if line.find(after) > -1: in_range = True elif line.find(before) > -1 and in_range: out.append(_set_line_indent(line, content, indent)) out.append(line) - body = os.linesep.join(out) + body = out if before and not after: _assert_occurrence(body, before, 'before') out = [] - lines = body.split(os.linesep) - for idx in range(len(lines)): - _line = lines[idx] + for idx in range(len(body)): + _line = body[idx] if _line.find(before) > -1: cnd = _set_line_indent(_line, content, indent) - if not idx or (idx and _starts_till(lines[idx - 1], cnd) < 0): # Job for replace instead + if not idx or (idx and _starts_till(body[idx - 1], cnd) < 0): # Job for replace instead out.append(cnd) out.append(_line) - body = os.linesep.join(out) + body = out elif after and not before: _assert_occurrence(body, after, 'after') out = [] - lines = body.split(os.linesep) - for idx, _line in enumerate(lines): + for idx, _line in enumerate(body): out.append(_line) cnd = _set_line_indent(_line, content, indent) # No duplicates or append, if "after" is the last line if (_line.find(after) > -1 and - (lines[((idx + 1) < len(lines)) and idx + 1 or idx].strip() != cnd or - idx + 1 == len(lines))): + (body[((idx + 1) < len(body)) and idx + 1 or idx].strip() != cnd or + idx + 1 == len(body))): out.append(cnd) - body = os.linesep.join(out) + body = out else: if location == 'start': - body = os.linesep.join((content, body)) + body.insert(0, content + os.linesep) elif location == 'end': - body = os.linesep.join((body, _set_line_indent(body[-1], content, indent) if body else content)) + body.append(_set_line_indent(body[-1], content, indent) if body else content) elif mode == 'ensure': after = after and after.strip() @@ -1968,7 +1965,6 @@ def line(path, content=None, match=None, mode=None, location=None, is_there = bool(body.count(content)) if not is_there: out = [] - body = body.split(os.linesep) for idx, line in enumerate(body): out.append(line) if line.find(content) > -1: @@ -1979,11 +1975,10 @@ def line(path, content=None, match=None, mode=None, location=None, elif line.find(after) > -1: raise CommandExecutionError('Found more than one line between ' 'boundaries "before" and "after".') - body = os.linesep.join(out) + body = out elif before and not after: _assert_occurrence(body, before, 'before') - body = body.split(os.linesep) out = [] for idx in range(len(body)): if body[idx].find(before) > -1: @@ -1992,11 +1987,10 @@ def line(path, content=None, match=None, mode=None, location=None, if _starts_till(out[prev], content) > -1: del out[prev] out.append(body[idx]) - body = os.linesep.join(out) + body = out elif not before and after: _assert_occurrence(body, after, 'after') - body = body.split(os.linesep) skip = None out = [] for idx in range(len(body)): @@ -2008,14 +2002,14 @@ def line(path, content=None, match=None, mode=None, location=None, if next_line is not None and _starts_till(next_line, content) > -1: skip = next_line out.append(_set_line_indent(body[idx], content, indent)) - body = os.linesep.join(out) + body = out else: raise CommandExecutionError("Wrong conditions? " "Unable to ensure line without knowing " "where to put it before and/or after.") - changed = body_before != hashlib.sha256(salt.utils.stringutils.to_bytes(body)).hexdigest() + changed = body_before != hashlib.sha256(salt.utils.stringutils.to_bytes(''.join(body))).hexdigest() if backup and changed and __opts__['test'] is False: try: @@ -2032,9 +2026,7 @@ def line(path, content=None, match=None, mode=None, location=None, path_content = [salt.utils.stringutils.to_unicode(x) for x in fp_.read().splitlines(True)] changes_diff = ''.join(difflib.unified_diff( - path_content, - [salt.utils.stringutils.to_unicode(x) - for x in body.splitlines(True)] + path_content, body )) if __opts__['test'] is False: fh_ = None @@ -2047,7 +2039,7 @@ def line(path, content=None, match=None, mode=None, location=None, mode = 'w' body = salt.utils.stringutils.to_str(body) fh_ = salt.utils.atomicfile.atomic_open(path, mode) - fh_.write(body) + fh_.write(''.join(body)) finally: if fh_: fh_.close() From 489e381100f3b707402b9857d07cc4d7e8d219c8 Mon Sep 17 00:00:00 2001 From: Jerzy Drozdz Date: Thu, 1 Mar 2018 17:56:29 +0100 Subject: [PATCH 04/26] Added _set_line_eol and _get_eol functions --- salt/modules/file.py | 16 +++++++++++++++- salt/modules/win_file.py | 32 +++++++++++++++++--------------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 9d778e6f22..4453dcc7c9 100755 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1730,7 +1730,7 @@ def _assert_occurrence(src, probe, target, amount=1): ''' Raise an exception, if there are different amount of specified occurrences in src. ''' - occ = src.count(probe) + occ = len([l for l in src if l.count(probe)]) if occ > amount: msg = 'more than' elif occ < amount: @@ -1762,6 +1762,20 @@ def _set_line_indent(src, line, indent): return ''.join(idt) + line.lstrip() +def _get_eol(line): + compiled = re.compile('((? Date: Thu, 1 Mar 2018 18:36:50 +0100 Subject: [PATCH 05/26] Setting end of line --- salt/modules/file.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 4453dcc7c9..d49dd6376e 100755 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1773,7 +1773,7 @@ def _set_line_eol(src, line): Add line ending ''' line_ending = _get_eol(src) or os.linesep - return line.rstrip('\r\n') + line_ending + return line.rstrip() + line_ending def line(path, content=None, match=None, mode=None, location=None, @@ -1916,8 +1916,8 @@ def line(path, content=None, match=None, mode=None, location=None, elif mode == 'delete': body = [line for line in body if line.find(match) < 0] elif mode == 'replace': - body = [(_set_line_indent(file_line, content, indent) - if (file_line.find(match) > -1 and not file_line == content) else file_line) + body = [(_set_line_indent(file_line, _set_line_eol(file_line, content), indent) + if (file_line.find(match) > -1 and not file_line.strip() == content) else file_line) for file_line in body] elif mode == 'insert': if not location and not before and not after: @@ -1933,17 +1933,17 @@ def line(path, content=None, match=None, mode=None, location=None, if line.find(after) > -1: in_range = True elif line.find(before) > -1 and in_range: - out.append(_set_line_indent(line, content, indent)) + out.append(_set_line_indent(line, _set_line_eol(line, content), indent)) out.append(line) body = out if before and not after: _assert_occurrence(body, before, 'before') out = [] - for idx in range(len(body)): - _line = body[idx] + for idx, _line in enumerate(body): if _line.find(before) > -1: cnd = _set_line_indent(_line, content, indent) + cnd = _set_line_eol(_line, cnd) if not idx or (idx and _starts_till(body[idx - 1], cnd) < 0): # Job for replace instead out.append(cnd) out.append(_line) @@ -1955,17 +1955,26 @@ def line(path, content=None, match=None, mode=None, location=None, for idx, _line in enumerate(body): out.append(_line) cnd = _set_line_indent(_line, content, indent) + cnd = _set_line_eol(_line, cnd) # No duplicates or append, if "after" is the last line if (_line.find(after) > -1 and - (body[((idx + 1) < len(body)) and idx + 1 or idx].strip() != cnd or + (body[((idx + 1) < len(body)) and idx + 1 or idx].strip() != content or idx + 1 == len(body))): out.append(cnd) body = out else: if location == 'start': - body.insert(0, content + os.linesep) + if body: + body.insert(0, _set_line_eol(body[0], content)) + else: + body.append(content + os.linesep) elif location == 'end': + #If file ends without new line add one + if not _get_eol(body[-1]) and len(body) > 1: + body[-1] = _set_line_eol(body[-2], body[-1]) + elif not _get_eol(body[-1]): + body[-1] += os.linesep body.append(_set_line_indent(body[-1], content, indent) if body else content) elif mode == 'ensure': @@ -1985,7 +1994,7 @@ def line(path, content=None, match=None, mode=None, location=None, is_there = True if not is_there: if idx < (len(body) - 1) and line.find(after) > -1 and body[idx + 1].find(before) > -1: - out.append(content) + out.append(_set_line_eol(line, content)) elif line.find(after) > -1: raise CommandExecutionError('Found more than one line between ' 'boundaries "before" and "after".') @@ -1997,7 +2006,7 @@ def line(path, content=None, match=None, mode=None, location=None, for idx in range(len(body)): if body[idx].find(before) > -1: prev = (idx > 0 and idx or 1) - 1 - out.append(_set_line_indent(body[idx], content, indent)) + out.append(_set_line_indent(body[idx], _set_line_eol(body[idx], content), indent)) if _starts_till(out[prev], content) > -1: del out[prev] out.append(body[idx]) @@ -2015,7 +2024,7 @@ def line(path, content=None, match=None, mode=None, location=None, next_line = idx + 1 < len(body) and body[idx + 1] or None if next_line is not None and _starts_till(next_line, content) > -1: skip = next_line - out.append(_set_line_indent(body[idx], content, indent)) + out.append(_set_line_indent(body[idx], _set_line_eol(body[idx], content), indent)) body = out else: From 7d6e3ad2e2467c656ca20cd9f071e8db8f3e8b3b Mon Sep 17 00:00:00 2001 From: Jerzy Drozdz Date: Thu, 1 Mar 2018 21:39:50 +0100 Subject: [PATCH 06/26] Make tests green --- salt/modules/file.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index d49dd6376e..5fd06b0af3 100755 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1912,7 +1912,7 @@ def line(path, content=None, match=None, mode=None, location=None, if os.stat(path).st_size == 0 and mode in ('delete', 'replace'): log.warning('Cannot find text to {0}. File \'{1}\' is empty.'.format(mode, path)) - body = '' + body = [] elif mode == 'delete': body = [line for line in body if line.find(match) < 0] elif mode == 'replace': @@ -1954,12 +1954,16 @@ def line(path, content=None, match=None, mode=None, location=None, out = [] for idx, _line in enumerate(body): out.append(_line) - cnd = _set_line_indent(_line, content, indent) - cnd = _set_line_eol(_line, cnd) # No duplicates or append, if "after" is the last line if (_line.find(after) > -1 and (body[((idx + 1) < len(body)) and idx + 1 or idx].strip() != content or idx + 1 == len(body))): + if not _get_eol(_line) and idx > 0: + out[-1] = _set_line_eol(body[-2], _line) + elif not _get_eol(_line): + out[-1] += os.linesep + cnd = _set_line_indent(_line, content, indent) + cnd = _set_line_eol(_line, cnd) out.append(cnd) body = out @@ -1975,17 +1979,16 @@ def line(path, content=None, match=None, mode=None, location=None, body[-1] = _set_line_eol(body[-2], body[-1]) elif not _get_eol(body[-1]): body[-1] += os.linesep + #body = os.linesep.join((body, _get_line_indent(body[-1], content, indent) if body else content)) body.append(_set_line_indent(body[-1], content, indent) if body else content) elif mode == 'ensure': - after = after and after.strip() - before = before and before.strip() if before and after: _assert_occurrence(body, before, 'before') _assert_occurrence(body, after, 'after') - is_there = bool(body.count(content)) + is_there = bool(len([l for l in body if l.count(content)])) if not is_there: out = [] for idx, line in enumerate(body): @@ -2024,6 +2027,10 @@ def line(path, content=None, match=None, mode=None, location=None, next_line = idx + 1 < len(body) and body[idx + 1] or None if next_line is not None and _starts_till(next_line, content) > -1: skip = next_line + if not _get_eol(body[idx]) and idx > 0: + out[-1] = _set_line_eol(out[-2], body[idx]) + elif not _get_eol(body[idx]): + out[-1] += os.linesep out.append(_set_line_indent(body[idx], _set_line_eol(body[idx], content), indent)) body = out @@ -2032,6 +2039,8 @@ def line(path, content=None, match=None, mode=None, location=None, "Unable to ensure line without knowing " "where to put it before and/or after.") + if body: body.append(body.pop().rstrip('\r\n')) + changed = body_before != hashlib.sha256(salt.utils.stringutils.to_bytes(''.join(body))).hexdigest() if backup and changed and __opts__['test'] is False: From 935a9b9d56d1d0ab26ba4493b64851429510d4e3 Mon Sep 17 00:00:00 2001 From: Jerzy Drozdz Date: Thu, 1 Mar 2018 22:56:41 +0100 Subject: [PATCH 07/26] test_line_insert_end fixed --- tests/unit/modules/test_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/modules/test_file.py b/tests/unit/modules/test_file.py index eb1a609b40..2309daa55e 100644 --- a/tests/unit/modules/test_file.py +++ b/tests/unit/modules/test_file.py @@ -1260,7 +1260,7 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): ' base:', ' - /srv/salt', ' - /srv/sugar', - cfg_content + ' ' + cfg_content ]) isfile_mock = MagicMock(side_effect=lambda x: True if x == name else DEFAULT) From 3af551ebe1740c353feb2a12d346d0c77ec50675 Mon Sep 17 00:00:00 2001 From: Jerzy Drozdz Date: Thu, 1 Mar 2018 23:45:45 +0100 Subject: [PATCH 08/26] /sr.* pattern should raise exception --- tests/unit/modules/test_file.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/modules/test_file.py b/tests/unit/modules/test_file.py index 2309daa55e..3f1333416a 100644 --- a/tests/unit/modules/test_file.py +++ b/tests/unit/modules/test_file.py @@ -1157,6 +1157,32 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): write_content = handles[0].write.call_args_list[0][0][0] assert write_content == file_modified, write_content + @patch('os.path.realpath', MagicMock()) + @patch('os.path.isfile', MagicMock(return_value=True)) + @patch('os.stat', MagicMock()) + def test_line_assert_exception_pattern(self): + ''' + Test for file.line for exception on insert with too general pattern. + + :return: + ''' + file_content = os.linesep.join([ + 'file_roots:', + ' base:', + ' - /srv/salt', + ' - /srv/sugar' + ]) + cfg_content = '- /srv/custom' + for before_line in ['/sr.*']: + files_fopen = mock_open(read_data=file_content) + with patch('salt.utils.files.fopen', files_fopen): + atomic_opener = mock_open() + with patch('salt.utils.atomicfile.atomic_open', atomic_opener): + with self.assertRaises(CommandExecutionError) as cm: + filemod.line('foo', content=cfg_content, before=before_line, mode='insert') + self.assertEqual(cm.exception.strerror, + 'Found more than expected occurrences in "before" expression') + @with_tempfile() def test_line_insert_before_after(self, name): ''' From 9b7df671a5318598b3f41e329979a503ca53f82d Mon Sep 17 00:00:00 2001 From: Jerzy Drozdz Date: Fri, 2 Mar 2018 00:59:35 +0100 Subject: [PATCH 09/26] file.line function refactored --- salt/modules/file.py | 134 ++++++++++++++++--------------------------- 1 file changed, 50 insertions(+), 84 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 5fd06b0af3..87c172ff69 100755 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1719,18 +1719,18 @@ def _regex_to_static(src, regex): try: compiled = re.compile(regex, re.DOTALL) - src = [res.group() for res in map(compiled.search, src) if res] + src = [line for line in src if compiled.search(line) or line.count(regex)] except Exception as ex: raise CommandExecutionError("{0}: '{1}'".format(_get_error_message(ex), regex)) - return src and src[0] or regex + return src and src or [] -def _assert_occurrence(src, probe, target, amount=1): +def _assert_occurrence(probe, target, amount=1): ''' Raise an exception, if there are different amount of specified occurrences in src. ''' - occ = len([l for l in src if l.count(probe)]) + occ = len(probe) if occ > amount: msg = 'more than' elif occ < amount: @@ -1906,6 +1906,9 @@ def line(path, content=None, match=None, mode=None, location=None, with salt.utils.files.fopen(path, mode='r') as fp_: body = [salt.utils.stringutils.to_unicode(line) for line in fp_.readlines()] body_before = hashlib.sha256(salt.utils.stringutils.to_bytes(''.join(body))).hexdigest() + if body and body[-1] and not _get_eol(body[-1]): + body[-1] += (len(body) > 1) and _get_eol(body[-2]) or os.linesep + after = _regex_to_static(body, after) before = _regex_to_static(body, before) match = _regex_to_static(body, match) @@ -1914,10 +1917,10 @@ def line(path, content=None, match=None, mode=None, location=None, log.warning('Cannot find text to {0}. File \'{1}\' is empty.'.format(mode, path)) body = [] elif mode == 'delete': - body = [line for line in body if line.find(match) < 0] + body = [line for line in body if line != match[0]] elif mode == 'replace': body = [(_set_line_indent(file_line, _set_line_eol(file_line, content), indent) - if (file_line.find(match) > -1 and not file_line.strip() == content) else file_line) + if (file_line == match[0] and not file_line.strip() == content) else file_line) for file_line in body] elif mode == 'insert': if not location and not before and not after: @@ -1925,47 +1928,37 @@ def line(path, content=None, match=None, mode=None, location=None, if not location: if before and after: - _assert_occurrence(body, before, 'before') - _assert_occurrence(body, after, 'after') + _assert_occurrence(before, 'before') + _assert_occurrence(after, 'after') + out = [] in_range = False for line in body: - if line.find(after) > -1: + if line == after[0]: in_range = True - elif line.find(before) > -1 and in_range: + elif line == before[0] and in_range: out.append(_set_line_indent(line, _set_line_eol(line, content), indent)) out.append(line) body = out if before and not after: - _assert_occurrence(body, before, 'before') - out = [] - for idx, _line in enumerate(body): - if _line.find(before) > -1: - cnd = _set_line_indent(_line, content, indent) - cnd = _set_line_eol(_line, cnd) - if not idx or (idx and _starts_till(body[idx - 1], cnd) < 0): # Job for replace instead - out.append(cnd) - out.append(_line) - body = out + _assert_occurrence(before, 'before') + + idx = body.index(before[0]) + if not idx or (idx and _starts_till(body[idx - 1], content) < 0): # Job for replace instead + cnd = _set_line_indent(body[idx], content, indent) + cnd = _set_line_eol(body[idx], cnd) + body.insert(idx, cnd) elif after and not before: - _assert_occurrence(body, after, 'after') - out = [] - for idx, _line in enumerate(body): - out.append(_line) - # No duplicates or append, if "after" is the last line - if (_line.find(after) > -1 and - (body[((idx + 1) < len(body)) and idx + 1 or idx].strip() != content or - idx + 1 == len(body))): - if not _get_eol(_line) and idx > 0: - out[-1] = _set_line_eol(body[-2], _line) - elif not _get_eol(_line): - out[-1] += os.linesep - cnd = _set_line_indent(_line, content, indent) - cnd = _set_line_eol(_line, cnd) - out.append(cnd) - body = out + _assert_occurrence(after, 'after') + + idx = body.index(after[0]) + # No duplicates or append, if "after" is the last line + if body[((idx + 1) < len(body)) and idx + 1 or idx].strip() != content or idx + 1 == len(body): + cnd = _set_line_indent(body[idx], content, indent) + cnd = _set_line_eol(body[idx], cnd) + body.insert(idx + 1, cnd) else: if location == 'start': @@ -1974,72 +1967,45 @@ def line(path, content=None, match=None, mode=None, location=None, else: body.append(content + os.linesep) elif location == 'end': - #If file ends without new line add one - if not _get_eol(body[-1]) and len(body) > 1: - body[-1] = _set_line_eol(body[-2], body[-1]) - elif not _get_eol(body[-1]): - body[-1] += os.linesep - #body = os.linesep.join((body, _get_line_indent(body[-1], content, indent) if body else content)) body.append(_set_line_indent(body[-1], content, indent) if body else content) elif mode == 'ensure': if before and after: - _assert_occurrence(body, before, 'before') - _assert_occurrence(body, after, 'after') + _assert_occurrence(before, 'before') + _assert_occurrence(after, 'after') - is_there = bool(len([l for l in body if l.count(content)])) + is_there = bool([l for l in body if l.count(content)]) if not is_there: - out = [] - for idx, line in enumerate(body): - out.append(line) - if line.find(content) > -1: - is_there = True - if not is_there: - if idx < (len(body) - 1) and line.find(after) > -1 and body[idx + 1].find(before) > -1: - out.append(_set_line_eol(line, content)) - elif line.find(after) > -1: - raise CommandExecutionError('Found more than one line between ' - 'boundaries "before" and "after".') - body = out + idx = body.index(after[0]) + if idx < (len(body) - 1) and body[idx + 1] == before[0]: + body.insert(idx + 1, _set_line_indent(body[idx], _set_line_eol(body[idx], content), indent)) + else: + raise CommandExecutionError('Found more than one line between ' + 'boundaries "before" and "after".') elif before and not after: - _assert_occurrence(body, before, 'before') - out = [] - for idx in range(len(body)): - if body[idx].find(before) > -1: - prev = (idx > 0 and idx or 1) - 1 - out.append(_set_line_indent(body[idx], _set_line_eol(body[idx], content), indent)) - if _starts_till(out[prev], content) > -1: - del out[prev] - out.append(body[idx]) - body = out + _assert_occurrence(before, 'before') + + idx = body.index(before[0]) + if body[idx-1].find(content) < 0: + body.insert(idx, _set_line_indent(body[idx], _set_line_eol(body[idx], content), indent)) elif not before and after: - _assert_occurrence(body, after, 'after') - skip = None - out = [] - for idx in range(len(body)): - if skip != body[idx]: - out.append(body[idx]) + _assert_occurrence(after, 'after') - if body[idx].find(after) > -1: - next_line = idx + 1 < len(body) and body[idx + 1] or None - if next_line is not None and _starts_till(next_line, content) > -1: - skip = next_line - if not _get_eol(body[idx]) and idx > 0: - out[-1] = _set_line_eol(out[-2], body[idx]) - elif not _get_eol(body[idx]): - out[-1] += os.linesep - out.append(_set_line_indent(body[idx], _set_line_eol(body[idx], content), indent)) - body = out + idx = body.index(after[0]) + next_line = idx + 1 < len(body) and body[idx + 1] or None + if next_line is None or _starts_till(next_line, content) < 0: + body.insert(idx+1, _set_line_indent(body[idx], _set_line_eol(body[idx], content), indent)) else: raise CommandExecutionError("Wrong conditions? " "Unable to ensure line without knowing " "where to put it before and/or after.") - if body: body.append(body.pop().rstrip('\r\n')) + if body: + body.append(body.pop().rstrip('\r\n')) changed = body_before != hashlib.sha256(salt.utils.stringutils.to_bytes(''.join(body))).hexdigest() From d2af81e9c7a9f238513b6b05d5d7b46dd63dcfe9 Mon Sep 17 00:00:00 2001 From: Jerzy Drozdz Date: Sun, 4 Mar 2018 03:00:42 +0100 Subject: [PATCH 10/26] Make integration green. Added test for mode ensure insert before first line --- salt/modules/file.py | 51 +++++++++++++++++++++------------ salt/modules/win_file.py | 8 ++++-- tests/unit/modules/test_file.py | 27 +++++++++++++++++ 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 87c172ff69..fd127dbb74 100755 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1776,6 +1776,22 @@ def _set_line_eol(src, line): return line.rstrip() + line_ending +def _insert_line_before(idx, body, content, indent): + if not idx or (idx and _starts_till(body[idx - 1], content) < 0): + cnd = _set_line_indent(body[idx], content, indent) + body.insert(idx, cnd) + return body + + +def _insert_line_after(idx, body, content, indent): + # No duplicates or append, if "after" is the last line + next_line = idx + 1 < len(body) and body[idx + 1] or None + if next_line is None or _starts_till(next_line, content) < 0: + cnd = _set_line_indent(body[idx], content, indent) + body.insert(idx + 1, cnd) + return body + + def line(path, content=None, match=None, mode=None, location=None, before=None, after=None, show_changes=True, backup=False, quiet=False, indent=True): @@ -1906,8 +1922,8 @@ def line(path, content=None, match=None, mode=None, location=None, with salt.utils.files.fopen(path, mode='r') as fp_: body = [salt.utils.stringutils.to_unicode(line) for line in fp_.readlines()] body_before = hashlib.sha256(salt.utils.stringutils.to_bytes(''.join(body))).hexdigest() - if body and body[-1] and not _get_eol(body[-1]): - body[-1] += (len(body) > 1) and _get_eol(body[-2]) or os.linesep + if body and _get_eol(body[-1]): + body.append('') after = _regex_to_static(body, after) before = _regex_to_static(body, before) @@ -1937,7 +1953,8 @@ def line(path, content=None, match=None, mode=None, location=None, if line == after[0]: in_range = True elif line == before[0] and in_range: - out.append(_set_line_indent(line, _set_line_eol(line, content), indent)) + cnd = _set_line_indent(line, content, indent) + out.append(cnd) out.append(line) body = out @@ -1945,20 +1962,13 @@ def line(path, content=None, match=None, mode=None, location=None, _assert_occurrence(before, 'before') idx = body.index(before[0]) - if not idx or (idx and _starts_till(body[idx - 1], content) < 0): # Job for replace instead - cnd = _set_line_indent(body[idx], content, indent) - cnd = _set_line_eol(body[idx], cnd) - body.insert(idx, cnd) + body = _insert_line_before(idx, body, content, indent) elif after and not before: _assert_occurrence(after, 'after') idx = body.index(after[0]) - # No duplicates or append, if "after" is the last line - if body[((idx + 1) < len(body)) and idx + 1 or idx].strip() != content or idx + 1 == len(body): - cnd = _set_line_indent(body[idx], content, indent) - cnd = _set_line_eol(body[idx], cnd) - body.insert(idx + 1, cnd) + body = _insert_line_after(idx, body, content, indent) else: if location == 'start': @@ -1979,7 +1989,8 @@ def line(path, content=None, match=None, mode=None, location=None, if not is_there: idx = body.index(after[0]) if idx < (len(body) - 1) and body[idx + 1] == before[0]: - body.insert(idx + 1, _set_line_indent(body[idx], _set_line_eol(body[idx], content), indent)) + cnd = _set_line_indent(body[idx], content, indent) + body.insert(idx + 1, cnd) else: raise CommandExecutionError('Found more than one line between ' 'boundaries "before" and "after".') @@ -1988,16 +1999,13 @@ def line(path, content=None, match=None, mode=None, location=None, _assert_occurrence(before, 'before') idx = body.index(before[0]) - if body[idx-1].find(content) < 0: - body.insert(idx, _set_line_indent(body[idx], _set_line_eol(body[idx], content), indent)) + body = _insert_line_before(idx, body, content, indent) elif not before and after: _assert_occurrence(after, 'after') idx = body.index(after[0]) - next_line = idx + 1 < len(body) and body[idx + 1] or None - if next_line is None or _starts_till(next_line, content) < 0: - body.insert(idx+1, _set_line_indent(body[idx], _set_line_eol(body[idx], content), indent)) + body = _insert_line_after(idx, body, content, indent) else: raise CommandExecutionError("Wrong conditions? " @@ -2005,7 +2013,12 @@ def line(path, content=None, match=None, mode=None, location=None, "where to put it before and/or after.") if body: - body.append(body.pop().rstrip('\r\n')) + for idx, line in enumerate(body): + if not _get_eol(line) and idx+1 < len(body): + prev = idx and idx-1 or 1 + body[idx] = _set_line_eol(body[prev], line) + if '' == body[-1]: + body.pop() changed = body_before != hashlib.sha256(salt.utils.stringutils.to_bytes(''.join(body))).hexdigest() diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py index cc2fbfa4d3..a8c96706f1 100755 --- a/salt/modules/win_file.py +++ b/salt/modules/win_file.py @@ -59,7 +59,8 @@ from salt.modules.file import (check_hash, # pylint: disable=W0611 lstat, path_exists_glob, write, pardir, join, HASHES, HASHES_REVMAP, comment, uncomment, _add_flags, comment_line, _regex_to_static, _set_line_indent, apply_template_on_contents, dirname, basename, - list_backups_dir, _assert_occurrence, _starts_till, _set_line_eol, _get_eol) + list_backups_dir, _assert_occurrence, _starts_till, _set_line_eol, _get_eol, + _insert_line_after, _insert_line_before) from salt.modules.file import normpath as normpath_ from salt.utils.functools import namespaced_function as _namespaced_function @@ -116,8 +117,9 @@ def __virtual__(): global blockreplace, prepend, seek_read, seek_write, rename, lstat global write, pardir, join, _add_flags, apply_template_on_contents global path_exists_glob, comment, uncomment, _mkstemp_copy - global _regex_to_static, _get_line_indent, dirname, basename + global _regex_to_static, _set_line_indent, dirname, basename global list_backups_dir, normpath_, _assert_occurrence, _starts_till + global _insert_line_before, _insert_line_after, _set_line_eol, _get_eol replace = _namespaced_function(replace, globals()) search = _namespaced_function(search, globals()) @@ -176,6 +178,8 @@ def __virtual__(): _set_line_indent = _namespaced_function(_set_line_indent, globals()) _set_line_eol = _namespaced_function(_set_line_eol, globals()) _get_eol = _namespaced_function(_get_eol, globals()) + _insert_line_after = _namespaced_function(_insert_line_after, globals()) + _insert_line_before = _namespaced_function(_insert_line_before, globals()) _mkstemp_copy = _namespaced_function(_mkstemp_copy, globals()) _add_flags = _namespaced_function(_add_flags, globals()) apply_template_on_contents = _namespaced_function(apply_template_on_contents, globals()) diff --git a/tests/unit/modules/test_file.py b/tests/unit/modules/test_file.py index 3f1333416a..b7240afcb9 100644 --- a/tests/unit/modules/test_file.py +++ b/tests/unit/modules/test_file.py @@ -1382,6 +1382,33 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): write_content = handles[0].write.call_args_list[0][0][0] assert write_content == file_modified, write_content + @patch('os.path.realpath', MagicMock()) + @patch('os.path.isfile', MagicMock(return_value=True)) + @patch('os.stat', MagicMock()) + def test_line_insert_ensure_before_first_line(self): + ''' + Test for file.line for insertion ensuring the line is before first line + :return: + ''' + cfg_content = '#!/bin/bash' + file_content = os.linesep.join([ + '/etc/init.d/someservice restart', + 'exit 0' + ]) + file_modified = os.linesep.join([ + cfg_content, + '/etc/init.d/someservice restart', + 'exit 0' + ]) + files_fopen = mock_open(read_data=file_content) + with patch('salt.utils.files.fopen', files_fopen): + atomic_opener = mock_open() + with patch('salt.utils.atomicfile.atomic_open', atomic_opener): + filemod.line('foo', content=cfg_content, before='/etc/init.d/someservice restart', mode='ensure') + self.assertEqual(len(atomic_opener().write.call_args_list), 1) + self.assertEqual(atomic_opener().write.call_args_list[0][0][0], + file_modified) + @with_tempfile() def test_line_insert_ensure_beforeafter_twolines(self, name): ''' From f29815b49b3c1c198bdb7e8d2616d9648b80e9f3 Mon Sep 17 00:00:00 2001 From: Jerzy Drozdz Date: Tue, 6 Mar 2018 22:46:59 +0100 Subject: [PATCH 11/26] Fixed file permissions --- salt/modules/file.py | 0 salt/modules/win_file.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 salt/modules/file.py mode change 100755 => 100644 salt/modules/win_file.py diff --git a/salt/modules/file.py b/salt/modules/file.py old mode 100755 new mode 100644 diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py old mode 100755 new mode 100644 From d3e8679e057e69a0b307ba1f912f818c734c63c7 Mon Sep 17 00:00:00 2001 From: Jerzy Drozdz Date: Thu, 8 Mar 2018 19:49:26 +0100 Subject: [PATCH 12/26] Removed regex compilation --- salt/modules/file.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index fd127dbb74..f8bae61000 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1763,8 +1763,7 @@ def _set_line_indent(src, line, indent): def _get_eol(line): - compiled = re.compile('((? Date: Mon, 21 May 2018 22:07:18 +0200 Subject: [PATCH 13/26] Comprehensions converting to unicode replaced by salt.utils.data.decode_list --- salt/modules/file.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index f8bae61000..1dd9289b17 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -61,6 +61,7 @@ import salt.utils.stringutils import salt.utils.templates import salt.utils.url import salt.utils.user +import salt.utils.data from salt.exceptions import CommandExecutionError, MinionError, SaltInvocationError, get_error_message as _get_error_message from salt.utils.files import HASHES, HASHES_REVMAP @@ -1919,7 +1920,7 @@ def line(path, content=None, match=None, mode=None, location=None, match = content with salt.utils.files.fopen(path, mode='r') as fp_: - body = [salt.utils.stringutils.to_unicode(line) for line in fp_.readlines()] + body = salt.utils.data.decode_list(fp_.readlines()) body_before = hashlib.sha256(salt.utils.stringutils.to_bytes(''.join(body))).hexdigest() if body and _get_eol(body[-1]): body.append('') @@ -2033,8 +2034,7 @@ def line(path, content=None, match=None, mode=None, location=None, if changed: if show_changes: with salt.utils.files.fopen(path, 'r') as fp_: - path_content = [salt.utils.stringutils.to_unicode(x) - for x in fp_.read().splitlines(True)] + path_content = salt.utils.data.decode_list(fp_.read().splitlines(True)) changes_diff = ''.join(difflib.unified_diff( path_content, body )) From adfa3aed0d6af5e6e2ade0fd03996912930fc57a Mon Sep 17 00:00:00 2001 From: Jerzy Drozdz Date: Mon, 21 May 2018 22:45:22 +0200 Subject: [PATCH 14/26] Empty match on delete or replace not causing IndexError exception --- salt/modules/file.py | 4 ++-- tests/unit/modules/test_file.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 1dd9289b17..f50ef69491 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1932,9 +1932,9 @@ def line(path, content=None, match=None, mode=None, location=None, if os.stat(path).st_size == 0 and mode in ('delete', 'replace'): log.warning('Cannot find text to {0}. File \'{1}\' is empty.'.format(mode, path)) body = [] - elif mode == 'delete': + elif mode == 'delete' and match: body = [line for line in body if line != match[0]] - elif mode == 'replace': + elif mode == 'replace' and match: body = [(_set_line_indent(file_line, _set_line_eol(file_line, content), indent) if (file_line == match[0] and not file_line.strip() == content) else file_line) for file_line in body] diff --git a/tests/unit/modules/test_file.py b/tests/unit/modules/test_file.py index b7240afcb9..8f510e03eb 100644 --- a/tests/unit/modules/test_file.py +++ b/tests/unit/modules/test_file.py @@ -940,6 +940,29 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): self.assertIn('Cannot find text to {0}'.format(mode), _log.warning.call_args_list[0][0][0]) + @patch('os.path.realpath', MagicMock()) + @patch('os.path.isfile', MagicMock(return_value=True)) + @patch('os.stat', MagicMock()) + def test_line_delete_no_match(self): + ''' + Tests that when calling file.line with ``mode=delete``, + with not matching pattern to delete returns False + :return: + ''' + file_content = os.linesep.join([ + 'file_roots:', + ' base:', + ' - /srv/salt', + ' - /srv/custom' + ]) + match = 'not matching' + for mode in ['delete', 'replace']: + files_fopen = mock_open(read_data=file_content) + with patch('salt.utils.files.fopen', files_fopen): + atomic_opener = mock_open() + with patch('salt.utils.atomicfile.atomic_open', atomic_opener): + self.assertFalse(filemod.line('foo', content='foo', match=match, mode=mode)) + @patch('os.path.realpath', MagicMock()) @patch('os.path.isfile', MagicMock(return_value=True)) def test_line_modecheck_failure(self): From f3517a1852776b4075a7f93817ec9d1f8cdd158f Mon Sep 17 00:00:00 2001 From: Jerzy Drozdz Date: Tue, 22 May 2018 21:04:38 +0200 Subject: [PATCH 15/26] List comprehension replaced --- salt/modules/file.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index f50ef69491..cee5799ff4 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1935,9 +1935,9 @@ def line(path, content=None, match=None, mode=None, location=None, elif mode == 'delete' and match: body = [line for line in body if line != match[0]] elif mode == 'replace' and match: - body = [(_set_line_indent(file_line, _set_line_eol(file_line, content), indent) - if (file_line == match[0] and not file_line.strip() == content) else file_line) - for file_line in body] + idx = body.index(match[0]) + file_line = body.pop(idx) + body.insert(idx, _set_line_indent(file_line, content, indent)) elif mode == 'insert': if not location and not before and not after: raise CommandExecutionError('On insert must be defined either "location" or "before/after" conditions.') From b9ddd53b0418a08b7bceb2725aa6670a3d9d88b9 Mon Sep 17 00:00:00 2001 From: Jerzy Drozdz Date: Tue, 22 May 2018 21:28:03 +0200 Subject: [PATCH 16/26] Added comments --- salt/modules/file.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/modules/file.py b/salt/modules/file.py index cee5799ff4..64a68741fb 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -1922,6 +1922,8 @@ def line(path, content=None, match=None, mode=None, location=None, with salt.utils.files.fopen(path, mode='r') as fp_: body = salt.utils.data.decode_list(fp_.readlines()) body_before = hashlib.sha256(salt.utils.stringutils.to_bytes(''.join(body))).hexdigest() + # Add empty line at the end if last line ends with eol. + # Allows simpler code if body and _get_eol(body[-1]): body.append('') @@ -2017,6 +2019,7 @@ def line(path, content=None, match=None, mode=None, location=None, if not _get_eol(line) and idx+1 < len(body): prev = idx and idx-1 or 1 body[idx] = _set_line_eol(body[prev], line) + # We do not need empty line at the end anymore if '' == body[-1]: body.pop() From e41f4922a0b6c0318ee5dc1c75e5eadc736387ba Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 12 Jul 2018 08:40:11 -0500 Subject: [PATCH 17/26] fix git.cloned doc versionadded Closes #48542 --- salt/states/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/git.py b/salt/states/git.py index e42305693a..b053b771eb 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -2682,7 +2682,7 @@ def cloned(name, https_pass=None, output_encoding=None): ''' - .. versionadded:: 2018.3.2,Fluorine + .. versionadded:: 2018.3.3, Fluorine Ensure that a repository has been cloned to the specified target directory. If not, clone that repository. No fetches will be performed once cloned. From 475f075d8effb95c9767c0c6b1471038f0059e8d Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 27 Jun 2018 10:24:38 -0400 Subject: [PATCH 18/26] Handle list of lines instead of strings in file.line func In `Fluorine`, in order to better handle line endings, the "body" variable has been changed to a list of strings. Therefore, we need to handle encoding or decoding the elements in the list, rather than using the `stringutils` functions. --- salt/modules/file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index 64a68741fb..ef8f5099d7 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -2047,10 +2047,10 @@ def line(path, content=None, match=None, mode=None, location=None, # Make sure we match the file mode from salt.utils.files.fopen if six.PY2 and salt.utils.platform.is_windows(): mode = 'wb' - body = salt.utils.stringutils.to_bytes(body) + body = salt.utils.data.encode_list(body) else: mode = 'w' - body = salt.utils.stringutils.to_str(body) + body = salt.utils.data.decode_list(body, to_str=True) fh_ = salt.utils.atomicfile.atomic_open(path, mode) fh_.write(''.join(body)) finally: From d8920cb61f84f6dc19d3444a4d6fa952b0426650 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 27 Jun 2018 10:51:29 -0400 Subject: [PATCH 19/26] Update `test_line_insert_ensure_before_first_line` to use new mock_open methodologies --- tests/unit/modules/test_file.py | 63 +++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/tests/unit/modules/test_file.py b/tests/unit/modules/test_file.py index 8f510e03eb..098aa2fc09 100644 --- a/tests/unit/modules/test_file.py +++ b/tests/unit/modules/test_file.py @@ -1369,6 +1369,42 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): write_content = handles[0].write.call_args_list[0][0][0] assert write_content == file_modified, write_content + @with_tempfile() + def test_line_insert_ensure_before_first_line(self, name): + ''' + Test for file.line for insertion ensuring the line is before first line + :return: + ''' + cfg_content = '#!/bin/bash' + file_content = os.linesep.join([ + '/etc/init.d/someservice restart', + 'exit 0' + ]) + file_modified = os.linesep.join([ + cfg_content, + '/etc/init.d/someservice restart', + 'exit 0' + ]) + + isfile_mock = MagicMock(side_effect=lambda x: True if x == name else DEFAULT) + with patch('os.path.isfile', isfile_mock), \ + patch('os.stat', MagicMock(return_value=DummyStat())), \ + patch('salt.utils.files.fopen', + mock_open(read_data=file_content)), \ + patch('salt.utils.atomicfile.atomic_open', + mock_open()) as atomic_open_mock: + filemod.line('foo', content=cfg_content, before='/etc/init.d/someservice restart', mode='ensure') + handles = atomic_open_mock.filehandles[name] + # We should only have opened the file once + open_count = len(handles) + assert open_count == 1, open_count + # We should only have invoked .write() once... + write_count = len(handles[0].write.call_args_list) + assert write_count == 1, write_count + # ... with the updated content + write_content = handles[0].write.call_args_list[0][0][0] + assert write_content == file_modified, write_content + @with_tempfile() def test_line_insert_ensure_after(self, name): ''' @@ -1405,33 +1441,6 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): write_content = handles[0].write.call_args_list[0][0][0] assert write_content == file_modified, write_content - @patch('os.path.realpath', MagicMock()) - @patch('os.path.isfile', MagicMock(return_value=True)) - @patch('os.stat', MagicMock()) - def test_line_insert_ensure_before_first_line(self): - ''' - Test for file.line for insertion ensuring the line is before first line - :return: - ''' - cfg_content = '#!/bin/bash' - file_content = os.linesep.join([ - '/etc/init.d/someservice restart', - 'exit 0' - ]) - file_modified = os.linesep.join([ - cfg_content, - '/etc/init.d/someservice restart', - 'exit 0' - ]) - files_fopen = mock_open(read_data=file_content) - with patch('salt.utils.files.fopen', files_fopen): - atomic_opener = mock_open() - with patch('salt.utils.atomicfile.atomic_open', atomic_opener): - filemod.line('foo', content=cfg_content, before='/etc/init.d/someservice restart', mode='ensure') - self.assertEqual(len(atomic_opener().write.call_args_list), 1) - self.assertEqual(atomic_opener().write.call_args_list[0][0][0], - file_modified) - @with_tempfile() def test_line_insert_ensure_beforeafter_twolines(self, name): ''' From d0b6e82ecb0bf6cb3f595def78e6ce7d59b6644b Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 28 Jun 2018 12:07:38 -0400 Subject: [PATCH 20/26] Add 'name' as the passed in filepath for test_line_insert_ensure_before_first_line test With 'foo', the test fails with an unknown file KeyError - we need to use the "name" variable instead. Fixes the test_line_insert_ensure_before_first_line test. --- tests/unit/modules/test_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/modules/test_file.py b/tests/unit/modules/test_file.py index 098aa2fc09..4cfd1b11af 100644 --- a/tests/unit/modules/test_file.py +++ b/tests/unit/modules/test_file.py @@ -1393,7 +1393,7 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): mock_open(read_data=file_content)), \ patch('salt.utils.atomicfile.atomic_open', mock_open()) as atomic_open_mock: - filemod.line('foo', content=cfg_content, before='/etc/init.d/someservice restart', mode='ensure') + filemod.line(name, content=cfg_content, before='/etc/init.d/someservice restart', mode='ensure') handles = atomic_open_mock.filehandles[name] # We should only have opened the file once open_count = len(handles) From 5d38aa8b3329d0a9870f78a78e74332c917d6929 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 28 Jun 2018 12:46:29 -0400 Subject: [PATCH 21/26] Update file.line to use `writelines` instead of joining --- salt/modules/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index ef8f5099d7..0fc8393d5b 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -2052,7 +2052,7 @@ def line(path, content=None, match=None, mode=None, location=None, mode = 'w' body = salt.utils.data.decode_list(body, to_str=True) fh_ = salt.utils.atomicfile.atomic_open(path, mode) - fh_.write(''.join(body)) + fh_.writelines(body) finally: if fh_: fh_.close() From 5e2e2a63fa0f94fd567c7fe4a7369c1171cadfa2 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 28 Jun 2018 15:41:43 -0400 Subject: [PATCH 22/26] Update file unit tests to handle "writelines" change Since file.line is now using "writelines()" instead of "write()", we need to handle the unittests accordingly. --- tests/unit/modules/test_file.py | 167 +++++++++++++++++++------------- 1 file changed, 100 insertions(+), 67 deletions(-) diff --git a/tests/unit/modules/test_file.py b/tests/unit/modules/test_file.py index 4cfd1b11af..d413a0f999 100644 --- a/tests/unit/modules/test_file.py +++ b/tests/unit/modules/test_file.py @@ -23,6 +23,7 @@ except ImportError: from salt.ext import six import salt.config import salt.loader +import salt.utils.data import salt.utils.files import salt.utils.platform import salt.utils.stringutils @@ -921,6 +922,19 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): } } + @staticmethod + def _get_body(content): + ''' + The body is written as bytestrings or strings depending on platform. + This func accepts a string of content and returns the appropriate list + of strings back. + ''' + ret = content.splitlines(True) + if six.PY2 and salt.utils.platform.is_windows(): + return salt.utils.data.encode_list(ret) + else: + return salt.utils.data.decode_list(ret, to_str=True) + @patch('os.path.realpath', MagicMock()) @patch('os.path.isfile', MagicMock(return_value=True)) def test_delete_line_in_empty_file(self): @@ -1049,12 +1063,13 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): # We should only have opened the file once open_count = len(handles) assert open_count == 1, open_count - # We should only have invoked .write() once... - write_count = len(handles[0].write.call_args_list) - assert write_count == 1, write_count + # We should only have invoked .writelines() once... + writelines_content = handles[0].writelines_calls + writelines_count = len(writelines_content) + assert writelines_count == 1, writelines_count # ... with the updated content - write_content = handles[0].write.call_args_list[0][0][0] - assert write_content == file_modified, write_content + expected = self._get_body(file_modified) + assert writelines_content[0] == expected, (writelines_content[0], expected) @with_tempfile() def test_line_insert_after_pattern(self, name): @@ -1099,12 +1114,17 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): # We should only have opened the file once open_count = len(handles) assert open_count == 1, open_count - # We should only have invoked .write() once... - write_count = len(handles[0].write.call_args_list) - assert write_count == 1, write_count + # We should only have invoked .writelines() once... + writelines_content = handles[0].writelines_calls + writelines_count = len(writelines_content) + assert writelines_count == 1, writelines_count # ... with the updated content - write_content = handles[0].write.call_args_list[0][0][0] - assert write_content == file_modified, write_content + expected = self._get_body(file_modified) + # We passed cfg_content with a newline in the middle, so it + # will be written as two lines in the same element of the list + # passed to .writelines() + expected[3] = expected[3] + expected.pop(4) + assert writelines_content[0] == expected, (writelines_content[0], expected) @with_tempfile() def test_line_insert_multi_line_content_after_unicode(self, name): @@ -1114,8 +1134,10 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): See issue #48113 :return: ''' - file_content = ("This is a line\nThis is another line") - file_modified = salt.utils.stringutils.to_str("This is a line\nThis is another line\nThis is a line with unicode Ŷ") + file_content = 'This is a line\nThis is another line' + file_modified = salt.utils.stringutils.to_str('This is a line\n' + 'This is another line\n' + 'This is a line with unicode Ŷ') cfg_content = "This is a line with unicode Ŷ" isfile_mock = MagicMock(side_effect=lambda x: True if x == name else DEFAULT) for after_line in ['This is another line']: @@ -1130,12 +1152,13 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): # We should only have opened the file once open_count = len(handles) assert open_count == 1, open_count - # We should only have invoked .write() once... - write_count = len(handles[0].write.call_args_list) - assert write_count == 1, write_count + # We should only have invoked .writelines() once... + writelines_content = handles[0].writelines_calls + writelines_count = len(writelines_content) + assert writelines_count == 1, writelines_count # ... with the updated content - write_content = handles[0].write.call_args_list[0][0][0] - assert write_content == file_modified, write_content + expected = self._get_body(file_modified) + assert writelines_content[0] == expected, (writelines_content[0], expected) @with_tempfile() def test_line_insert_before(self, name): @@ -1173,12 +1196,13 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): # We should only have opened the file once open_count = len(handles) assert open_count == 1, open_count - # We should only have invoked .write() once... - write_count = len(handles[0].write.call_args_list) - assert write_count == 1, write_count + # We should only have invoked .writelines() once... + writelines_content = handles[0].writelines_calls + writelines_count = len(writelines_content) + assert writelines_count == 1, writelines_count # ... with the updated content - write_content = handles[0].write.call_args_list[0][0][0] - assert write_content == file_modified, write_content + expected = self._get_body(file_modified) + assert writelines_content[0] == expected, (writelines_content[0], expected) @patch('os.path.realpath', MagicMock()) @patch('os.path.isfile', MagicMock(return_value=True)) @@ -1244,12 +1268,13 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): # We should only have opened the file once open_count = len(handles) assert open_count == 1, open_count - # We should only have invoked .write() once... - write_count = len(handles[0].write.call_args_list) - assert write_count == 1, write_count + # We should only have invoked .writelines() once... + writelines_content = handles[0].writelines_calls + writelines_count = len(writelines_content) + assert writelines_count == 1, writelines_count # ... with the updated content - write_content = handles[0].write.call_args_list[0][0][0] - assert write_content == file_modified, write_content + expected = self._get_body(file_modified) + assert writelines_content[0] == expected, (writelines_content[0], expected) @with_tempfile() def test_line_insert_start(self, name): @@ -1284,12 +1309,13 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): # We should only have opened the file once open_count = len(handles) assert open_count == 1, open_count - # We should only have invoked .write() once... - write_count = len(handles[0].write.call_args_list) - assert write_count == 1, write_count + # We should only have invoked .writelines() once... + writelines_content = handles[0].writelines_calls + writelines_count = len(writelines_content) + assert writelines_count == 1, writelines_count # ... with the updated content - write_content = handles[0].write.call_args_list[0][0][0] - assert write_content == file_modified, write_content + expected = self._get_body(file_modified) + assert writelines_content[0] == expected, (writelines_content[0], expected) @with_tempfile() def test_line_insert_end(self, name): @@ -1324,12 +1350,13 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): # We should only have opened the file once open_count = len(handles) assert open_count == 1, open_count - # We should only have invoked .write() once... - write_count = len(handles[0].write.call_args_list) - assert write_count == 1, write_count + # We should only have invoked .writelines() once... + writelines_content = handles[0].writelines_calls + writelines_count = len(writelines_content) + assert writelines_count == 1, writelines_count # ... with the updated content - write_content = handles[0].write.call_args_list[0][0][0] - assert write_content == file_modified, write_content + expected = self._get_body(file_modified) + assert writelines_content[0] == expected, (writelines_content[0], expected) @with_tempfile() def test_line_insert_ensure_before(self, name): @@ -1362,12 +1389,13 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): # We should only have opened the file once open_count = len(handles) assert open_count == 1, open_count - # We should only have invoked .write() once... - write_count = len(handles[0].write.call_args_list) - assert write_count == 1, write_count + # We should only have invoked .writelines() once... + writelines_content = handles[0].writelines_calls + writelines_count = len(writelines_content) + assert writelines_count == 1, writelines_count # ... with the updated content - write_content = handles[0].write.call_args_list[0][0][0] - assert write_content == file_modified, write_content + expected = self._get_body(file_modified) + assert writelines_content[0] == expected, (writelines_content[0], expected) @with_tempfile() def test_line_insert_ensure_before_first_line(self, name): @@ -1398,12 +1426,13 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): # We should only have opened the file once open_count = len(handles) assert open_count == 1, open_count - # We should only have invoked .write() once... - write_count = len(handles[0].write.call_args_list) - assert write_count == 1, write_count + # We should only have invoked .writelines() once... + writelines_content = handles[0].writelines_calls + writelines_count = len(writelines_content) + assert writelines_count == 1, writelines_count # ... with the updated content - write_content = handles[0].write.call_args_list[0][0][0] - assert write_content == file_modified, write_content + expected = self._get_body(file_modified) + assert writelines_content[0] == expected, (writelines_content[0], expected) @with_tempfile() def test_line_insert_ensure_after(self, name): @@ -1434,12 +1463,13 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): # We should only have opened the file once open_count = len(handles) assert open_count == 1, open_count - # We should only have invoked .write() once... - write_count = len(handles[0].write.call_args_list) - assert write_count == 1, write_count + # We should only have invoked .writelines() once... + writelines_content = handles[0].writelines_calls + writelines_count = len(writelines_content) + assert writelines_count == 1, writelines_count # ... with the updated content - write_content = handles[0].write.call_args_list[0][0][0] - assert write_content == file_modified, write_content + expected = self._get_body(file_modified) + assert writelines_content[0] == expected, (writelines_content[0], expected) @with_tempfile() def test_line_insert_ensure_beforeafter_twolines(self, name): @@ -1470,12 +1500,13 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): # We should only have opened the file once open_count = len(handles) assert open_count == 1, open_count - # We should only have invoked .write() once... - write_count = len(handles[0].write.call_args_list) - assert write_count == 1, write_count + # We should only have invoked .writelines() once... + writelines_content = handles[0].writelines_calls + writelines_count = len(writelines_content) + assert writelines_count == 1, writelines_count # ... with the updated content - write_content = handles[0].write.call_args_list[0][0][0] - assert write_content == file_modified, write_content + expected = self._get_body(file_modified) + assert writelines_content[0] == expected, (writelines_content[0], expected) @with_tempfile() def test_line_insert_ensure_beforeafter_twolines_exists(self, name): @@ -1564,12 +1595,13 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): # We should only have opened the file once open_count = len(handles) assert open_count == 1, open_count - # We should only have invoked .write() once... - write_count = len(handles[0].write.call_args_list) - assert write_count == 1, write_count + # We should only have invoked .writelines() once... + writelines_content = handles[0].writelines_calls + writelines_count = len(writelines_content) + assert writelines_count == 1, writelines_count # ... with the updated content - write_content = handles[0].write.call_args_list[0][0][0] - assert write_content == file_modified, write_content + expected = self._get_body(file_modified) + assert writelines_content[0] == expected, (writelines_content[0], expected) @with_tempfile() def test_line_replace(self, name): @@ -1604,12 +1636,13 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): # We should only have opened the file once open_count = len(handles) assert open_count == 1, open_count - # We should only have invoked .write() once... - write_count = len(handles[0].write.call_args_list) - assert write_count == 1, write_count + # We should only have invoked .writelines() once... + writelines_content = handles[0].writelines_calls + writelines_count = len(writelines_content) + assert writelines_count == 1, writelines_count # ... with the updated content - write_content = handles[0].write.call_args_list[0][0][0] - assert write_content == file_modified, write_content + expected = self._get_body(file_modified) + assert writelines_content[0] == expected, (writelines_content[0], expected) class FileBasicsTestCase(TestCase, LoaderModuleMockMixin): From 8bc71688ea026a78685af3efa9d9130a8cb5a745 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 12 Jul 2018 11:24:44 -0400 Subject: [PATCH 23/26] Remove /sr.* pattern from test /sr.* pattern should raise exception --- tests/unit/modules/test_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/modules/test_file.py b/tests/unit/modules/test_file.py index d413a0f999..916991b30f 100644 --- a/tests/unit/modules/test_file.py +++ b/tests/unit/modules/test_file.py @@ -1184,7 +1184,7 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): cfg_content = '- /srv/custom' isfile_mock = MagicMock(side_effect=lambda x: True if x == name else DEFAULT) - for before_line in ['/srv/salt', '/srv/sa.*t', '/sr.*']: + for before_line in ['/srv/salt', '/srv/sa.*t']: with patch('os.path.isfile', isfile_mock), \ patch('os.stat', MagicMock(return_value=DummyStat())), \ patch('salt.utils.files.fopen', From 17e69382d5352e0f0e12aecafd66f3921096da67 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Fri, 13 Jul 2018 12:35:41 -0400 Subject: [PATCH 24/26] Fix python2 syslog returner expecting string bytes not unicode --- salt/returners/syslog_return.py | 12 ++---- tests/unit/returners/test_syslog_return.py | 44 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 tests/unit/returners/test_syslog_return.py diff --git a/salt/returners/syslog_return.py b/salt/returners/syslog_return.py index 736df2fd6f..08014259b8 100644 --- a/salt/returners/syslog_return.py +++ b/salt/returners/syslog_return.py @@ -192,14 +192,10 @@ def returner(ret): logoption = logoption | getattr(syslog, opt) # Open syslog correctly based on options and tag - try: - if 'tag' in _options: - syslog.openlog(ident=_options['tag'], logoption=logoption) - else: - syslog.openlog(logoption=logoption) - except TypeError: - # Python 2.6 syslog.openlog does not accept keyword args - syslog.openlog(_options.get('tag', 'salt-minion'), logoption) + if 'tag' in _options: + syslog.openlog(ident=salt.utils.stringutils.to_str(_options['tag']), logoption=logoption) + else: + syslog.openlog(logoption=logoption) # Send log of given level and facility syslog.syslog(facility | level, salt.utils.json.dumps(ret)) diff --git a/tests/unit/returners/test_syslog_return.py b/tests/unit/returners/test_syslog_return.py new file mode 100644 index 0000000000..408f0f817a --- /dev/null +++ b/tests/unit/returners/test_syslog_return.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Megan Wilhite (mwilhite@saltstack.com)` + + + tests.unit.returners.test_syslog_return + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +''' + +# Import Python libs +from __future__ import absolute_import, print_function, unicode_literals + +# Import Salt Testing libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch + +# Import salt libs +import salt.returners.syslog_return as syslog + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class SyslogReturnerTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test Syslog returner + ''' + def setup_loader_modules(self): + return {syslog: {}} + + def test_syslog_returner_unicode(self): + ''' + test syslog returner with unicode + ''' + ret = {'fun_args': [], 'jid': '20180713160901624786', 'return': True, + 'retcode': 0, 'success': True, 'fun': 'test.ping', 'id': '02e10e971a30'} + opts = {u'level': u'LOG_INFO', u'options': [], + u'facility': u'LOG_USER', 'tag': u'salt-minion'} + + with patch('salt.returners.syslog_return._get_options', + MagicMock(return_value=opts)): + try: + syslog.returner(ret) + except Exception as e: + self.fail('syslog.returner() failed with exception: {0}'.format(e)) From d8288a0ffc03e7db0ba73cd4dd842874c80f2200 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Fri, 13 Jul 2018 12:39:09 -0400 Subject: [PATCH 25/26] add unicode type for syslog test for tag --- tests/unit/returners/test_syslog_return.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/returners/test_syslog_return.py b/tests/unit/returners/test_syslog_return.py index 408f0f817a..cc28182179 100644 --- a/tests/unit/returners/test_syslog_return.py +++ b/tests/unit/returners/test_syslog_return.py @@ -34,7 +34,7 @@ class SyslogReturnerTestCase(TestCase, LoaderModuleMockMixin): ret = {'fun_args': [], 'jid': '20180713160901624786', 'return': True, 'retcode': 0, 'success': True, 'fun': 'test.ping', 'id': '02e10e971a30'} opts = {u'level': u'LOG_INFO', u'options': [], - u'facility': u'LOG_USER', 'tag': u'salt-minion'} + u'facility': u'LOG_USER', u'tag': u'salt-minion'} with patch('salt.returners.syslog_return._get_options', MagicMock(return_value=opts)): From 42fd3d5f675706dc184d31bfc7a10574f1025d81 Mon Sep 17 00:00:00 2001 From: Andrey Tuzhilin Date: Fri, 13 Jul 2018 23:51:58 +0300 Subject: [PATCH 26/26] replaced meta tag --- salt/runners/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/runners/vault.py b/salt/runners/vault.py index c2f30068ae..5f7dcb4252 100644 --- a/salt/runners/vault.py +++ b/salt/runners/vault.py @@ -72,7 +72,7 @@ def generate_token(minion_id, signature, impersonated_by_master=False): payload = { 'policies': _get_policies(minion_id, config), 'num_uses': 1, - 'metadata': audit_data + 'meta': audit_data } if payload['policies'] == []: