mirror of
https://github.com/valitydev/salt.git
synced 2024-11-07 08:58:59 +00:00
py3: fix salt/modules/file.py and unit tests
Most of the routines in the file execution module that operate on file contents assume that they are dealing with text files (line-oriented operations, regular expressions, etc). The exceptions are get_sum(), get_hash(), and check_hash(). Outside of these three routines, files are now opened in text mode.
This commit is contained in:
Michael Steed
parent
12be97ab47
commit
bd8538db97
@ -1319,8 +1319,8 @@ def line(path, content, match=None, mode=None, location=None,
|
||||
if before is None and after is None and not match:
|
||||
match = content
|
||||
|
||||
body = salt.utils.fopen(path, mode='rb').read()
|
||||
body_before = hashlib.sha256(body).hexdigest()
|
||||
body = salt.utils.fopen(path, mode='r').read()
|
||||
body_before = hashlib.sha256(salt.utils.to_bytes(body)).hexdigest()
|
||||
after = _regex_to_static(body, after)
|
||||
before = _regex_to_static(body, before)
|
||||
match = _regex_to_static(body, match)
|
||||
@ -1447,7 +1447,7 @@ def line(path, content, match=None, mode=None, location=None,
|
||||
"Unable to ensure line without knowing "
|
||||
"where to put it before and/or after.")
|
||||
|
||||
changed = body_before != hashlib.sha256(body).hexdigest()
|
||||
changed = body_before != hashlib.sha256(salt.utils.to_bytes(body)).hexdigest()
|
||||
|
||||
if backup and changed:
|
||||
try:
|
||||
@ -1460,10 +1460,10 @@ def line(path, content, match=None, mode=None, location=None,
|
||||
|
||||
if changed:
|
||||
if show_changes:
|
||||
changes_diff = ''.join(difflib.unified_diff(salt.utils.fopen(path, 'rb').readlines(), body.splitlines()))
|
||||
changes_diff = ''.join(difflib.unified_diff(salt.utils.fopen(path, 'r').readlines(), body.splitlines()))
|
||||
fh_ = None
|
||||
try:
|
||||
fh_ = salt.utils.atomicfile.atomic_open(path, 'wb')
|
||||
fh_ = salt.utils.atomicfile.atomic_open(path, 'w')
|
||||
fh_.write(body)
|
||||
finally:
|
||||
if fh_:
|
||||
@ -1656,7 +1656,7 @@ def replace(path,
|
||||
try:
|
||||
# Use a read-only handle to open the file
|
||||
with salt.utils.fopen(path,
|
||||
mode='rb',
|
||||
mode='r',
|
||||
buffering=bufsize) as r_file:
|
||||
for line in r_file:
|
||||
result, nrepl = re.subn(cpattern, repl, line, count)
|
||||
@ -1700,12 +1700,12 @@ def replace(path,
|
||||
try:
|
||||
# Open the file in write mode
|
||||
with salt.utils.fopen(path,
|
||||
mode='wb',
|
||||
mode='w',
|
||||
buffering=bufsize) as w_file:
|
||||
try:
|
||||
# Open the temp file in read mode
|
||||
with salt.utils.fopen(temp_file,
|
||||
mode='rb',
|
||||
mode='r',
|
||||
buffering=bufsize) as r_file:
|
||||
for line in r_file:
|
||||
result, nrepl = re.subn(cpattern, repl,
|
||||
@ -1746,7 +1746,7 @@ def replace(path,
|
||||
raise CommandExecutionError("Exception: {0}".format(exc))
|
||||
# write new content in the file while avoiding partial reads
|
||||
try:
|
||||
fh_ = salt.utils.atomicfile.atomic_open(path, 'wb')
|
||||
fh_ = salt.utils.atomicfile.atomic_open(path, 'w')
|
||||
for line in new_file:
|
||||
fh_.write(line)
|
||||
finally:
|
||||
@ -1898,7 +1898,7 @@ def blockreplace(path,
|
||||
try:
|
||||
fi_file = fileinput.input(path,
|
||||
inplace=False, backup=False,
|
||||
bufsize=1, mode='rb')
|
||||
bufsize=1, mode='r')
|
||||
for line in fi_file:
|
||||
|
||||
result = line
|
||||
@ -1984,7 +1984,7 @@ def blockreplace(path,
|
||||
|
||||
# write new content in the file while avoiding partial reads
|
||||
try:
|
||||
fh_ = salt.utils.atomicfile.atomic_open(path, 'wb')
|
||||
fh_ = salt.utils.atomicfile.atomic_open(path, 'w')
|
||||
for line in new_file:
|
||||
fh_.write(line)
|
||||
finally:
|
||||
@ -2250,23 +2250,25 @@ def append(path, *args, **kwargs):
|
||||
else:
|
||||
args = [kwargs['args']]
|
||||
|
||||
with salt.utils.fopen(path, "r+") as ofile:
|
||||
# Make sure we have a newline at the end of the file
|
||||
# Make sure we have a newline at the end of the file. Do this in binary
|
||||
# mode so SEEK_END with nonzero offset will work.
|
||||
with salt.utils.fopen(path, 'rb+') as ofile:
|
||||
linesep = salt.utils.to_bytes(os.linesep)
|
||||
try:
|
||||
ofile.seek(-1, os.SEEK_END)
|
||||
ofile.seek(-len(linesep), os.SEEK_END)
|
||||
except IOError as exc:
|
||||
if exc.errno == errno.EINVAL or exc.errno == errno.ESPIPE:
|
||||
if exc.errno in (errno.EINVAL, errno.ESPIPE):
|
||||
# Empty file, simply append lines at the beginning of the file
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
if ofile.read(1) != '\n':
|
||||
if ofile.read(len(linesep)) != linesep:
|
||||
ofile.seek(0, os.SEEK_END)
|
||||
ofile.write('\n')
|
||||
else:
|
||||
ofile.seek(0, os.SEEK_END)
|
||||
# Append lines
|
||||
ofile.write(linesep)
|
||||
# Append lines in text mode
|
||||
with salt.utils.fopen(path, 'r+') as ofile:
|
||||
ofile.seek(0, os.SEEK_END)
|
||||
for line in args:
|
||||
ofile.write('{0}\n'.format(line))
|
||||
|
||||
@ -2507,7 +2509,7 @@ def truncate(path, length):
|
||||
salt '*' file.truncate /path/to/file 512
|
||||
'''
|
||||
path = os.path.expanduser(path)
|
||||
with salt.utils.fopen(path, 'r+') as seek_fh:
|
||||
with salt.utils.fopen(path, 'rb+') as seek_fh:
|
||||
seek_fh.truncate(int(length))
|
||||
|
||||
|
||||
@ -3566,8 +3568,8 @@ def check_file_meta(
|
||||
changes['diff'] = bdiff
|
||||
else:
|
||||
with contextlib.nested(
|
||||
salt.utils.fopen(sfn, 'rb'),
|
||||
salt.utils.fopen(name, 'rb')) as (src, name_):
|
||||
salt.utils.fopen(sfn, 'r'),
|
||||
salt.utils.fopen(name, 'r')) as (src, name_):
|
||||
slines = src.readlines()
|
||||
nlines = name_.readlines()
|
||||
changes['diff'] = \
|
||||
@ -3582,8 +3584,8 @@ def check_file_meta(
|
||||
tmp_.write(str(contents))
|
||||
# Compare the static contents with the named file
|
||||
with contextlib.nested(
|
||||
salt.utils.fopen(tmp, 'rb'),
|
||||
salt.utils.fopen(name, 'rb')) as (src, name_):
|
||||
salt.utils.fopen(tmp, 'r'),
|
||||
salt.utils.fopen(name, 'r')) as (src, name_):
|
||||
slines = src.readlines()
|
||||
nlines = name_.readlines()
|
||||
if ''.join(nlines) != ''.join(slines):
|
||||
@ -3783,8 +3785,8 @@ def manage_file(name,
|
||||
ret['changes']['diff'] = bdiff
|
||||
else:
|
||||
with contextlib.nested(
|
||||
salt.utils.fopen(sfn, 'rb'),
|
||||
salt.utils.fopen(real_name, 'rb')) as (src, name_):
|
||||
salt.utils.fopen(sfn, 'r'),
|
||||
salt.utils.fopen(real_name, 'r')) as (src, name_):
|
||||
slines = src.readlines()
|
||||
nlines = name_.readlines()
|
||||
|
||||
@ -3811,8 +3813,8 @@ def manage_file(name,
|
||||
|
||||
# Compare contents of files to know if we need to replace
|
||||
with contextlib.nested(
|
||||
salt.utils.fopen(tmp, 'rb'),
|
||||
salt.utils.fopen(real_name, 'rb')) as (src, name_):
|
||||
salt.utils.fopen(tmp, 'r'),
|
||||
salt.utils.fopen(real_name, 'r')) as (src, name_):
|
||||
slines = src.readlines()
|
||||
nlines = name_.readlines()
|
||||
different = ''.join(slines) != ''.join(nlines)
|
||||
|
||||
@ -53,7 +53,7 @@ class FileReplaceTestCase(TestCase):
|
||||
''')
|
||||
|
||||
def setUp(self):
|
||||
self.tfile = tempfile.NamedTemporaryFile(delete=False)
|
||||
self.tfile = tempfile.NamedTemporaryFile(delete=False, mode='w+')
|
||||
self.tfile.write(self.MULTILINE_STRING)
|
||||
self.tfile.close()
|
||||
|
||||
@ -63,7 +63,7 @@ class FileReplaceTestCase(TestCase):
|
||||
def test_replace(self):
|
||||
filemod.replace(self.tfile.name, r'Etiam', 'Salticus', backup=False)
|
||||
|
||||
with salt.utils.fopen(self.tfile.name, 'rb') as fp:
|
||||
with salt.utils.fopen(self.tfile.name, 'r') as fp:
|
||||
self.assertIn('Salticus', fp.read())
|
||||
|
||||
def test_replace_append_if_not_found(self):
|
||||
@ -78,26 +78,26 @@ class FileReplaceTestCase(TestCase):
|
||||
base = 'foo=1\nbar=2'
|
||||
expected = '{base}\n{repl}\n'.format(base=base, **args)
|
||||
# File ending with a newline, no match
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile(mode='w+') as tfile:
|
||||
tfile.write(base + '\n')
|
||||
tfile.flush()
|
||||
filemod.replace(tfile.name, **args)
|
||||
with salt.utils.fopen(tfile.name) as tfile2:
|
||||
self.assertEqual(tfile2.read(), expected)
|
||||
# File not ending with a newline, no match
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile('w+') as tfile:
|
||||
tfile.write(base)
|
||||
tfile.flush()
|
||||
filemod.replace(tfile.name, **args)
|
||||
with salt.utils.fopen(tfile.name) as tfile2:
|
||||
self.assertEqual(tfile2.read(), expected)
|
||||
# A newline should not be added in empty files
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile('w+') as tfile:
|
||||
filemod.replace(tfile.name, **args)
|
||||
with salt.utils.fopen(tfile.name) as tfile2:
|
||||
self.assertEqual(tfile2.read(), args['repl'] + '\n')
|
||||
# Using not_found_content, rather than repl
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile('w+') as tfile:
|
||||
args['not_found_content'] = 'baz=3'
|
||||
expected = '{base}\n{not_found_content}\n'.format(base=base, **args)
|
||||
tfile.write(base)
|
||||
@ -106,7 +106,7 @@ class FileReplaceTestCase(TestCase):
|
||||
with salt.utils.fopen(tfile.name) as tfile2:
|
||||
self.assertEqual(tfile2.read(), expected)
|
||||
# not appending if matches
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile('w+') as tfile:
|
||||
base = 'foo=1\n#baz=42\nbar=2\n'
|
||||
expected = 'foo=1\nbaz=42\nbar=2\n'
|
||||
tfile.write(base)
|
||||
@ -203,7 +203,8 @@ class FileBlockReplaceTestCase(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tfile = tempfile.NamedTemporaryFile(delete=False,
|
||||
prefix='blockrepltmp')
|
||||
prefix='blockrepltmp',
|
||||
mode='w+')
|
||||
self.tfile.write(self.MULTILINE_STRING)
|
||||
self.tfile.close()
|
||||
manage_mode_mock = MagicMock()
|
||||
@ -224,7 +225,7 @@ class FileBlockReplaceTestCase(TestCase):
|
||||
new_multiline_content,
|
||||
backup=False)
|
||||
|
||||
with salt.utils.fopen(self.tfile.name, 'rb') as fp:
|
||||
with salt.utils.fopen(self.tfile.name, 'r') as fp:
|
||||
filecontent = fp.read()
|
||||
self.assertIn('#-- START BLOCK 1'
|
||||
+ "\n" + new_multiline_content
|
||||
@ -246,7 +247,7 @@ class FileBlockReplaceTestCase(TestCase):
|
||||
append_if_not_found=False,
|
||||
backup=False
|
||||
)
|
||||
with salt.utils.fopen(self.tfile.name, 'rb') as fp:
|
||||
with salt.utils.fopen(self.tfile.name, 'r') as fp:
|
||||
self.assertNotIn('#-- START BLOCK 2'
|
||||
+ "\n" + new_content + "\n"
|
||||
+ '#-- END BLOCK 2', fp.read())
|
||||
@ -258,7 +259,7 @@ class FileBlockReplaceTestCase(TestCase):
|
||||
backup=False,
|
||||
append_if_not_found=True)
|
||||
|
||||
with salt.utils.fopen(self.tfile.name, 'rb') as fp:
|
||||
with salt.utils.fopen(self.tfile.name, 'r') as fp:
|
||||
self.assertIn('#-- START BLOCK 2'
|
||||
+ "\n" + new_content
|
||||
+ "\n" + '#-- END BLOCK 2', fp.read())
|
||||
@ -278,21 +279,21 @@ class FileBlockReplaceTestCase(TestCase):
|
||||
block = '{marker_start}\n{content}\n{marker_end}\n'.format(**args)
|
||||
expected = base + '\n' + block
|
||||
# File ending with a newline
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile(mode='w+') as tfile:
|
||||
tfile.write(base + '\n')
|
||||
tfile.flush()
|
||||
filemod.blockreplace(tfile.name, **args)
|
||||
with salt.utils.fopen(tfile.name) as tfile2:
|
||||
self.assertEqual(tfile2.read(), expected)
|
||||
# File not ending with a newline
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile(mode='w+') as tfile:
|
||||
tfile.write(base)
|
||||
tfile.flush()
|
||||
filemod.blockreplace(tfile.name, **args)
|
||||
with salt.utils.fopen(tfile.name) as tfile2:
|
||||
self.assertEqual(tfile2.read(), expected)
|
||||
# A newline should not be added in empty files
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile(mode='w+') as tfile:
|
||||
filemod.blockreplace(tfile.name, **args)
|
||||
with salt.utils.fopen(tfile.name) as tfile2:
|
||||
self.assertEqual(tfile2.read(), block)
|
||||
@ -310,7 +311,7 @@ class FileBlockReplaceTestCase(TestCase):
|
||||
prepend_if_not_found=False,
|
||||
backup=False
|
||||
)
|
||||
with salt.utils.fopen(self.tfile.name, 'rb') as fp:
|
||||
with salt.utils.fopen(self.tfile.name, 'r') as fp:
|
||||
self.assertNotIn(
|
||||
'#-- START BLOCK 2' + "\n"
|
||||
+ new_content + "\n" + '#-- END BLOCK 2',
|
||||
@ -322,7 +323,7 @@ class FileBlockReplaceTestCase(TestCase):
|
||||
backup=False,
|
||||
prepend_if_not_found=True)
|
||||
|
||||
with salt.utils.fopen(self.tfile.name, 'rb') as fp:
|
||||
with salt.utils.fopen(self.tfile.name, 'r') as fp:
|
||||
self.assertTrue(
|
||||
fp.read().startswith(
|
||||
'#-- START BLOCK 2'
|
||||
@ -336,7 +337,7 @@ class FileBlockReplaceTestCase(TestCase):
|
||||
'new content 1',
|
||||
backup=False)
|
||||
|
||||
with salt.utils.fopen(self.tfile.name, 'rb') as fp:
|
||||
with salt.utils.fopen(self.tfile.name, 'r') as fp:
|
||||
filecontent = fp.read()
|
||||
self.assertIn('new content 1', filecontent)
|
||||
self.assertNotIn('to be removed', filecontent)
|
||||
@ -426,7 +427,7 @@ class FileBlockReplaceTestCase(TestCase):
|
||||
|
||||
class FileModuleTestCase(TestCase):
|
||||
def test_sed_limit_escaped(self):
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile(mode='w+') as tfile:
|
||||
tfile.write(SED_CONTENT)
|
||||
tfile.seek(0, 0)
|
||||
|
||||
@ -437,7 +438,7 @@ class FileModuleTestCase(TestCase):
|
||||
|
||||
filemod.sed(path, before, after, limit=limit)
|
||||
|
||||
with salt.utils.fopen(path, 'rb') as newfile:
|
||||
with salt.utils.fopen(path, 'r') as newfile:
|
||||
self.assertEqual(
|
||||
SED_CONTENT.replace(before, ''),
|
||||
newfile.read()
|
||||
@ -449,21 +450,21 @@ class FileModuleTestCase(TestCase):
|
||||
newlines at end of file.
|
||||
'''
|
||||
# File ending with a newline
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile(mode='w+') as tfile:
|
||||
tfile.write('foo\n')
|
||||
tfile.flush()
|
||||
filemod.append(tfile.name, 'bar')
|
||||
with salt.utils.fopen(tfile.name) as tfile2:
|
||||
self.assertEqual(tfile2.read(), 'foo\nbar\n')
|
||||
# File not ending with a newline
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile(mode='w+') as tfile:
|
||||
tfile.write('foo')
|
||||
tfile.flush()
|
||||
filemod.append(tfile.name, 'bar')
|
||||
with salt.utils.fopen(tfile.name) as tfile2:
|
||||
self.assertEqual(tfile2.read(), 'foo\nbar\n')
|
||||
# A newline should not be added in empty files
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile(mode='w+') as tfile:
|
||||
filemod.append(tfile.name, 'bar')
|
||||
with salt.utils.fopen(tfile.name) as tfile2:
|
||||
self.assertEqual(tfile2.read(), 'bar\n')
|
||||
@ -473,7 +474,7 @@ class FileModuleTestCase(TestCase):
|
||||
Check various hash file formats.
|
||||
'''
|
||||
# With file name
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile(mode='w+') as tfile:
|
||||
tfile.write('rc.conf ef6e82e4006dee563d98ada2a2a80a27\n')
|
||||
tfile.write(
|
||||
'ead48423703509d37c4a90e6a0d53e143b6fc268 example.tar.gz\n')
|
||||
@ -490,7 +491,7 @@ class FileModuleTestCase(TestCase):
|
||||
'hash_type': 'sha1'
|
||||
})
|
||||
# Solohash - no file name (Maven repo checksum file format)
|
||||
with tempfile.NamedTemporaryFile() as tfile:
|
||||
with tempfile.NamedTemporaryFile(mode='w+') as tfile:
|
||||
tfile.write('ead48423703509d37c4a90e6a0d53e143b6fc268\n')
|
||||
tfile.flush()
|
||||
result = filemod.extract_hash(tfile.name, '', '/testfile')
|
||||
|
||||
Loading…
Reference in New Issue
Block a user