mirror of
https://github.com/valitydev/salt.git
synced 2024-11-09 01:36:48 +00:00
3184168365
This PR is part of what will be an ongoing effort to use explicit unicode strings in Salt. Because Python 3 does not suport Python 2's raw unicode string syntax (i.e. `ur'\d+'`), we must use `salt.utils.locales.sdecode()` to ensure that the raw string is unicode. However, because of how `salt/utils/__init__.py` has evolved into the hulking monstrosity it is today, this means importing a large module in places where it is not needed, which could negatively impact performance. For this reason, this PR also breaks out some of the functions from `salt/utils/__init__.py` into new/existing modules under `salt/utils/`. The long term goal will be that the modules within this directory do not depend on importing `salt.utils`. A summary of the changes in this PR is as follows: * Moves the following functions from `salt.utils` to new locations (including a deprecation warning if invoked from `salt.utils`): `to_bytes`, `to_str`, `to_unicode`, `str_to_num`, `is_quoted`, `dequote`, `is_hex`, `is_bin_str`, `rand_string`, `contains_whitespace`, `clean_kwargs`, `invalid_kwargs`, `which`, `which_bin`, `path_join`, `shlex_split`, `rand_str`, `is_windows`, `is_proxy`, `is_linux`, `is_darwin`, `is_sunos`, `is_smartos`, `is_smartos_globalzone`, `is_smartos_zone`, `is_freebsd`, `is_netbsd`, `is_openbsd`, `is_aix` * Moves the functions already deprecated by @rallytime to the bottom of `salt/utils/__init__.py` for better organization, so we can keep the deprecated ones separate from the ones yet to be deprecated as we continue to break up `salt.utils` * Updates `salt/*.py` and all files under `salt/client/` to use explicit unicode string literals. * Gets rid of implicit imports of `salt.utils` (e.g. `from salt.utils import foo` becomes `import salt.utils.foo as foo`). * Renames the `test.rand_str` function to `test.random_hash` to more accurately reflect what it does * Modifies `salt.utils.stringutils.random()` (née `salt.utils.rand_string()`) such that it returns a string matching the passed size. Previously this function would get `size` bytes from `os.urandom()`, base64-encode it, and return the result, which would in most cases not be equal to the passed size.
165 lines
5.7 KiB
Python
165 lines
5.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Import python libs
|
|
from __future__ import absolute_import
|
|
import os
|
|
import sys
|
|
import time
|
|
import signal
|
|
import multiprocessing
|
|
|
|
# Import Salt Testing libs
|
|
from tests.support.unit import TestCase, skipIf
|
|
|
|
# Import salt libs
|
|
import salt.utils
|
|
import salt.utils.process
|
|
|
|
# Import 3rd-party libs
|
|
from salt.ext import six
|
|
from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin
|
|
|
|
|
|
class TestProcessManager(TestCase):
|
|
|
|
def test_basic(self):
|
|
'''
|
|
Make sure that the process is alive 2s later
|
|
'''
|
|
def spin():
|
|
salt.utils.appendproctitle('test_basic')
|
|
while True:
|
|
time.sleep(1)
|
|
|
|
process_manager = salt.utils.process.ProcessManager()
|
|
process_manager.add_process(spin)
|
|
initial_pid = next(six.iterkeys(process_manager._process_map))
|
|
time.sleep(2)
|
|
process_manager.check_children()
|
|
try:
|
|
assert initial_pid == next(six.iterkeys(process_manager._process_map))
|
|
finally:
|
|
process_manager.stop_restarting()
|
|
process_manager.kill_children()
|
|
time.sleep(0.5)
|
|
# Are there child processes still running?
|
|
if process_manager._process_map.keys():
|
|
process_manager.send_signal_to_processes(signal.SIGKILL)
|
|
process_manager.stop_restarting()
|
|
process_manager.kill_children()
|
|
|
|
def test_kill(self):
|
|
def spin():
|
|
salt.utils.appendproctitle('test_kill')
|
|
while True:
|
|
time.sleep(1)
|
|
|
|
process_manager = salt.utils.process.ProcessManager()
|
|
process_manager.add_process(spin)
|
|
initial_pid = next(six.iterkeys(process_manager._process_map))
|
|
# kill the child
|
|
os.kill(initial_pid, signal.SIGKILL)
|
|
# give the OS time to give the signal...
|
|
time.sleep(0.1)
|
|
process_manager.check_children()
|
|
try:
|
|
assert initial_pid != next(six.iterkeys(process_manager._process_map))
|
|
finally:
|
|
process_manager.stop_restarting()
|
|
process_manager.kill_children()
|
|
time.sleep(0.5)
|
|
# Are there child processes still running?
|
|
if process_manager._process_map.keys():
|
|
process_manager.send_signal_to_processes(signal.SIGKILL)
|
|
process_manager.stop_restarting()
|
|
process_manager.kill_children()
|
|
|
|
def test_restarting(self):
|
|
'''
|
|
Make sure that the process is alive 2s later
|
|
'''
|
|
def die():
|
|
salt.utils.appendproctitle('test_restarting')
|
|
|
|
process_manager = salt.utils.process.ProcessManager()
|
|
process_manager.add_process(die)
|
|
initial_pid = next(six.iterkeys(process_manager._process_map))
|
|
time.sleep(2)
|
|
process_manager.check_children()
|
|
try:
|
|
assert initial_pid != next(six.iterkeys(process_manager._process_map))
|
|
finally:
|
|
process_manager.stop_restarting()
|
|
process_manager.kill_children()
|
|
time.sleep(0.5)
|
|
# Are there child processes still running?
|
|
if process_manager._process_map.keys():
|
|
process_manager.send_signal_to_processes(signal.SIGKILL)
|
|
process_manager.stop_restarting()
|
|
process_manager.kill_children()
|
|
|
|
@skipIf(sys.version_info < (2, 7), 'Needs > Py 2.7 due to bug in stdlib')
|
|
def test_counter(self):
|
|
def incr(counter, num):
|
|
salt.utils.appendproctitle('test_counter')
|
|
for _ in range(0, num):
|
|
counter.value += 1
|
|
counter = multiprocessing.Value('i', 0)
|
|
process_manager = salt.utils.process.ProcessManager()
|
|
process_manager.add_process(incr, args=(counter, 2))
|
|
time.sleep(1)
|
|
process_manager.check_children()
|
|
time.sleep(1)
|
|
# we should have had 2 processes go at it
|
|
try:
|
|
assert counter.value == 4
|
|
finally:
|
|
process_manager.stop_restarting()
|
|
process_manager.kill_children()
|
|
time.sleep(0.5)
|
|
# Are there child processes still running?
|
|
if process_manager._process_map.keys():
|
|
process_manager.send_signal_to_processes(signal.SIGKILL)
|
|
process_manager.stop_restarting()
|
|
process_manager.kill_children()
|
|
|
|
|
|
class TestThreadPool(TestCase):
|
|
|
|
def test_basic(self):
|
|
'''
|
|
Make sure the threadpool can do things
|
|
'''
|
|
def incr_counter(counter):
|
|
counter.value += 1
|
|
counter = multiprocessing.Value('i', 0)
|
|
|
|
pool = salt.utils.process.ThreadPool()
|
|
sent = pool.fire_async(incr_counter, args=(counter,))
|
|
self.assertTrue(sent)
|
|
time.sleep(1) # Sleep to let the threads do things
|
|
self.assertEqual(counter.value, 1)
|
|
self.assertEqual(pool._job_queue.qsize(), 0)
|
|
|
|
def test_full_queue(self):
|
|
'''
|
|
Make sure that a full threadpool acts as we expect
|
|
'''
|
|
def incr_counter(counter):
|
|
counter.value += 1
|
|
counter = multiprocessing.Value('i', 0)
|
|
|
|
# Create a pool with no workers and 1 queue size
|
|
pool = salt.utils.process.ThreadPool(0, 1)
|
|
# make sure we can put the one item in
|
|
sent = pool.fire_async(incr_counter, args=(counter,))
|
|
self.assertTrue(sent)
|
|
# make sure we can't put more in
|
|
sent = pool.fire_async(incr_counter, args=(counter,))
|
|
self.assertFalse(sent)
|
|
time.sleep(1) # Sleep to let the threads do things
|
|
# make sure no one updated the counter
|
|
self.assertEqual(counter.value, 0)
|
|
# make sure the queue is still full
|
|
self.assertEqual(pool._job_queue.qsize(), 1)
|