valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-08 01:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #2027 from s0undt3ch/develop

Browse Source 
Inform the users about the current max open files situation.
This commit is contained in:
Thomas S Hatch 2012-09-16 09:38:59 -07:00
commit a358885829
5 changed files with 244 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
salt/log.py
View File

@ -120,7 +120,7 @@ def setup_console_logger(log_level='error', log_format=None, date_format=None):
Setup the console logger
'''
if is_console_configured():
logging.getLogger(__name__).warning("Console logging already configured")
logging.getLogger(__name__).warn('Console logging already configured')
return
init()
@ -157,7 +157,7 @@ def setup_logfile_logger(log_path, log_level='error', log_format=None,
'''
if is_logfile_configured():
logging.getLogger(__name__).warning("Logfile logging already configured")
logging.getLogger(__name__).warn('Logfile logging already configured')
return
init()

7
salt/master.py
View File

@ -40,6 +40,7 @@ import salt.pillar
import salt.state
import salt.runner
import salt.utils.event
import salt.utils.verify
from salt.utils.debug import enable_sigusr1_handler
@ -1166,15 +1167,19 @@ class ClearFuncs(object):
Authenticate the client, use the sent public key to encrypt the aes key
which was generated at start up.
This method fires an event over the master event manager. The evnt is
This method fires an event over the master event manager. The event is
tagged "auth" and returns a dict with information about the auth
event
'''
# 0. Check for max open files
# 1. Verify that the key we are receiving matches the stored key
# 2. Store the key if it is not there
# 3. make an rsa key with the pub key
# 4. encrypt the aes key as an encrypted salt.payload
# 5. package the return and return it
salt.utils.verify.check_max_open_files(self.opts)
log.info('Authentication request from {id}'.format(**load))
pubfn = os.path.join(self.opts['pki_dir'],
'minions',

61
salt/utils/verify.py
View File

@ -9,12 +9,14 @@ import stat
import socket
import getpass
import logging
import resource
from salt.log import is_console_configured
from salt.exceptions import SaltClientError
log = logging.getLogger(__name__)
def zmq_version():
'''
ZeroMQ python bindings >= 2.1.9 are required
@ -235,6 +237,7 @@ def check_user(user):
return False
return True
def check_parent_dirs(fname, user='root'):
'''
Walk from the root up to a directory and verify that the current
@ -246,10 +249,10 @@ def check_parent_dirs(fname, user='root'):
dir_comps = fname.split(os.path.sep)[1:-1]
# Loop over all parent directories of the minion key
# to properly test if salt has read access to them.
for i,dirname in enumerate(dir_comps):
for i, dirname in enumerate(dir_comps):
# Create the full path to the directory using a list slice
d = os.path.join(os.path.sep, *dir_comps[:i + 1])
msg ='Could not access directory {0}.'.format(d)
msg = 'Could not access directory {0}.'.format(d)
current_user = getpass.getuser()
# Make the error message more intelligent based on how
# the user invokes salt-call or whatever other script.
@ -261,3 +264,57 @@ def check_parent_dirs(fname, user='root'):
# Propagate this exception up so there isn't a sys.exit()
# in the middle of code that could be imported elsewhere.
raise SaltClientError(msg)
def check_max_open_files(opts):
mof_c = opts.get('max_open_files', 100000)
mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
accepted_keys_dir = os.path.join(opts.get('pki_dir'), 'minions')
accepted_count = len([
key for key in os.listdir(accepted_keys_dir) if
os.path.isfile(os.path.join(accepted_keys_dir, key))
])
log.debug(
'This salt-master instance has accepted {0} minion keys.'.format(
accepted_count
)
)
level = logging.INFO
if (accepted_count * 4) <= mof_s:
# We check for the soft value of max open files here because that's the
# value the user chose to raise to.
#
# The number of accepted keys multiplied by four(4) is lower than the
# soft value, everything should be OK
return
msg = (
'The number of accepted minion keys({0}) should be lower than 1/4 '
'of the max open files soft setting({1}). '.format(
accepted_count, mof_s
)
)
if accepted_count >= mof_s:
# This should never occur, it might have already crashed
msg += 'salt-master will crash pretty soon! '
level = logging.CRITICAL
elif (accepted_count * 2) >= mof_s:
# This is way too low, CRITICAL
level = logging.CRITICAL
elif (accepted_count * 3) >= mof_s:
level = logging.WARNING
# The accepted count is more than 3 time, WARN
elif (accepted_count * 4) >= mof_s:
level = logging.INFO
if mof_c < mof_h:
msg += ('According to the system\'s hard limit, there\'s still a '
'margin of {0} to raise the salt\'s max_open_files '
'setting. ').format(mof_h - mof_c)
msg += 'Please consider raising this value.'
log.log(level=level, msg=msg)

79
tests/saltunittest.py
View File

@ -10,19 +10,31 @@ test from here
# Import python libs
import os
import sys
import logging
# support python < 2.7 via unittest2
if sys.version_info[0:2] < (2, 7):
try:
from unittest2 import TestLoader, TextTestRunner,\
TestCase, expectedFailure, \
TestSuite, skipIf
from unittest2 import (
TestLoader,
TextTestRunner,
TestCase,
expectedFailure,
TestSuite,
skipIf
)
except ImportError:
raise SystemExit("You need to install unittest2 to run the salt tests")
else:
from unittest import TestLoader, TextTestRunner,\
TestCase, expectedFailure, \
TestSuite, skipIf
from unittest import (
TestLoader,
TextTestRunner,
TestCase,
expectedFailure,
TestSuite,
skipIf
)
# Set up paths
TEST_DIR = os.path.dirname(os.path.normpath(os.path.abspath(__file__)))
@ -31,3 +43,58 @@ SALT_LIBS = os.path.dirname(TEST_DIR)
for dir_ in [TEST_DIR, SALT_LIBS]:
if not dir_ in sys.path:
sys.path.insert(0, dir_)
class TestsLoggingHandler(object):
'''
Simple logging handler which can be used to test if certain logging
messages get emitted or not::
..code-block: python
with TestsLoggingHandler() as handler:
# (...) Do what ever you wish here
handler.messages # here are the emitted log messages
'''
def __init__(self, level=0, format='%(levelname)s:%(message)s'):
self.level = level
self.format = format
self.activated = False
def activate(self):
class Handler(logging.Handler):
def __init__(self, level):
logging.Handler.__init__(self, level)
self.messages = []
def emit(self, record):
self.messages.append(self.format(record))
self.handler = Handler(self.level)
formatter = logging.Formatter(self.format)
self.handler.setFormatter(formatter)
logging.root.addHandler(self.handler)
self.activated = True
def deactivate(self):
if not self.activated:
return
logging.root.removeHandler(self.handler)
@property
def messages(self):
if not self.activated:
return []
return self.handler.messages
def clear(self):
self.handler.messages = []
def __enter__(self):
self.activate()
return self
def __exit__(self, type, value, traceback):
self.deactivate()
self.activated = False

105
tests/unit/utils/verify_test.py
View File

@ -2,15 +2,18 @@ import getpass
import os
import sys
import stat
import shutil
import resource
import tempfile
from saltunittest import skipIf, TestCase
from saltunittest import skipIf, TestCase, TestsLoggingHandler
from salt.utils.verify import (
check_user,
verify_env,
verify_socket,
zmq_version,
check_max_open_files
)
@ -62,3 +65,103 @@ class TestVerify(TestCase):
def test_verify_socket(self):
self.assertTrue(verify_socket('', 18000, 18001))
@skipIf(os.environ.get('TRAVIS_PYTHON_VERSION', None) is not None,
'Travis environment does not like too many open files')
def test_max_open_files(self):
with TestsLoggingHandler() as handler:
logmsg_dbg = (
'DEBUG:This salt-master instance has accepted {0} minion keys.'
)
logmsg_chk = (
'{0}:The number of accepted minion keys({1}) should be lower '
'than 1/4 of the max open files soft setting({2}). According '
'to the system\'s hard limit, there\'s still a margin of {3} '
'to raise the salt\'s max_open_files setting. Please consider '
'raising this value.'
)
logmsg_crash = (
'{0}:The number of accepted minion keys({1}) should be lower '
'than 1/4 of the max open files soft setting({2}). '
'salt-master will crash pretty soon! According to the '
'system\'s hard limit, there\'s still a margin of {3} to '
'raise the salt\'s max_open_files setting. Please consider '
'raising this value.'
)
mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
tempdir = tempfile.mkdtemp(prefix='fake-keys')
keys_dir = os.path.join(tempdir, 'minions')
os.makedirs(keys_dir)
mof_test = 256
resource.setrlimit(resource.RLIMIT_NOFILE, (mof_test, mof_h))
try:
prev = 0
for newmax, level in ((24, None), (66, 'INFO'),
(127, 'WARNING'), (196, 'CRITICAL')):
for n in range(prev, newmax):
with open(os.path.join(keys_dir, str(n)), 'w') as fp_:
fp_.write(str(n))
opts = {
'max_open_files': newmax,
'pki_dir': tempdir
}
check_max_open_files(opts)
self.assertIn(logmsg_dbg.format(newmax), handler.messages)
if level is None:
# No log message is triggered, only the DEBUG one which
# tells us how many minion keys were accepted.
self.assertEqual(
[logmsg_dbg.format(newmax)],
handler.messages
)
else:
self.assertIn(
logmsg_chk.format(
level,
newmax,
mof_test,
mof_h - newmax,
),
handler.messages
)
handler.clear()
prev = newmax
newmax = mof_test
for n in range(prev, newmax):
with open(os.path.join(keys_dir, str(n)), 'w') as fp_:
fp_.write(str(n))
opts = {
'max_open_files': newmax,
'pki_dir': tempdir
}
check_max_open_files(opts)
self.assertIn(logmsg_dbg.format(newmax), handler.messages)
self.assertIn(
logmsg_crash.format(
'CRITICAL',
newmax,
mof_test,
mof_h - newmax,
),
handler.messages
)
handler.clear()
except IOError, err:
if err.errno == 24:
# Too many open files
self.skipTest('We\'ve hit the max open files setting')
raise
finally:
shutil.rmtree(tempdir)
resource.setrlimit(resource.RLIMIT_NOFILE, (mof_s, mof_h))