valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-08 01:18:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'highstate'

Browse Source
This commit is contained in:
Thomas S Hatch 2011-05-28 23:38:39 -06:00
commit 3f32e1d3e5
9 changed files with 322 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
conf/master
View File

@ -43,6 +43,15 @@
# public keys from the minions
#auto_accept: False
##### State System settings #####
##########################################
# The state system uses a "top" file to tell the minions what environment to
# use and what modules to use. The state_top file is defined relative to the
# root of the base environment
#state_top: top.yml
#
# The renderer to use on the minions to render the state data
#renderer: yaml_jinja
##### File Server settings #####
##########################################
@ -50,8 +59,25 @@
# minions. This file server is built into the master daemon and does not
# require a dedicated port.
# The file root is the root directory exposed by the file server
#file_root: /srv/salt
# The file server works on environments passed to the master, each environment
# can have multiple root directories, the subdirectories in the multiple file
# roots cannot match, otherwise the downloaded files will not be able to be
# reliably ensured. A base environment is required to house the top file
# Example:
# file_roots:
# base:
# - /srv/salt/
# dev:
# - /srv/salt/dev/services
# - /srv/salt/dev/states
# prod:
# - /srv/salt/prod/services
# - /srv/salt/prod/states
#
# Default:
#file_roots:
# base:
# - /srv/salt
# The hash_type is the hash to use when discovering the hash of a file on
# the master server, the default is md5, but sha1, sha224, sha256, sha384

2
salt/__init__.py
View File

@ -71,7 +71,6 @@ class Master(object):
for name, level in self.opts['log_granular_levels'].iteritems():
salt.log.set_logger_level(name, level)
import logging
self.opts['logger'] = logging.getLogger('salt.stop-using-me')
verify_env([os.path.join(self.opts['pki_dir'], 'minions'),
os.path.join(self.opts['pki_dir'], 'minions_pre'),
os.path.join(self.opts['cachedir'], 'jobs'),
@ -139,7 +138,6 @@ class Minion(object):
salt.log.set_logger_level(name, level)
import logging
self.opts['logger'] = logging.getLogger('salt.stop-using-me')
verify_env([self.opts['pki_dir'], self.opts['cachedir']])
if self.cli['daemon']:

6
salt/config.py
View File

@ -70,12 +70,16 @@ def master_config(path):
'keep_jobs': 24,
'pki_dir': '/etc/salt/pki',
'cachedir': '/var/cache/salt',
'file_root': '/srv/salt',
'file_roots': {
'base': ['/srv/salt'],
},
'file_buffer_size': 1048576,
'hash_type': 'md5',
'conf_file': path,
'open_mode': False,
'auto_accept': False,
'renderer': 'yaml_jinja',
'state_top': 'top.yml',
'log_file': '/var/log/salt/master',
'log_level': 'warning',
'log_granular_levels': {},

52
salt/master.py
View File

@ -325,34 +325,54 @@ class MWorker(multiprocessing.Process):
self._send_cluster()
return ret
def _find_file(self, path, env='base'):
'''
Search the environment for the relative path
'''
fnd = {'path': '',
'rel': ''}
if not self.opts['file_roots'].has_key(env):
return fnd
for root in self.opts['file_roots'][env]:
full = os.path.join(root, path)
if os.path.isfile(full):
fnd['path'] = full
fnd['rel'] = path
return fnd
return fnd
def _serve_file(self, load):
'''
Return a chunk from a file based on the data received
'''
if not load.has_key('path') or not load.has_key('loc'):
return False
path = load['path']
if path.startswith('/'):
path = load['path'][1:]
path = os.path.join(self.opts['file_root'], path)
if not os.path.isfile(path):
return ''
fn_ = open(path, 'rb')
ret = {'data': '',
'dest': ''}
if not load.has_key('path')\
or not load.has_key('loc')\
or not load.has_key('env'):
return self.crypticle.dumps(ret)
fnd = self._find_file(load['path'], load['env'])
if not fnd['path']:
return self.crypticle.dumps(ret)
ret['dest'] = fnd['rel']
fn_ = open(fnd['path'], 'rb')
fn_.seek(load['loc'])
return self.crypticle.dumps(fn_.read(self.opts['file_buffer_size']))
ret['data'] = fn_.read(self.opts['file_buffer_size'])
return self.crypticle.dumps(ret)
def _file_hash(self, load):
'''
Return a file hash, the hash type is set in the master config file
'''
if not load.has_key('path'):
return False
path = os.path.join(self.opts['file_root'], load['path'])
if not os.path.isfile(path):
if not load.has_key('path')\
or not load.has_key('env'):
return False
path = self._find_file(load['path'], load['env'])
if not path:
return self.crypticle.dumps('')
ret = {}
ret['hsum'] = getattr(hashlib, self.opts['hash_type'])(open(path,
'rb').read()).hexdigest()
ret['hsum'] = getattr(hashlib, self.opts['hash_type'])(
open(path, 'rb').read()).hexdigest()
ret['hash_type'] = self.opts['hash_type']
return self.crypticle.dumps(ret)

199
salt/minion.py
View File

@ -49,6 +49,7 @@ class Minion(object):
self.opts = opts
self.mod_opts = self.__prep_mod_opts()
self.functions, self.returners = self.__load_modules()
self.matcher = Matcher(self.opts, self.functions)
self.authenticate()
def __prep_mod_opts(self):
@ -98,10 +99,10 @@ class Minion(object):
return
# Verify that the publication applies to this minion
if data.has_key('tgt_type'):
if not getattr(self, '_' + data['tgt_type'] + '_match')(data['tgt']):
if not getattr(self.matcher, data['tgt_type'] + '_match')(data['tgt']):
return
else:
if not self._glob_match(data['tgt']):
if not self.matcher.glob_match(data['tgt']):
return
self._handle_decoded_payload(data)
@ -140,46 +141,6 @@ class Minion(object):
target=lambda: self._thread_return(data)
).start()
def _glob_match(self, tgt):
'''
Returns true if the passed glob matches the id
'''
tmp_dir = tempfile.mkdtemp()
cwd = os.getcwd()
os.chdir(tmp_dir)
open(self.opts['id'], 'w+').write('salt')
ret = bool(glob.glob(tgt))
os.chdir(cwd)
shutil.rmtree(tmp_dir)
return ret
def _pcre_match(self, tgt):
'''
Returns true if the passed pcre regex matches
'''
return bool(re.match(tgt, self.opts['id']))
def _list_match(self, tgt):
'''
Determines if this host is on the list
'''
return bool(tgt.count(self.opts['id']))
def _grain_match(self, tgt):
'''
Reads in the grains regular expression match
'''
comps = tgt.split(':')
return bool(re.match(comps[1], self.opts['grains'][comps[0]]))
def _exsel_match(self, tgt):
'''
Runs a function and return the exit code
'''
if not self.functions.has_key(tgt):
return False
return(self.functions[tgt]())
def _thread_return(self, data):
'''
This method should be used as a threading target, start the actual
@ -297,6 +258,76 @@ class Minion(object):
payload = socket.recv_pyobj()
self._handle_payload(payload)
class Matcher(object):
'''
Use to return the value for matching calls from the master
'''
def __init__(self, opts, functions=None):
self.opts = opts
if not functions:
functions = salt.loader.minion_mods(self.opts)
else:
self.functions = functions
def confirm_top(self, data):
'''
Takes the data passed to a top file environment and determines if the
data matches this minion
'''
matcher = 'glob'
for item in data:
if type(item) == type(dict()):
if item.has_key('match'):
matcher = item['match']
if hasattr(self, matcher + '_match'):
return getattr(self, matcher + '_match')
else:
log.error('Attempting to match with unknown matcher: %s', matcher)
return False
def glob_match(self, tgt):
'''
Returns true if the passed glob matches the id
'''
tmp_dir = tempfile.mkdtemp()
cwd = os.getcwd()
os.chdir(tmp_dir)
open(self.opts['id'], 'w+').write('salt')
ret = bool(glob.glob(tgt))
os.chdir(cwd)
shutil.rmtree(tmp_dir)
return ret
def pcre_match(self, tgt):
'''
Returns true if the passed pcre regex matches
'''
return bool(re.match(tgt, self.opts['id']))
def list_match(self, tgt):
'''
Determines if this host is on the list
'''
return bool(tgt.count(self.opts['id']))
def grain_match(self, tgt):
'''
Reads in the grains regular expression match
'''
comps = tgt.split(':')
return bool(re.match(comps[1], self.opts['grains'][comps[0]]))
def exsel_match(self, tgt):
'''
Runs a function and return the exit code
'''
if not self.functions.has_key(tgt):
return False
return(self.functions[tgt]())
class FileClient(object):
'''
Interact with the salt master file server.
@ -319,54 +350,70 @@ class FileClient(object):
'''
Make sure that this path is intended for the salt master and trim it
'''
print path
if not path.startswith('salt://'):
raise MinionError('Unsupported path')
return path[:7]
return path[7:]
def get_file(self, path, dest, makedirs=False):
def get_file(self, path, dest='', makedirs=False, env='base'):
'''
Get a single file from the salt-master
'''
path = self._check_proto(path)
payload = {'enc': 'aes'}
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return False
fn_ = open(dest, 'w+')
fn_ = None
if dest:
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
if makedirs:
os.makedirs(destdir)
else:
return False
fn_ = open(dest, 'w+')
load = {'path': path,
'env': env,
'cmd': '_serve_file'}
while True:
load['loc'] = fn_.tell()
payload['load'] = self.crypticle.dumps(load)
if not fn_:
load['loc'] = 0
else:
load['loc'] = fn_.tell()
payload['load'] = self.auth.crypticle.dumps(load)
self.socket.send_pyobj(payload)
data = self.auth.crypticle.loads(self.socket.recv_pyobj())
if not data:
if not data['data']:
break
fn_.write(data)
if not fn_:
dest = os.path.join(
self.opts['cachedir'],
'files',
data['dest']
)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
fn_ = open(dest, 'w+')
fn_.write(data['data'])
return dest
def cache_file(self, path):
def cache_file(self, path, env='base'):
'''
Pull a file down from the file server and store it in the minion file
cache
'''
dest = os.path.join(self.opts['cachedir'], 'files', path)
return self.get_file(path, dest, True)
return self.get_file(path, '', True, env)
def cache_files(self, paths):
def cache_files(self, paths, env='base'):
'''
Download a list of files stored on the master and put them in the minion
file cache
'''
ret = []
for path in paths:
ret.append(self.cache_file(path))
ret.append(self.cache_file(path, env))
return ret
def hash_file(self, path):
def hash_file(self, path, env='base'):
'''
Return the hash of a file, to get the hash of a file on the
salt master file server prepend the path with salt://<file on server>
@ -375,7 +422,35 @@ class FileClient(object):
path = self._check_proto(path)
payload = {'enc': 'aes'}
load = {'path': path,
'env': env,
'cmd': '_file_hash'}
payload['load'] = auth.crypticle.dumps(load)
payload['load'] = self.auth.crypticle.dumps(load)
self.socket.send_pyobj(payload)
return self.auth.crypticle.loads(socket.recv_pyobj())
def get_state(self, sls, env):
'''
Get a state file from the master and store it in the local minion cache
return the location of the file
'''
if sls.count('.'):
sls = sls.replace('.', '/')
for path in [
'salt://' + sls + '.sls',
os.path.join('salt://', sls, 'init.sls')
]:
dest = self.cache_file(path, env)
if dest:
return dest
return False
def master_opts(self):
'''
Return the master opts data
'''
payload = {'enc': 'aes'}
load = {'cmd': '_master_opts'}
payload['load'] = self.auth.crypticle.dumps(load)
self.socket.send_pyobj(payload)
return self.auth.crypticle.loads(self.socket.recv_pyobj())

16
salt/modules/cp.py
View File

@ -38,30 +38,30 @@ def recv(files, dest):
return ret
def get_file(path, dest):
def get_file(path, dest, env='base'):
'''
Used to get a single file from the salt master
'''
client = salt.minion.FileClient(__opts__)
return client.get_file(path, dest)
return client.get_file(path, dest, False, env)
def cache_files(paths):
def cache_files(paths, env='base'):
'''
Used to gather many files from the master, the gathered files will be
saved in the minion cachedir reflective to the paths retrieved from the
master.
'''
client = salt.minion.FileClient(__opts__)
return client.cache_files(paths)
return client.cache_files(paths, env)
def cache_file(path):
def cache_file(path, env='base'):
'''
Used to cache a single file in the local salt-master file cache.
'''
client = salt.minion.FileClient(__opts__)
return client.cache_file(path)
return client.cache_file(path, env)
def hash_file(path):
def hash_file(path, env='base'):
'''
Return the hash of a file, to get the hash of a file on the
salt master file server prepend the path with salt://<file on server>
@ -70,4 +70,4 @@ def hash_file(path):
CLI Example:
'''
client = salt.minion.FileClient(__opts__)
return client.hash_file(path)
return client.hash_file(path, env)

9
salt/modules/state.py
View File

@ -50,3 +50,12 @@ def template_str(tem):
st_ = salt.state.State(__opts__)
return st_.call_template_str(tem)
def highstate():
'''
Retrive the state data from the salt master for this minion and execute it
CLI Example:
salt '*' state.highstate
'''
st_ = salt.state.HighState(__opts__)
return st_.call_highstate()

97
salt/state.py
View File

@ -16,8 +16,14 @@ import os
import copy
import inspect
import tempfile
import logging
# Import Salt modules
import salt.loader
import salt.minion
log = logging.getLogger(__name__)
class StateError(Exception): pass
class State(object):
'''
@ -271,3 +277,94 @@ class State(object):
if high:
return self.call_high(high)
return high
class HighState(object):
'''
Generate and execute the salt "High State". The High State is the compound
state derived from a group of template files stored on the salt master or
in a the local cache.
'''
def __init__(self, opts):
self.client = salt.minion.FileClient(opts)
self.opts = self.__gen_opts(opts)
self.state = State(self.opts)
self.matcher = salt.minion.Matcher(self.opts)
def __gen_opts(self, opts):
'''
The options used by the High State object are derived from options on
the minion and the master, or just the minion if the high state call is
entirely local.
'''
# If the state is intended to be applied locally, then the local opts
# should have all of the needed data, otherwise overwrite the local
# data items with data from the master
if opts.has_key('local_state'):
if opts['local_state']:
return opts
mopts = self.client.master_opts()
opts['renderer'] = mopts['renderer']
if mopts['state_top'].startswith('salt://'):
opts['state_top'] = mopts['state_top']
elif mopts['state_top'].startswith('/'):
opts['state_top'] = os.path.join('salt://', mopts['state_top'][1:])
else:
opts['state_top'] = os.path.join('salt://', mopts['state_top'])
return opts
def get_top(self):
'''
Returns the high data derived from the top file
'''
top = self.client.cache_file(self.opts['state_top'], 'base')
return self.state.compile_template(top)
def top_matches(self, top):
'''
Search through the top high data for matches and return the states that
this minion needs to execute.
Returns:
{'env': ['state1', 'state2', ...]}
'''
matches = {}
for env, body in top.items():
for match, data in body.items():
if self.matcher.confirm_top(data):
if not matches.has_key(env):
matches[env] = []
for item in data:
if type(item) == type(str()):
matches[env].append(item)
return matches
def gather_states(self, matches):
'''
Gather the template files from the master
'''
group = []
for env, states in matches.items():
for sls in states:
state = self.client.get_state(sls, env)
if state:
group.append(state)
return group
def render_highstate(self, group):
'''
Renders the collection of states into a single highstate data structure
'''
highstate = {}
for sls in group:
highstate.update( self.state.compile_template(sls))
return highstate
def call_highstate(self):
'''
Run the sequence to execute the salt highstate for this minion
'''
top = self.get_top()
matches = self.top_matches(top)
group = self.gather_states(matches)
high = self.render_highstate(group)
return self.state.call_high(high)

3
salt/states/pkg.py
View File

@ -27,8 +27,9 @@ def latest(name):
'''
Verify that the latest package is installed
'''
changes = {}
version = __salt__['pkg.version'](name)
avail = ['pkg.available_version'](name)
avail = __salt__['pkg.available_version'](name)
if avail > version:
changes = __salt__['pkg.install'](name, True)
if not changes: