mirror of
https://github.com/valitydev/salt.git
synced 2024-11-08 01:18:58 +00:00
Merge with upstream develop
This commit is contained in:
Dan Colish
commit
fefcc60e5d
@ -8,6 +8,7 @@ import logging
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import string
|
||||
import subprocess
|
||||
import urllib2
|
||||
@ -477,12 +478,12 @@ class RemoteClient(Client):
|
||||
data['dest']
|
||||
)
|
||||
destdir = os.path.dirname(dest)
|
||||
cumask = os.umask(191)
|
||||
cumask = os.umask(stat.S_IRWXG | stat.S_IRWXO)
|
||||
if not os.path.isdir(destdir):
|
||||
os.makedirs(destdir)
|
||||
if not os.path.exists(dest):
|
||||
open(dest, 'w+').write(data['data'])
|
||||
os.chmod(dest, 384)
|
||||
os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR)
|
||||
os.umask(cumask)
|
||||
break
|
||||
if not fn_:
|
||||
@ -493,11 +494,11 @@ class RemoteClient(Client):
|
||||
data['dest']
|
||||
)
|
||||
destdir = os.path.dirname(dest)
|
||||
cumask = os.umask(191)
|
||||
cumask = os.umask(stat.S_IRWXG | stat.S_IRWXO)
|
||||
if not os.path.isdir(destdir):
|
||||
os.makedirs(destdir)
|
||||
fn_ = open(dest, 'w+')
|
||||
os.chmod(dest, 384)
|
||||
os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR)
|
||||
os.umask(cumask)
|
||||
fn_.write(data['data'])
|
||||
return dest
|
||||
|
||||
361
salt/minion.py
361
salt/minion.py
@ -702,364 +702,3 @@ class Matcher(object):
|
||||
if tgt in nodegroups:
|
||||
return self.compound_match(nodegroups[tgt])
|
||||
return False
|
||||
|
||||
|
||||
class FileClient(object):
|
||||
'''
|
||||
Interact with the salt master file server.
|
||||
'''
|
||||
def __init__(self, opts):
|
||||
self.opts = opts
|
||||
self.serial = salt.payload.Serial(self.opts)
|
||||
self.auth = salt.crypt.SAuth(opts)
|
||||
self.socket = self.__get_socket()
|
||||
|
||||
def __get_socket(self):
|
||||
'''
|
||||
Return the ZeroMQ socket to use
|
||||
'''
|
||||
context = zmq.Context()
|
||||
socket = context.socket(zmq.REQ)
|
||||
socket.connect(self.opts['master_uri'])
|
||||
return socket
|
||||
|
||||
def _check_proto(self, path):
|
||||
'''
|
||||
Make sure that this path is intended for the salt master and trim it
|
||||
'''
|
||||
if not path.startswith('salt://'):
|
||||
raise MinionError('Unsupported path: {0}'.format(path))
|
||||
return path[7:]
|
||||
|
||||
def _file_local_list(self, dest):
|
||||
'''
|
||||
Helper util to return a list of files in a directory
|
||||
'''
|
||||
if os.path.isdir(dest):
|
||||
destdir = dest
|
||||
else:
|
||||
destdir = os.path.dirname(dest)
|
||||
|
||||
filelist = []
|
||||
|
||||
for root, dirs, files in os.walk(destdir):
|
||||
for name in files:
|
||||
path = os.path.join(root, name)
|
||||
filelist.append(path)
|
||||
|
||||
return filelist
|
||||
|
||||
def get_file(self, path, dest='', makedirs=False, env='base'):
|
||||
'''
|
||||
Get a single file from the salt-master
|
||||
'''
|
||||
path = self._check_proto(path)
|
||||
payload = {'enc': 'aes'}
|
||||
fn_ = None
|
||||
if dest:
|
||||
destdir = os.path.dirname(dest)
|
||||
if not os.path.isdir(destdir):
|
||||
if makedirs:
|
||||
os.makedirs(destdir)
|
||||
else:
|
||||
return False
|
||||
fn_ = open(dest, 'w+')
|
||||
load = {'path': path,
|
||||
'env': env,
|
||||
'cmd': '_serve_file'}
|
||||
while True:
|
||||
if not fn_:
|
||||
load['loc'] = 0
|
||||
else:
|
||||
load['loc'] = fn_.tell()
|
||||
payload['load'] = self.auth.crypticle.dumps(load)
|
||||
self.socket.send(self.serial.dumps(payload))
|
||||
data = self.auth.crypticle.loads(self.serial.loads(self.socket.recv()))
|
||||
if not data['data']:
|
||||
if not fn_ and data['dest']:
|
||||
# This is a 0 byte file on the master
|
||||
dest = os.path.join(
|
||||
self.opts['cachedir'],
|
||||
'files',
|
||||
env,
|
||||
data['dest']
|
||||
)
|
||||
destdir = os.path.dirname(dest)
|
||||
cumask = os.umask(stat.S_IRWXG | stat.S_IRWXO)
|
||||
if not os.path.isdir(destdir):
|
||||
os.makedirs(destdir)
|
||||
if not os.path.exists(dest):
|
||||
open(dest, 'w+').write(data['data'])
|
||||
os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR)
|
||||
os.umask(cumask)
|
||||
break
|
||||
if not fn_:
|
||||
dest = os.path.join(
|
||||
self.opts['cachedir'],
|
||||
'files',
|
||||
env,
|
||||
data['dest']
|
||||
)
|
||||
destdir = os.path.dirname(dest)
|
||||
cumask = os.umask(stat.S_IRWXG | stat.S_IRWXO)
|
||||
if not os.path.isdir(destdir):
|
||||
os.makedirs(destdir)
|
||||
fn_ = open(dest, 'w+')
|
||||
os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR)
|
||||
os.umask(cumask)
|
||||
fn_.write(data['data'])
|
||||
return dest
|
||||
|
||||
def get_dir(self, path, dest='', env='base'):
|
||||
'''
|
||||
Get a directory recursively from the salt-master
|
||||
'''
|
||||
ret = []
|
||||
# Strip trailing slash
|
||||
path = string.rstrip(self._check_proto(path), '/')
|
||||
# Break up the path into a list containing the bottom-level directory
|
||||
# (the one being recursively copied) and the directories preceding it
|
||||
separated = string.rsplit(path,'/',1)
|
||||
if len(separated) != 2:
|
||||
# No slashes in path. (This means all files in env will be copied)
|
||||
prefix = ''
|
||||
else:
|
||||
prefix = separated[0]
|
||||
|
||||
# Copy files from master
|
||||
for fn_ in self.file_list(env):
|
||||
if fn_.startswith(path):
|
||||
# Remove the leading directories from path to derive
|
||||
# the relative path on the minion.
|
||||
minion_relpath = string.lstrip(fn_[len(prefix):],'/')
|
||||
ret.append(self.get_file('salt://{0}'.format(fn_),
|
||||
'%s/%s' % (dest,minion_relpath),
|
||||
True, env))
|
||||
# Replicate empty dirs from master
|
||||
for fn_ in self.file_list_emptydirs(env):
|
||||
if fn_.startswith(path):
|
||||
# Remove the leading directories from path to derive
|
||||
# the relative path on the minion.
|
||||
minion_relpath = string.lstrip(fn_[len(prefix):],'/')
|
||||
minion_mkdir = '%s/%s' % (dest,minion_relpath)
|
||||
os.makedirs(minion_mkdir)
|
||||
ret.append(minion_mkdir)
|
||||
ret.sort()
|
||||
return ret
|
||||
|
||||
def get_url(self, url, dest, makedirs=False, env='base'):
|
||||
'''
|
||||
Get a single file from a URL.
|
||||
'''
|
||||
url_data = urlparse.urlparse(url)
|
||||
if url_data.scheme == 'salt':
|
||||
return self.get_file(url, dest, makedirs, env)
|
||||
if dest:
|
||||
destdir = os.path.dirname(dest)
|
||||
if not os.path.isdir(destdir):
|
||||
if makedirs:
|
||||
os.makedirs(destdir)
|
||||
else:
|
||||
return ''
|
||||
else:
|
||||
dest = os.path.join(
|
||||
self.opts['cachedir'],
|
||||
'extrn_files',
|
||||
env,
|
||||
os.path.join(
|
||||
url_data.netloc,
|
||||
os.path.relpath(url_data.path, '/'))
|
||||
)
|
||||
destdir = os.path.dirname(dest)
|
||||
if not os.path.isdir(destdir):
|
||||
os.makedirs(destdir)
|
||||
try:
|
||||
with contextlib.closing(urllib2.urlopen(url)) as srcfp:
|
||||
with open(dest, 'wb') as destfp:
|
||||
shutil.copyfileobj(srcfp, destfp)
|
||||
return dest
|
||||
except urllib2.HTTPError, ex:
|
||||
raise MinionError('HTTP error {0} reading {1}: {3}'.format(
|
||||
ex.code,
|
||||
url,
|
||||
*BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
|
||||
except urllib2.URLError, ex:
|
||||
raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
|
||||
return ''
|
||||
|
||||
def cache_file(self, path, env='base'):
|
||||
'''
|
||||
Pull a file down from the file server and store it in the minion
|
||||
file cache
|
||||
'''
|
||||
return self.get_url(path, '', True, env)
|
||||
|
||||
def cache_files(self, paths, env='base'):
|
||||
'''
|
||||
Download a list of files stored on the master and put them in the
|
||||
minion file cache
|
||||
'''
|
||||
ret = []
|
||||
for path in paths:
|
||||
ret.append(self.cache_file(path, env))
|
||||
return ret
|
||||
|
||||
def cache_master(self, env='base'):
|
||||
'''
|
||||
Download and cache all files on a master in a specified environment
|
||||
'''
|
||||
ret = []
|
||||
for path in self.file_list(env):
|
||||
ret.append(self.cache_file('salt://{0}'.format(path), env))
|
||||
return ret
|
||||
|
||||
def cache_dir(self, path, env='base'):
|
||||
'''
|
||||
Download all of the files in a subdir of the master
|
||||
'''
|
||||
ret = []
|
||||
path = self._check_proto(path)
|
||||
for fn_ in self.file_list(env):
|
||||
if fn_.startswith(path):
|
||||
local = self.cache_file('salt://{0}'.format(fn_), env)
|
||||
if not fn_.strip():
|
||||
continue
|
||||
ret.append(local)
|
||||
return ret
|
||||
|
||||
def cache_local_file(self, path, **kwargs):
|
||||
'''
|
||||
Cache a local file on the minion in the localfiles cache
|
||||
'''
|
||||
dest = os.path.join(self.opts['cachedir'], 'localfiles',
|
||||
path.lstrip('/'))
|
||||
destdir = os.path.dirname(dest)
|
||||
|
||||
if not os.path.isdir(destdir):
|
||||
os.makedirs(destdir)
|
||||
|
||||
shutil.copyfile(path, dest)
|
||||
return dest
|
||||
|
||||
def file_list(self, env='base'):
|
||||
'''
|
||||
List the files on the master
|
||||
'''
|
||||
payload = {'enc': 'aes'}
|
||||
load = {'env': env,
|
||||
'cmd': '_file_list'}
|
||||
payload['load'] = self.auth.crypticle.dumps(load)
|
||||
self.socket.send(self.serial.dumps(payload))
|
||||
return self.auth.crypticle.loads(self.serial.loads(self.socket.recv()))
|
||||
|
||||
def file_list_emptydirs(self, env='base'):
|
||||
'''
|
||||
List the empty dirs on the master
|
||||
'''
|
||||
payload = {'enc': 'aes'}
|
||||
load = {'env': env,
|
||||
'cmd': '_file_list_emptydirs'}
|
||||
payload['load'] = self.auth.crypticle.dumps(load)
|
||||
self.socket.send(self.serial.dumps(payload))
|
||||
return self.auth.crypticle.loads(self.serial.loads(self.socket.recv()))
|
||||
|
||||
def file_local_list(self, env='base'):
|
||||
'''
|
||||
List files in the local minion files and localfiles caches
|
||||
'''
|
||||
filesdest = os.path.join(self.opts['cachedir'], 'files', env)
|
||||
localfilesdest = os.path.join(self.opts['cachedir'], 'localfiles')
|
||||
|
||||
return sorted(self._file_local_list(filesdest) +
|
||||
self._file_local_list(localfilesdest))
|
||||
|
||||
def is_cached(self, path, env='base'):
|
||||
'''
|
||||
Returns the full path to a file if it is cached locally on the minion
|
||||
otherwise returns a blank string
|
||||
'''
|
||||
localsfilesdest = os.path.join(
|
||||
self.opts['cachedir'], 'localfiles', path.lstrip('/'))
|
||||
filesdest = os.path.join(
|
||||
self.opts['cachedir'], 'files', env, path.lstrip('salt://'))
|
||||
|
||||
if os.path.exists(filesdest):
|
||||
return filesdest
|
||||
elif os.path.exists(localsfilesdest):
|
||||
return localsfilesdest
|
||||
|
||||
return ''
|
||||
|
||||
def hash_file(self, path, env='base'):
|
||||
'''
|
||||
Return the hash of a file, to get the hash of a file on the salt
|
||||
master file server prepend the path with salt://<file on server>
|
||||
otherwise, prepend the file with / for a local file.
|
||||
'''
|
||||
try:
|
||||
path = self._check_proto(path)
|
||||
except MinionError:
|
||||
if not os.path.isfile(path):
|
||||
err = ('Specified file {0} is not present to generate '
|
||||
'hash').format(path)
|
||||
log.warning(err)
|
||||
return {}
|
||||
else:
|
||||
ret = {}
|
||||
ret['hsum'] = hashlib.md5(open(path, 'rb').read()).hexdigest()
|
||||
ret['hash_type'] = 'md5'
|
||||
return ret
|
||||
payload = {'enc': 'aes'}
|
||||
load = {'path': path,
|
||||
'env': env,
|
||||
'cmd': '_file_hash'}
|
||||
payload['load'] = self.auth.crypticle.dumps(load)
|
||||
self.socket.send(self.serial.dumps(payload))
|
||||
return self.auth.crypticle.loads(self.serial.loads(self.socket.recv()))
|
||||
|
||||
def list_env(self, path, env='base'):
|
||||
'''
|
||||
Return a list of the files in the file server's specified environment
|
||||
'''
|
||||
payload = {'enc': 'aes'}
|
||||
load = {'env': env,
|
||||
'cmd': '_file_list'}
|
||||
payload['load'] = self.auth.crypticle.dumps(load)
|
||||
self.socket.send(self.serial.dumps(payload))
|
||||
return self.auth.crypticle.loads(self.serial.loads(self.socket.recv()))
|
||||
|
||||
def get_state(self, sls, env):
|
||||
'''
|
||||
Get a state file from the master and store it in the local minion
|
||||
cache return the location of the file
|
||||
'''
|
||||
if '.' in sls:
|
||||
sls = sls.replace('.', '/')
|
||||
for path in ['salt://' + sls + '.sls',
|
||||
os.path.join('salt://', sls, 'init.sls')]:
|
||||
dest = self.cache_file(path, env)
|
||||
if dest:
|
||||
return dest
|
||||
return False
|
||||
|
||||
def master_opts(self):
|
||||
'''
|
||||
Return the master opts data
|
||||
'''
|
||||
payload = {'enc': 'aes'}
|
||||
load = {'cmd': '_master_opts'}
|
||||
payload['load'] = self.auth.crypticle.dumps(load)
|
||||
self.socket.send(self.serial.dumps(payload))
|
||||
return self.auth.crypticle.loads(self.serial.loads(self.socket.recv()))
|
||||
|
||||
def ext_nodes(self):
|
||||
'''
|
||||
Return the metadata derived from the external nodes system on the
|
||||
master.
|
||||
'''
|
||||
payload = {'enc': 'aes'}
|
||||
load = {'cmd': '_ext_nodes',
|
||||
'id': self.opts['id']}
|
||||
payload['load'] = self.auth.crypticle.dumps(load)
|
||||
self.socket.send(self.serial.dumps(payload))
|
||||
return self.auth.crypticle.loads(self.serial.loads(self.socket.recv()))
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
'''
|
||||
Minion side functions for salt-cp
|
||||
'''
|
||||
|
||||
# Import python libs
|
||||
import os
|
||||
|
||||
# Import salt libs
|
||||
import salt.minion
|
||||
import salt.fileclient
|
||||
|
||||
|
||||
def recv(files, dest):
|
||||
@ -46,7 +48,7 @@ def get_file(path, dest, env='base'):
|
||||
if not hash_file(path,env):
|
||||
return ''
|
||||
else:
|
||||
client = salt.minion.FileClient(__opts__)
|
||||
client = salt.fileclient.get_file_client(__opts__)
|
||||
return client.get_file(path, dest, False, env)
|
||||
|
||||
|
||||
@ -58,7 +60,7 @@ def get_dir(path, dest, env='base'):
|
||||
|
||||
salt '*' cp.get_dir salt://path/to/dir/ /minion/dest
|
||||
'''
|
||||
client = salt.minion.FileClient(__opts__)
|
||||
client = salt.fileclient.get_file_client(__opts__)
|
||||
return client.get_dir(path, dest, env)
|
||||
|
||||
|
||||
@ -71,7 +73,7 @@ def get_url(path, dest, env='base'):
|
||||
salt '*' cp.get_url salt://my/file /tmp/mine
|
||||
salt '*' cp.get_url http://www.slashdot.org /tmp/index.html
|
||||
'''
|
||||
client = salt.minion.FileClient(__opts__)
|
||||
client = salt.fileclient.get_file_client(__opts__)
|
||||
return client.get_url(path, dest, False, env)
|
||||
|
||||
|
||||
@ -83,7 +85,7 @@ def cache_file(path, env='base'):
|
||||
|
||||
salt '*' cp.cache_file salt://path/to/file
|
||||
'''
|
||||
client = salt.minion.FileClient(__opts__)
|
||||
client = salt.fileclient.get_file_client(__opts__)
|
||||
return client.cache_file(path, env)
|
||||
|
||||
|
||||
@ -97,7 +99,7 @@ def cache_files(paths, env='base'):
|
||||
|
||||
salt '*' cp.cache_files salt://pathto/file1,salt://pathto/file1
|
||||
'''
|
||||
client = salt.minion.FileClient(__opts__)
|
||||
client = salt.fileclient.get_file_client(__opts__)
|
||||
return client.cache_files(paths, env)
|
||||
|
||||
|
||||
@ -109,7 +111,7 @@ def cache_dir(path, env='base'):
|
||||
|
||||
salt '*' cp.cache_dir salt://path/to/dir
|
||||
'''
|
||||
client = salt.minion.FileClient(__opts__)
|
||||
client = salt.fileclient.get_file_client(__opts__)
|
||||
return client.cache_dir(path, env)
|
||||
|
||||
|
||||
@ -121,7 +123,7 @@ def cache_master(env='base'):
|
||||
|
||||
salt '*' cp.cache_master
|
||||
'''
|
||||
client = salt.minion.FileClient(__opts__)
|
||||
client = salt.fileclient.get_file_client(__opts__)
|
||||
return client.cache_master(env)
|
||||
|
||||
|
||||
@ -147,7 +149,7 @@ def cache_local_file(path):
|
||||
return path_cached
|
||||
|
||||
# The file hasn't been cached or has changed; cache it
|
||||
client = salt.minion.FileClient(__opts__)
|
||||
client = salt.fileclient.get_file_client(__opts__)
|
||||
return client.cache_local_file(path)
|
||||
|
||||
|
||||
@ -159,7 +161,7 @@ def list_master(env='base'):
|
||||
|
||||
salt '*' cp.list_master
|
||||
'''
|
||||
client = salt.minion.FileClient(__opts__)
|
||||
client = salt.fileclient.get_file_client(__opts__)
|
||||
return client.file_list(env)
|
||||
|
||||
|
||||
@ -171,7 +173,7 @@ def list_minion(env='base'):
|
||||
|
||||
salt '*' cp.list_minion
|
||||
'''
|
||||
client = salt.minion.FileClient(__opts__)
|
||||
client = salt.fileclient.get_file_client(__opts__)
|
||||
return client.file_local_list(env)
|
||||
|
||||
|
||||
@ -184,7 +186,7 @@ def is_cached(path, env='base'):
|
||||
|
||||
salt '*' cp.is_cached salt://path/to/file
|
||||
'''
|
||||
client = salt.minion.FileClient(__opts__)
|
||||
client = salt.fileclient.get_file_client(__opts__)
|
||||
return client.is_cached(path, env)
|
||||
|
||||
|
||||
@ -198,5 +200,5 @@ def hash_file(path, env='base'):
|
||||
|
||||
salt '*' cp.hash_file salt://path/to/file
|
||||
'''
|
||||
client = salt.minion.FileClient(__opts__)
|
||||
client = salt.fileclient.get_file_client(__opts__)
|
||||
return client.hash_file(path, env)
|
||||
|
||||
@ -11,6 +11,7 @@ The data sent to the state calls is as follows:
|
||||
}
|
||||
'''
|
||||
|
||||
# Import python libs
|
||||
import os
|
||||
import copy
|
||||
import inspect
|
||||
@ -19,9 +20,11 @@ import logging
|
||||
import tempfile
|
||||
import collections
|
||||
|
||||
# Import Salt Libs
|
||||
import salt.utils
|
||||
import salt.loader
|
||||
import salt.minion
|
||||
import salt.fileclient
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@ -797,7 +800,7 @@ class HighState(object):
|
||||
salt master or in the local cache.
|
||||
'''
|
||||
def __init__(self, opts):
|
||||
self.client = salt.minion.FileClient(opts)
|
||||
self.client = salt.fileclient.get_file_client(opts)
|
||||
self.opts = self.__gen_opts(opts)
|
||||
self.state = State(self.opts)
|
||||
self.matcher = salt.minion.Matcher(self.opts)
|
||||
|
||||
@ -1,8 +1,17 @@
|
||||
'''
|
||||
Jinja loading utils to enable a more powerful backend for jinja templates
|
||||
'''
|
||||
# Import python libs
|
||||
from os import path
|
||||
|
||||
# Import third-party libs
|
||||
from jinja2 import Template, BaseLoader, Environment
|
||||
from jinja2.loaders import split_template_path
|
||||
from jinja2.exceptions import TemplateNotFound
|
||||
|
||||
# Import Salt libs
|
||||
import salt
|
||||
import salt.fileclient
|
||||
|
||||
|
||||
def get_template(filename, opts, env):
|
||||
@ -40,7 +49,7 @@ class SaltCacheLoader(BaseLoader):
|
||||
Return a file client. Instantiates on first call.
|
||||
'''
|
||||
if not self._file_client:
|
||||
self._file_client = salt.minion.FileClient(self.opts)
|
||||
self._file_client = salt.fileclient.get_file_client(self.opts)
|
||||
return self._file_client
|
||||
|
||||
def cache_file(self, template):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user