diff --git a/salt/fileclient.py b/salt/fileclient.py index 202a9a1243..328d9c3512 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -8,6 +8,7 @@ import logging import hashlib import os import shutil +import stat import string import subprocess import urllib2 @@ -477,12 +478,12 @@ class RemoteClient(Client): data['dest'] ) destdir = os.path.dirname(dest) - cumask = os.umask(191) + cumask = os.umask(stat.S_IRWXG | stat.S_IRWXO) if not os.path.isdir(destdir): os.makedirs(destdir) if not os.path.exists(dest): open(dest, 'w+').write(data['data']) - os.chmod(dest, 384) + os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR) os.umask(cumask) break if not fn_: @@ -493,11 +494,11 @@ class RemoteClient(Client): data['dest'] ) destdir = os.path.dirname(dest) - cumask = os.umask(191) + cumask = os.umask(stat.S_IRWXG | stat.S_IRWXO) if not os.path.isdir(destdir): os.makedirs(destdir) fn_ = open(dest, 'w+') - os.chmod(dest, 384) + os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR) os.umask(cumask) fn_.write(data['data']) return dest diff --git a/salt/minion.py b/salt/minion.py index 2b042b89c7..b74196bcd4 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -702,364 +702,3 @@ class Matcher(object): if tgt in nodegroups: return self.compound_match(nodegroups[tgt]) return False - - -class FileClient(object): - ''' - Interact with the salt master file server. - ''' - def __init__(self, opts): - self.opts = opts - self.serial = salt.payload.Serial(self.opts) - self.auth = salt.crypt.SAuth(opts) - self.socket = self.__get_socket() - - def __get_socket(self): - ''' - Return the ZeroMQ socket to use - ''' - context = zmq.Context() - socket = context.socket(zmq.REQ) - socket.connect(self.opts['master_uri']) - return socket - - def _check_proto(self, path): - ''' - Make sure that this path is intended for the salt master and trim it - ''' - if not path.startswith('salt://'): - raise MinionError('Unsupported path: {0}'.format(path)) - return path[7:] - - def _file_local_list(self, dest): - ''' - Helper util to return a list of files in a directory - ''' - if os.path.isdir(dest): - destdir = dest - else: - destdir = os.path.dirname(dest) - - filelist = [] - - for root, dirs, files in os.walk(destdir): - for name in files: - path = os.path.join(root, name) - filelist.append(path) - - return filelist - - def get_file(self, path, dest='', makedirs=False, env='base'): - ''' - Get a single file from the salt-master - ''' - path = self._check_proto(path) - payload = {'enc': 'aes'} - fn_ = None - if dest: - destdir = os.path.dirname(dest) - if not os.path.isdir(destdir): - if makedirs: - os.makedirs(destdir) - else: - return False - fn_ = open(dest, 'w+') - load = {'path': path, - 'env': env, - 'cmd': '_serve_file'} - while True: - if not fn_: - load['loc'] = 0 - else: - load['loc'] = fn_.tell() - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - data = self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) - if not data['data']: - if not fn_ and data['dest']: - # This is a 0 byte file on the master - dest = os.path.join( - self.opts['cachedir'], - 'files', - env, - data['dest'] - ) - destdir = os.path.dirname(dest) - cumask = os.umask(stat.S_IRWXG | stat.S_IRWXO) - if not os.path.isdir(destdir): - os.makedirs(destdir) - if not os.path.exists(dest): - open(dest, 'w+').write(data['data']) - os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR) - os.umask(cumask) - break - if not fn_: - dest = os.path.join( - self.opts['cachedir'], - 'files', - env, - data['dest'] - ) - destdir = os.path.dirname(dest) - cumask = os.umask(stat.S_IRWXG | stat.S_IRWXO) - if not os.path.isdir(destdir): - os.makedirs(destdir) - fn_ = open(dest, 'w+') - os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR) - os.umask(cumask) - fn_.write(data['data']) - return dest - - def get_dir(self, path, dest='', env='base'): - ''' - Get a directory recursively from the salt-master - ''' - ret = [] - # Strip trailing slash - path = string.rstrip(self._check_proto(path), '/') - # Break up the path into a list containing the bottom-level directory - # (the one being recursively copied) and the directories preceding it - separated = string.rsplit(path,'/',1) - if len(separated) != 2: - # No slashes in path. (This means all files in env will be copied) - prefix = '' - else: - prefix = separated[0] - - # Copy files from master - for fn_ in self.file_list(env): - if fn_.startswith(path): - # Remove the leading directories from path to derive - # the relative path on the minion. - minion_relpath = string.lstrip(fn_[len(prefix):],'/') - ret.append(self.get_file('salt://{0}'.format(fn_), - '%s/%s' % (dest,minion_relpath), - True, env)) - # Replicate empty dirs from master - for fn_ in self.file_list_emptydirs(env): - if fn_.startswith(path): - # Remove the leading directories from path to derive - # the relative path on the minion. - minion_relpath = string.lstrip(fn_[len(prefix):],'/') - minion_mkdir = '%s/%s' % (dest,minion_relpath) - os.makedirs(minion_mkdir) - ret.append(minion_mkdir) - ret.sort() - return ret - - def get_url(self, url, dest, makedirs=False, env='base'): - ''' - Get a single file from a URL. - ''' - url_data = urlparse.urlparse(url) - if url_data.scheme == 'salt': - return self.get_file(url, dest, makedirs, env) - if dest: - destdir = os.path.dirname(dest) - if not os.path.isdir(destdir): - if makedirs: - os.makedirs(destdir) - else: - return '' - else: - dest = os.path.join( - self.opts['cachedir'], - 'extrn_files', - env, - os.path.join( - url_data.netloc, - os.path.relpath(url_data.path, '/')) - ) - destdir = os.path.dirname(dest) - if not os.path.isdir(destdir): - os.makedirs(destdir) - try: - with contextlib.closing(urllib2.urlopen(url)) as srcfp: - with open(dest, 'wb') as destfp: - shutil.copyfileobj(srcfp, destfp) - return dest - except urllib2.HTTPError, ex: - raise MinionError('HTTP error {0} reading {1}: {3}'.format( - ex.code, - url, - *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code])) - except urllib2.URLError, ex: - raise MinionError('Error reading {0}: {1}'.format(url, ex.reason)) - return '' - - def cache_file(self, path, env='base'): - ''' - Pull a file down from the file server and store it in the minion - file cache - ''' - return self.get_url(path, '', True, env) - - def cache_files(self, paths, env='base'): - ''' - Download a list of files stored on the master and put them in the - minion file cache - ''' - ret = [] - for path in paths: - ret.append(self.cache_file(path, env)) - return ret - - def cache_master(self, env='base'): - ''' - Download and cache all files on a master in a specified environment - ''' - ret = [] - for path in self.file_list(env): - ret.append(self.cache_file('salt://{0}'.format(path), env)) - return ret - - def cache_dir(self, path, env='base'): - ''' - Download all of the files in a subdir of the master - ''' - ret = [] - path = self._check_proto(path) - for fn_ in self.file_list(env): - if fn_.startswith(path): - local = self.cache_file('salt://{0}'.format(fn_), env) - if not fn_.strip(): - continue - ret.append(local) - return ret - - def cache_local_file(self, path, **kwargs): - ''' - Cache a local file on the minion in the localfiles cache - ''' - dest = os.path.join(self.opts['cachedir'], 'localfiles', - path.lstrip('/')) - destdir = os.path.dirname(dest) - - if not os.path.isdir(destdir): - os.makedirs(destdir) - - shutil.copyfile(path, dest) - return dest - - def file_list(self, env='base'): - ''' - List the files on the master - ''' - payload = {'enc': 'aes'} - load = {'env': env, - 'cmd': '_file_list'} - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) - - def file_list_emptydirs(self, env='base'): - ''' - List the empty dirs on the master - ''' - payload = {'enc': 'aes'} - load = {'env': env, - 'cmd': '_file_list_emptydirs'} - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) - - def file_local_list(self, env='base'): - ''' - List files in the local minion files and localfiles caches - ''' - filesdest = os.path.join(self.opts['cachedir'], 'files', env) - localfilesdest = os.path.join(self.opts['cachedir'], 'localfiles') - - return sorted(self._file_local_list(filesdest) + - self._file_local_list(localfilesdest)) - - def is_cached(self, path, env='base'): - ''' - Returns the full path to a file if it is cached locally on the minion - otherwise returns a blank string - ''' - localsfilesdest = os.path.join( - self.opts['cachedir'], 'localfiles', path.lstrip('/')) - filesdest = os.path.join( - self.opts['cachedir'], 'files', env, path.lstrip('salt://')) - - if os.path.exists(filesdest): - return filesdest - elif os.path.exists(localsfilesdest): - return localsfilesdest - - return '' - - def hash_file(self, path, env='base'): - ''' - Return the hash of a file, to get the hash of a file on the salt - master file server prepend the path with salt:// - otherwise, prepend the file with / for a local file. - ''' - try: - path = self._check_proto(path) - except MinionError: - if not os.path.isfile(path): - err = ('Specified file {0} is not present to generate ' - 'hash').format(path) - log.warning(err) - return {} - else: - ret = {} - ret['hsum'] = hashlib.md5(open(path, 'rb').read()).hexdigest() - ret['hash_type'] = 'md5' - return ret - payload = {'enc': 'aes'} - load = {'path': path, - 'env': env, - 'cmd': '_file_hash'} - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) - - def list_env(self, path, env='base'): - ''' - Return a list of the files in the file server's specified environment - ''' - payload = {'enc': 'aes'} - load = {'env': env, - 'cmd': '_file_list'} - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) - - def get_state(self, sls, env): - ''' - Get a state file from the master and store it in the local minion - cache return the location of the file - ''' - if '.' in sls: - sls = sls.replace('.', '/') - for path in ['salt://' + sls + '.sls', - os.path.join('salt://', sls, 'init.sls')]: - dest = self.cache_file(path, env) - if dest: - return dest - return False - - def master_opts(self): - ''' - Return the master opts data - ''' - payload = {'enc': 'aes'} - load = {'cmd': '_master_opts'} - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) - - def ext_nodes(self): - ''' - Return the metadata derived from the external nodes system on the - master. - ''' - payload = {'enc': 'aes'} - load = {'cmd': '_ext_nodes', - 'id': self.opts['id']} - payload['load'] = self.auth.crypticle.dumps(load) - self.socket.send(self.serial.dumps(payload)) - return self.auth.crypticle.loads(self.serial.loads(self.socket.recv())) diff --git a/salt/modules/cp.py b/salt/modules/cp.py index 355c9e1f77..cdc0ad0f85 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -1,10 +1,12 @@ ''' Minion side functions for salt-cp ''' - +# Import python libs import os +# Import salt libs import salt.minion +import salt.fileclient def recv(files, dest): @@ -46,7 +48,7 @@ def get_file(path, dest, env='base'): if not hash_file(path,env): return '' else: - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.get_file(path, dest, False, env) @@ -58,7 +60,7 @@ def get_dir(path, dest, env='base'): salt '*' cp.get_dir salt://path/to/dir/ /minion/dest ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.get_dir(path, dest, env) @@ -71,7 +73,7 @@ def get_url(path, dest, env='base'): salt '*' cp.get_url salt://my/file /tmp/mine salt '*' cp.get_url http://www.slashdot.org /tmp/index.html ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.get_url(path, dest, False, env) @@ -83,7 +85,7 @@ def cache_file(path, env='base'): salt '*' cp.cache_file salt://path/to/file ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.cache_file(path, env) @@ -97,7 +99,7 @@ def cache_files(paths, env='base'): salt '*' cp.cache_files salt://pathto/file1,salt://pathto/file1 ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.cache_files(paths, env) @@ -109,7 +111,7 @@ def cache_dir(path, env='base'): salt '*' cp.cache_dir salt://path/to/dir ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.cache_dir(path, env) @@ -121,7 +123,7 @@ def cache_master(env='base'): salt '*' cp.cache_master ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.cache_master(env) @@ -147,7 +149,7 @@ def cache_local_file(path): return path_cached # The file hasn't been cached or has changed; cache it - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.cache_local_file(path) @@ -159,7 +161,7 @@ def list_master(env='base'): salt '*' cp.list_master ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.file_list(env) @@ -171,7 +173,7 @@ def list_minion(env='base'): salt '*' cp.list_minion ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.file_local_list(env) @@ -184,7 +186,7 @@ def is_cached(path, env='base'): salt '*' cp.is_cached salt://path/to/file ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.is_cached(path, env) @@ -198,5 +200,5 @@ def hash_file(path, env='base'): salt '*' cp.hash_file salt://path/to/file ''' - client = salt.minion.FileClient(__opts__) + client = salt.fileclient.get_file_client(__opts__) return client.hash_file(path, env) diff --git a/salt/state.py b/salt/state.py index 4db99955cb..1fea73e599 100644 --- a/salt/state.py +++ b/salt/state.py @@ -11,6 +11,7 @@ The data sent to the state calls is as follows: } ''' +# Import python libs import os import copy import inspect @@ -19,9 +20,11 @@ import logging import tempfile import collections +# Import Salt Libs import salt.utils import salt.loader import salt.minion +import salt.fileclient log = logging.getLogger(__name__) @@ -797,7 +800,7 @@ class HighState(object): salt master or in the local cache. ''' def __init__(self, opts): - self.client = salt.minion.FileClient(opts) + self.client = salt.fileclient.get_file_client(opts) self.opts = self.__gen_opts(opts) self.state = State(self.opts) self.matcher = salt.minion.Matcher(self.opts) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 5ec8407461..3655f1144d 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -1,8 +1,17 @@ +''' +Jinja loading utils to enable a more powerful backend for jinja templates +''' +# Import python libs from os import path + +# Import third-party libs from jinja2 import Template, BaseLoader, Environment from jinja2.loaders import split_template_path from jinja2.exceptions import TemplateNotFound + +# Import Salt libs import salt +import salt.fileclient def get_template(filename, opts, env): @@ -40,7 +49,7 @@ class SaltCacheLoader(BaseLoader): Return a file client. Instantiates on first call. ''' if not self._file_client: - self._file_client = salt.minion.FileClient(self.opts) + self._file_client = salt.fileclient.get_file_client(self.opts) return self._file_client def cache_file(self, template):