valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-07 17:09:03 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' of https://github.com/saltstack/salt-cloud into release

Browse Source
This commit is contained in:
Joseph Hall 2012-11-15 19:20:35 +00:00
commit 08816a77eb
8 changed files with 482 additions and 250 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
AUTHORS
View File

@ -33,6 +33,7 @@ Peter Baumgartner
Robert Fielding
Seth House
Thomas S Hatch <thatch@saltstack.com>
Pedro Algarvio <pedro@algarvio.me>
Growing Community

37
conf/cloud
View File

@ -38,3 +38,40 @@
#JOYENT.password: zaphod77
#JOYENT.private_key: /root/joyent.pem
#
##### Logging settings #####
##########################################
# The location of the master log file
#log_file: /var/log/salt/cloud
#
# The level of messages to send to the console.
# One of 'garbage', 'trace', 'debug', info', 'warning', 'error', 'critical'.
# Default: 'warning'
#log_level: warning
#
# The level of messages to send to the log file.
# One of 'garbage', 'trace', 'debug', info', 'warning', 'error', 'critical'.
# Default: 'warning'
#log_level_logfile:
#
# The date and time format used in log messages. Allowed date/time formating
# can be seen here:
# http://docs.python.org/library/time.html#time.strftime
#log_datefmt: '%Y-%m-%d %H:%M:%S'
#
# The format of the console logging messages. Allowed formatting options can
# be seen here:
# http://docs.python.org/library/logging.html#logrecord-attributes
#log_fmt_console: '[%(levelname)-8s] %(message)s'
#log_fmt_logfile: '%(asctime)s,%(msecs)03.0f [%(name)-17s][%(levelname)-8s] %(message)s'
#
# Logger levels can be used to tweak specific loggers logging levels.
# For example, if you want to have the salt library at the 'warning' level,
# but you still wish to have 'salt.modules' at the 'debug' level:
# log_granular_levels:
# 'salt': 'warning',
# 'salt.modules': 'debug'
# 'saltcloud': 'info'
#
#log_granular_levels: {}

277
saltcloud/cli.py
View File

@ -9,6 +9,7 @@ Primary interfaces for the salt-cloud system
#
# The cli, master and cloud configs will merge for opts
# the vm data will be in opts['vm']
# Import python libs
import optparse
import os
@ -19,255 +20,67 @@ import saltcloud.output
import salt.config
import salt.output
from saltcloud.version import __version__ as VERSION
# Import saltcloud libs
from saltcloud.utils import parsers
class SaltCloud(object):
'''
Create a cli SaltCloud object
'''
def __init__(self):
self.opts = self.parse()
def parse(self):
'''
Parse the command line and merge the config
'''
# Grab data from the 4 sources
cli = self._parse_cli()
cloud = saltcloud.config.cloud_config(cli['cloud_config'])
opts = salt.config.master_config(cli['master_config'])
vms = saltcloud.config.vm_config(cli['vm_config'])
# Load the data in order
opts.update(cloud)
opts.update(cli)
opts['vm'] = vms
return opts
def _parse_cli(self):
'''
Parse the cli and return a dict of the options
'''
parser = optparse.OptionParser()
parser.add_option(
'--version',
dest='version',
default=False,
action='store_true',
help='Show program version number and exit')
parser.add_option('-p',
'--profile',
dest='profile',
default='',
help='Specify a profile to use for the vms')
parser.add_option('-m',
'--map',
dest='map',
default='',
help='Specify a cloud map file to use for deployment')
parser.add_option('-H',
'--hard',
dest='hard',
default=False,
action='store_true',
help=('Delete all vms that are not defined in the map file '
'CAUTION!!! This operation can irrevocably destroy vms!')
)
parser.add_option('-d',
'--destroy',
dest='destroy',
default=False,
action='store_true',
help='Specify a vm to destroy')
parser.add_option('-P',
'--parallel',
dest='parallel',
default=False,
action='store_true',
help='Build all of the specified virtual machines in parallel')
parser.add_option('-Q',
'--query',
dest='query',
default=False,
action='store_true',
help=('Execute a query and return some information about the '
'nodes running on configured cloud providers'))
parser.add_option('-F',
'--full-query',
dest='full_query',
default=False,
action='store_true',
help=('Execute a query and return all information about the '
'nodes running on configured cloud providers'))
parser.add_option('-S',
'--select-query',
dest='select_query',
default=False,
action='store_true',
help=('Execute a query and return select information about '
'the nodes running on configured cloud providers'))
parser.add_option('-l',
'--log-level',
dest='log_level',
default='warn',
help=("Console logging log level. One of 'all', 'garbage',"
"'trace', 'debug', 'info', 'warning', 'error', 'quiet'."
"For the log file setting see the configuration file."
"Default: 'warning'."))
parser.add_option('--list-locations',
dest='list_locations',
default=False,
help=('Display a list of locations available in configured '
'cloud providers. Pass the cloud provider that '
'available locations are desired on, aka "linode", '
'or pass "all" to list locations for all configured '
'cloud providers'))
parser.add_option('--list-images',
dest='list_images',
default=False,
help=('Display a list of images available in configured '
'cloud providers. Pass the cloud provider that '
'available images are desired on, aka "linode", '
'or pass "all" to list images for all configured '
'cloud providers'))
parser.add_option('--list-sizes',
dest='list_sizes',
default=False,
help=('Display a list of sizes available in configured '
'cloud providers. Pass the cloud provider that '
'available sizes are desired on, aka "AWS", '
'or pass "all" to list sizes for all configured '
'cloud providers'))
parser.add_option('-C',
'--cloud-config',
dest='cloud_config',
default='/etc/salt/cloud',
help='The location of the saltcloud config file')
parser.add_option('-M',
'--master-config',
dest='master_config',
default='/etc/salt/master',
help='The location of the salt master config file')
parser.add_option('-V',
'--profiles',
'--vm_config',
dest='vm_config',
default='/etc/salt/cloud.profiles',
help='The location of the saltcloud vm config file')
parser.add_option('--raw-out',
default=False,
action='store_true',
dest='raw_out',
help=('Print the output from the salt command in raw python '
'form, this is suitable for re-reading the output into '
'an executing python script with eval.'))
parser.add_option('--text-out',
default=False,
action='store_true',
dest='txt_out',
help=('Print the output from the salt command in the same '
'form the shell would.'))
parser.add_option('--yaml-out',
default=False,
action='store_true',
dest='yaml_out',
help='Print the output from the salt command in yaml.')
parser.add_option('--json-out',
default=False,
action='store_true',
dest='json_out',
help='Print the output from the salt command in json.')
parser.add_option('--no-color',
default=False,
action='store_true',
dest='no_color',
help='Disable all colored output')
options, args = parser.parse_args()
cli = {}
for k, v in options.__dict__.items():
if v is not None:
cli[k] = v
if args:
cli['names'] = args
return cli
class SaltCloud(parsers.SaltCloudParser):
def run(self):
'''
Exeute the salt cloud execution run
Execute the salt-cloud command line
'''
import salt.log
salt.log.setup_logfile_logger(
self.opts['log_file'], self.opts['log_level']
)
for name, level in self.opts['log_granular_levels'].iteritems():
salt.log.set_logger_level(name, level)
import logging
# If statement here for when cloud query is added
# Parse shell arguments
self.parse_args()
# Setup log file logging
self.setup_logfile_logger()
# Late imports so logging works as expected
import saltcloud.cloud
mapper = saltcloud.cloud.Map(self.opts)
mapper = saltcloud.cloud.Map(self.config)
if self.opts['query'] or self.opts['full_query'] or self.opts['select_query']:
query = 'list_nodes'
if self.opts['full_query']:
query = 'list_nodes_full'
elif self.opts['select_query']:
query = 'list_nodes_select'
query_map = {}
if self.opts['map']:
query_map = mapper.interpolated_map(query=query)
if self.selected_query_option is not None:
if self.options.map:
query_map = mapper.interpolated_map(
query=self.selected_query_option
)
else:
query_map = mapper.map_providers(query=query)
salt.output.display_output(query_map, '', self.opts)
query_map = mapper.map_providers(
query=self.selected_query_option
)
salt.output.display_output(query_map, '', self.config)
if self.opts['version']:
print VERSION
if self.opts['list_locations']:
if self.options.list_locations is not None:
saltcloud.output.double_layer(
mapper.location_list(self.opts['list_locations'])
mapper.location_list(self.options.list_locations)
)
if self.opts['list_images']:
self.exit(0)
if self.options.list_images is not None:
saltcloud.output.double_layer(
mapper.image_list(self.opts['list_images'])
mapper.image_list(self.options.list_images)
)
if self.opts['list_sizes']:
self.exit(0)
if self.options.list_sizes is not None:
saltcloud.output.double_layer(
mapper.size_list(self.opts['list_sizes'])
mapper.size_list(self.options.list_sizes)
)
elif self.opts['destroy'] and (self.opts.get('names') or self.opts['map']):
names = []
if self.opts['map']:
self.exit(0)
if self.options.destroy and (self.config.get('names', None) or
self.options.map):
if self.options.map:
names = mapper.delete_map(query='list_nodes')
else:
names = self.opts.get('names')
names = self.config.get('names', None)
mapper.destroy(names)
elif self.opts.get('names', False) and self.opts['profile']:
self.exit(0)
if self.options.profile and self.config.get('names', False):
mapper.run_profile()
elif self.opts['map'] and not (self.opts['query'] or self.opts['full_query'] or self.opts['destroy']):
self.exit(0)
if self.options.map and self.options.list_images is not None:
mapper.run_map()
self.exit(0)

46
saltcloud/clouds/openstack.py
View File

@ -40,6 +40,7 @@ import tempfile
import time
import sys
import logging
import socket
# Import libcloud
from libcloud.compute.types import Provider
@ -55,6 +56,7 @@ log = logging.getLogger(__name__)
# Some of the libcloud functions need to be in the same namespace as the
# functions defined in the module, so we create new function objects inside
# this module namespace
avail_locations = types.FunctionType(avail_locations.__code__, globals())
avail_images = types.FunctionType(avail_images.__code__, globals())
avail_sizes = types.FunctionType(avail_sizes.__code__, globals())
script = types.FunctionType(script.__code__, globals())
@ -69,7 +71,7 @@ def __virtual__():
'''
Set up the libcloud functions and check for OPENSTACK configs
'''
if 'OPENSTACK.user' in __opts__ and 'OPENSTACK.password' in __opts__:
if 'OPENSTACK.user' in __opts__:
log.debug('Loading Openstack cloud module')
return 'openstack'
return False
@ -101,6 +103,22 @@ def get_conn():
)
def preferred_ip(vm_, ips):
'''
Return the preferred Internet protocol. Either 'ipv4' (default) or 'ipv6'.
'''
proto = vm_.get('protocol', __opts__.get('OPENSTACK.protocol', 'ipv4'))
family = socket.AF_INET
if proto == 'ipv6':
family = socket.AF_INET6
for ip in ips:
try:
socket.inet_pton(family, ip)
return ip
except:
continue
return False
def ssh_interface(vm_):
'''
Return the ssh_interface type to connect to. Either 'public_ips' (default) or 'private_ips'.
@ -139,6 +157,7 @@ def create(vm_):
sys.stderr.write(err)
return False
if 'OPENSTACK.ssh_key_name' in __opts__:
kwargs['ex_keyname'] = __opts__['OPENSTACK.ssh_key_name']
try:
@ -154,9 +173,9 @@ def create(vm_):
not_ready = True
nr_count = 0
while not_ready:
print('Looking for IP addresses')
log.warn('Looking for IP addresses')
log.debug('Looking for IP addresses')
while not_ready:
nodelist = list_nodes()
private = nodelist[vm_['name']]['private_ips']
public = nodelist[vm_['name']]['public_ips']
@ -164,19 +183,22 @@ def create(vm_):
print('Private IPs returned, but not public... checking for misidentified IPs')
log.warn('Private IPs returned, but not public... checking for misidentified IPs')
for private_ip in private:
private_ip = preferred_ip(vm_, [private_ip])
if saltcloud.utils.is_public_ip(private_ip):
print('{0} is a public ip'.format(private_ip))
log.warn('{0} is a public ip'.format(private_ip))
data.public_ips.append(private_ip)
not_ready = False
else:
print('{0} is a private ip'.format(private_ip))
log.warn('{0} is a private ip'.format(private_ip))
if private_ip not in data.private_ips:
data.private_ips.append(private_ip)
if ssh_interface(vm_) == 'private_ips' and data.private_ips:
break
if public:
data.public_ips = public
not_ready = False
nr_count += 1
if nr_count > 50:
log.warn('Timed out waiting for a public ip, continuing anyway')
@ -184,9 +206,13 @@ def create(vm_):
time.sleep(1)
if ssh_interface(vm_) == 'private_ips':
ip_address = data.private_ips[0]
ip_address = preferred_ip(vm_, data.private_ips)
else:
ip_address = data.public_ips[0]
ip_address = preferred_ip(vm_, data.public_ips)
log.debug('Using IP address {0}'.format(ip_address))
if not ip_address:
raise
deployargs = {
'host': ip_address,
@ -199,14 +225,18 @@ def create(vm_):
deployargs['username'] = vm_['ssh_username']
else:
deployargs['username'] = 'root'
log.debug('Using {0} as SSH username'.format(deployargs['username']))
if 'OPENSTACK.ssh_key_file' in __opts__:
deployargs['key_filename'] = __opts__['OPENSTACK.ssh_key_file']
log.debug('Using {0} as SSH key file'.format(deployargs['key_filename']))
elif 'password' in data.extra:
deployargs['password'] = data.extra['password']
log.debug('Logging into SSH using password')
if 'sudo' in vm_:
deployargs['sudo'] = vm_['sudo']
log.debug('Running root commands using sudo')
deployed = saltcloud.utils.deploy_script(**deployargs)
if deployed:
@ -214,7 +244,7 @@ def create(vm_):
log.warn('Salt installed on {0}'.format(vm_['name']))
else:
print('Failed to start Salt on Cloud VM {0}'.format(vm_['name']))
log.warn('Failed to start Salt on Cloud VM {0}'.format(vm_['name']))
log.error('Failed to start Salt on Cloud VM {0}'.format(vm_['name']))
print('Created Cloud VM {0} with the following values:'.format(vm_['name']))
log.warn('Created Cloud VM {0} with the following values:'.format(vm_['name']))

1
saltcloud/clouds/rackspace.py
View File

@ -40,6 +40,7 @@ log = logging.getLogger(__name__)
# Some of the libcloud functions need to be in the same namespace as the
# functions defined in the module, so we create new function objects inside
# this module namespace
avail_locations = types.FunctionType(avail_locations.__code__, globals())
avail_images = types.FunctionType(avail_images.__code__, globals())
avail_sizes = types.FunctionType(avail_sizes.__code__, globals())
script = types.FunctionType(script.__code__, globals())

5
saltcloud/config.py
View File

@ -20,6 +20,10 @@ def cloud_config(path):
'ssh_auth': '',
'keysize': 4096,
'os': '',
# Logging defaults
'log_level': 'info',
'log_level_logfile': 'info',
'log_file': '/var/log/salt/cloud'
}
salt.config.load_config(opts, path, 'SALT_CLOUD_CONFIG')
@ -29,6 +33,7 @@ def cloud_config(path):
return opts
def vm_config(path):
'''
Read in the salt cloud vm config file

315
saltcloud/utils/parsers.py Normal file
View File

@ -0,0 +1,315 @@
# -*- coding: utf-8 -*-
# vim: sw=4 ts=4 fenc=utf-8
"""
:copyright: © 2012 UfSoft.org - :email:`Pedro Algarvio (pedro@algarvio.me)`
:license: Apache 2.0, see LICENSE for more details
"""
# Import python libs
import os
import sys
import optparse
from functools import partial
# Import salt libs
import salt.config
from salt.utils import parsers
# Import salt cloud libs
from saltcloud import config, version
class CloudConfigMixIn(object):
__metaclass__ = parsers.MixInMeta
_mixin_prio_ = -1000 # First options seen
config = {'log_level': None}
def _mixin_setup(self):
group = self.config_group = optparse.OptionGroup(
self,
"Configuration Options",
# Include description here as a string
)
group.add_option(
'-C', '--cloud-config',
default='/etc/salt/cloud',
help='The location of the saltcloud config file. Default: %default'
)
group.add_option(
'-M', '--master-config',
default='/etc/salt/master',
help=('The location of the salt master config file. Default: '
'%default')
)
group.add_option(
'-V', '--profiles', '--vm_config',
dest='vm_config',
default='/etc/salt/cloud.profiles',
help=('The location of the saltcloud vm config file. Default: '
'%default')
)
self.add_option_group(group)
def __assure_absolute_paths(self, name):
# Need to check if file exists?
optvalue = getattr(self.options, name)
if optvalue:
setattr(self.options, name, os.path.abspath(optvalue))
def __merge_config_with_cli(self, *args):
# Taken from https://github.com/saltstack/salt/blob/develop/salt/utils/parsers.py#L175
# Merge parser options
for option in self.option_list:
if option.dest is None:
# --version does not have dest attribute set for example.
# All options defined by us, even if not explicitly(by kwarg),
# will have the dest attribute set
continue
# Get the passed value from shell. If empty get the default one
default = self.defaults.get(option.dest)
value = getattr(self.options, option.dest, default)
if option.dest not in self.config:
# There's no value in the configuration file
if value is not None:
# There's an actual value, add it to the config
self.config[option.dest] = value
elif value is not None and value != default:
# Only set the value in the config file IF it's not the default
# value, this allows to tweak settings on the configuration
# files bypassing the shell option flags
self.config[option.dest] = value
# Merge parser group options if any
for group in self.option_groups:
for option in group.option_list:
if option.dest is None:
continue
# Get the passed value from shell. If empty get the default one
default = self.defaults.get(option.dest)
value = getattr(self.options, option.dest, default)
if option.dest not in self.config:
# There's no value in the configuration file
if value is not None:
# There's an actual value, add it to the config
self.config[option.dest] = value
else:
if value is not None and value != default:
# Only set the value in the config file IF it's not the
# default value, this allows to tweak settings on the
# configuration files bypassing the shell option flags
self.config[option.dest] = value
def _mixin_after_parsed(self):
for option in self.config_group.option_list:
if option.dest is None:
# This should not happen.
#
# --version does not have dest attribute set for example.
# All options defined by us, even if not explicitly(by kwarg),
# will have the dest attribute set
continue
self.__assure_absolute_paths(option.dest)
# Grab data from the 4 sources
# 1st - Master config
self.config.update(
salt.config.master_config(self.options.master_config)
)
# 2nd Override master config with salt-cloud config
self.config.update(config.cloud_config(self.options.cloud_config))
## Fix conf_file set on master config so that salt parsers don't fail
#self.config['conf_file'] = self.options.cloud_config
# 3rd - Override config with cli options
self.__merge_config_with_cli()
# 4th - Include vm config
self.config['vm'] = config.vm_config(self.options.vm_config)
class ExecutionOptionsMixIn(object):
__metaclass__ = parsers.MixInMeta
_mixin_prio_ = 10
def _mixin_setup(self):
group = self.execution_group = optparse.OptionGroup(
self,
"Execution Options",
# Include description here as a string
)
group.add_option(
'-p', '--profile',
default='',
help='Specify a profile to use for the vms'
)
group.add_option(
'-m', '--map',
default='',
help='Specify a cloud map file to use for deployment'
)
group.add_option(
'-H', '--hard',
default=False,
action='store_true',
help=('Delete all vms that are not defined in the map file '
'CAUTION!!! This operation can irrevocably destroy vms!')
)
group.add_option(
'-d', '--destroy',
default=False,
action='store_true',
help='Specify a vm to destroy'
)
group.add_option(
'-P', '--parallel',
default=False,
action='store_true',
help='Build all of the specified virtual machines in parallel'
)
self.add_option_group(group)
class CloudQueriesMixIn(object):
__metaclass__ = parsers.MixInMeta
_mixin_prio_ = 20
selected_query_option = None
def _mixin_setup(self):
group = self.cloud_queries_group = optparse.OptionGroup(
self,
"Query Options",
# Include description here as a string
)
group.add_option(
'-Q', '--query',
default=False,
action='store_true',
help=('Execute a query and return some information about the '
'nodes running on configured cloud providers')
)
group.add_option(
'-F', '--full-query',
default=False,
action='store_true',
help=('Execute a query and return all information about the '
'nodes running on configured cloud providers')
)
group.add_option(
'-S', '--select-query',
default=False,
action='store_true',
help=('Execute a query and return select information about '
'the nodes running on configured cloud providers')
)
self.add_option_group(group)
self._create_process_functions()
def _create_process_functions(self):
for option in self.cloud_queries_group.option_list:
def process(opt):
if getattr(self.options, opt.dest):
query = 'list_nodes'
if opt.dest == 'full_query':
query += '_full'
elif opt.dest == 'select_query':
query += '_select'
self.selected_query_option = query
funcname = 'process_{0}'.format(option.dest)
if not hasattr(self, funcname):
setattr(self, funcname, partial(process, option))
def _mixin_after_parsed(self):
group_options_selected = filter(
lambda option: getattr(self.options, option.dest) is True,
self.cloud_queries_group.option_list
)
if len(group_options_selected) > 1:
self.error(
"The options {0} are mutually exclusive. Please only choose "
"one of them".format('/'.join([
option.get_opt_string() for option in
group_options_selected
]))
)
self.config['selected_query_option'] = self.selected_query_option
class CloudProvidersListsMixIn(object):
__metaclass__ = parsers.MixInMeta
_mixin_prio_ = 30
def _mixin_setup(self):
group = self.providers_listings_group = optparse.OptionGroup(
self,
"Cloud Providers Listings",
# Include description here as a string
)
group.add_option(
'--list-locations',
default=None,
help=('Display a list of locations available in configured cloud '
'providers. Pass the cloud provider that available '
'locations are desired on, aka "linode", or pass "all" to '
'list locations for all configured cloud providers')
)
group.add_option(
'--list-images',
default=None,
help=('Display a list of images available in configured cloud '
'providers. Pass the cloud provider that available images '
'are desired on, aka "linode", or pass "all" to list images '
'for all configured cloud providers')
)
group.add_option(
'--list-sizes',
default=None,
help=('Display a list of sizes available in configured cloud '
'providers. Pass the cloud provider that available sizes '
'are desired on, aka "AWS", or pass "all" to list sizes '
'for all configured cloud providers')
)
self.add_option_group(group)
def _mixin_after_parsed(self):
list_options_selected = filter(
lambda option: getattr(self.options, option.dest) is True,
self.providers_listings_group.option_list
)
if len(list_options_selected) > 1:
self.error(
"The options {0} are mutually exclusive. Please only choose "
"one of them".format('/'.join([
option.get_opt_string() for option in
list_options_selected
]))
)
class SaltCloudParser(parsers.OptionParser,
parsers.LogLevelMixIn,
parsers.OutputOptionsWithTextMixIn,
CloudConfigMixIn,
CloudQueriesMixIn,
ExecutionOptionsMixIn,
CloudProvidersListsMixIn):
__metaclass__ = parsers.OptionParserMeta
_default_logging_level_ = "info"
VERSION = version.__version__
def print_versions_report(self, file=sys.stdout):
print >> file, '\n'.join(version.versions_report())
self.exit()
def _mixin_after_parsed(self):
if self.args:
self.config['names'] = self.args

30
saltcloud/version.py
View File

@ -1,2 +1,32 @@
import sys
__version_info__ = (0, 8, 1)
__version__ = '.'.join(map(str, __version_info__))
def versions_report():
libs = (
("Apache Libcloud", "libcloud", "__version__"),
("Paramiko", "paramiko", "__version__"),
("PyYAML", "yaml", "__version__"),
)
padding = len(max([lib[0] for lib in libs], key=len)) + 1
fmt = '{0:>{pad}}: {1}'
yield fmt.format("Salt", __version__, pad=padding)
yield fmt.format(
"Python", sys.version.rsplit('\n')[0].strip(), pad=padding
)
for name, imp, attr in libs:
try:
imp = __import__(imp)
version = getattr(imp, attr)
if not isinstance(version, basestring):
version = '.'.join(map(str, version))
yield fmt.format(name, version, pad=padding)
except ImportError:
yield fmt.format(name, "not installed", pad=padding)