From 8dd88066f7211f842d9640ad38bcbb9e4bbf906c Mon Sep 17 00:00:00 2001 From: Colin Johnson Date: Mon, 4 Aug 2014 00:04:20 +0000 Subject: [PATCH] boto_asg now support use of security group names in launch configs --- salt/modules/boto_secgroup.py | 37 ++++++ salt/modules/boto_vpc.py | 169 ++++++++++++++++++++++++++++ salt/states/boto_asg.py | 58 +++++++++- tests/unit/modules/boto_secgroup.py | 83 +++++++++++++- tests/unit/modules/boto_vpc.py | 100 ++++++++++++++++ 5 files changed, 443 insertions(+), 4 deletions(-) create mode 100644 salt/modules/boto_vpc.py create mode 100644 tests/unit/modules/boto_vpc.py diff --git a/salt/modules/boto_secgroup.py b/salt/modules/boto_secgroup.py index 344f8d95af..63796c0581 100644 --- a/salt/modules/boto_secgroup.py +++ b/salt/modules/boto_secgroup.py @@ -105,6 +105,43 @@ def _split_rules(rules): return split +def get_group_id(name, vpc_id=None, region=None, key=None, keyid=None, profile=None): + ''' + Get a Group ID given a Group Name or Group Name and VPC ID + + CLI example:: + + salt myminion boto_secgroup.get_group_id mysecgroup + + ''' + conn = _get_conn(region, key, keyid, profile) + if not conn: + return False + if vpc_id is None: + logging.debug('getting group_id for {0}'.format(name)) + group_filter = {'group-name': name} + filtered_groups = conn.get_all_security_groups(filters=group_filter) + # security groups can have the same name if groups exist in both + # EC2-Classic and EC2-VPC + # iterate through groups to ensure we return the EC2-Classic + # security group + for group in filtered_groups: + # a group in EC2-Classic will have vpc_id set to None + if group.vpc_id is None: + return group.id + return False + elif vpc_id: + logging.debug('getting group_id for {0} in vpc_id {1}'.format(name, vpc_id)) + group_filter = {'group-name': name, 'vpc_id': vpc_id} + filtered_groups = conn.get_all_security_groups(filters=group_filter) + if len(filtered_groups) == 1: + return filtered_groups[0].id + else: + return False + else: + return False + + def get_config(name=None, group_id=None, region=None, key=None, keyid=None, profile=None): ''' diff --git a/salt/modules/boto_vpc.py b/salt/modules/boto_vpc.py new file mode 100644 index 0000000000..4a78ab8e86 --- /dev/null +++ b/salt/modules/boto_vpc.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- +''' +Connection module for Amazon VPC + +.. versionadded:: Helium + +:configuration: This module accepts explicit autoscale credentials but can also + utilize IAM roles assigned to the instance trough Instance Profiles. + Dynamic credentials are then automatically obtained from AWS API and no + further configuration is necessary. More Information available at:: + + http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + + If IAM roles are not used you need to specify them either in a pillar or + in the minion's config file:: + + asg.keyid: GKTADJGHEIQSXMKKRBJ08H + asg.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + + A region may also be specified in the configuration:: + + asg.region: us-east-1 + + If a region is not specified, the default is us-east-1. + + It's also possible to specify key, keyid and region via a profile, either + as a passed in dict, or as a string to pull from pillars or minion config: + + myprofile: + keyid: GKTADJGHEIQSXMKKRBJ08H + key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + region: us-east-1 + +:depends: boto + +''' + +# Import Python libs +import logging + +log = logging.getLogger(__name__) + +# Import third party libs +try: + import boto + import boto.vpc + logging.getLogger('boto').setLevel(logging.CRITICAL) + HAS_BOTO = True +except ImportError: + HAS_BOTO = False + +from salt._compat import string_types + + +def __virtual__(): + ''' + Only load if boto libraries exist. + ''' + if not HAS_BOTO: + return False + return True + + +def get_subnet_association(subnets, region=None, key=None, keyid=None, + profile=None): + ''' + Given a subnet (aka: a vpc zone identifier) or list of subnets, returns + vpc association. + + Returns a VPC ID if the given subnets are associated with the same VPC ID. + Returns False on an error or if the given subnets are associated with + different VPC IDs. + + CLI Examples:: + + .. code-block:: bash + + salt myminion boto_vpc.get_subnet_association subnet-61b47516 + + .. code-block:: bash + + salt myminion boto_vpc.get_subnet_association ['subnet-61b47516','subnet-2cb9785b'] + + ''' + conn = _get_conn(region, key, keyid, profile) + if not conn: + return False + try: + # subnet_ids=subnets can accept either a string or a list + subnets = conn.get_all_subnets(subnet_ids=subnets) + except boto.exception.BotoServerError as e: + log.debug(e) + return False + # using a set to store vpc_ids - the use of set prevents duplicate + # vpc_id values + vpc_ids = set() + for subnet in subnets: + log.debug('examining subnet id: {0} for vpc_id'.format(subnet.id)) + if subnet in subnets: + log.debug('subnet id: {0} is associated with vpc id: {1}' + .format(subnet.id, subnet.vpc_id)) + vpc_ids.add(subnet.vpc_id) + if len(vpc_ids) == 1: + vpc_id = vpc_ids.pop() + log.debug('all subnets are associated with vpc id: {0}'.format(vpc_id)) + return vpc_id + else: + log.debug('given subnets are associated with fewer than 1 or greater' + ' than 1 subnets') + return False + + +def exists(vpc_id, region=None, key=None, keyid=None, profile=None): + ''' + Given a VPC ID, check to see if the given VPC ID exists. + + Returns True if the given VPC ID exists and returns False if the given + VPC ID does not exist. + + CLI example:: + + .. code-block:: bash + + salt myminion boto_vpc.exists myvpc + + ''' + conn = _get_conn(region, key, keyid, profile) + if not conn: + return False + try: + conn.get_all_vpcs(vpc_ids=[vpc_id]) + return True + except boto.exception.BotoServerError as e: + log.debug(e) + return False + + +def _get_conn(region, key, keyid, profile): + ''' + Get a boto connection to vpc. + ''' + if profile: + if isinstance(profile, string_types): + _profile = __salt__['config.option'](profile) + elif isinstance(profile, dict): + _profile = profile + key = _profile.get('key', None) + keyid = _profile.get('keyid', None) + region = _profile.get('region', None) + + if not region and __salt__['config.option']('vpc.region'): + region = __salt__['config.option']('vpc.region') + + if not region: + region = 'us-east-1' + + if not key and __salt__['config.option']('vpc.key'): + key = __salt__['config.option']('vpc.key') + if not keyid and __salt__['config.option']('vpc.keyid'): + keyid = __salt__['config.option']('vpc.keyid') + + try: + conn = boto.vpc.connect_to_region(region, aws_access_key_id=keyid, + aws_secret_access_key=key) + except boto.exception.NoAuthHandlerFound: + log.error('No authentication credentials found when attempting to' + ' make boto autoscale connection.') + return None + return conn diff --git a/salt/states/boto_asg.py b/salt/states/boto_asg.py index 59baa00ebe..06013f7d27 100644 --- a/salt/states/boto_asg.py +++ b/salt/states/boto_asg.py @@ -101,7 +101,12 @@ as a passed in dict, or as a string to pull from pillars or minion config: - force: True ''' +# Import Python libs import hashlib +import logging +import re + +log = logging.getLogger(__name__) def __virtual__(): @@ -221,6 +226,14 @@ def present( that contains a dict with region, key and keyid. ''' ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} + if vpc_zone_identifier: + vpc_id = __salt__['boto_vpc.get_subnet_association'](vpc_zone_identifier, region, key, keyid, profile) + log.debug('Auto Scaling Group {0} is associated with VPC ID {1}' + .format(name, vpc_id)) + else: + vpc_id = None + log.debug('Auto Scaling Group {0} has no VPC Association' + .format(name)) # if launch_config is defined, manage the launch config first. # hash the launch_config dict to create a unique name suffix and then # ensure it is present @@ -233,6 +246,21 @@ def present( 'keyid': keyid, 'profile': profile } + + if vpc_id: + log.debug('Auto Scaling Group {0} is a associated with a vpc') + # locate the security groups attribute of a launch config + sg_index = None + for index, item in enumerate(launch_config): + if 'security_groups' in item: + sg_index = index + break + # if security groups exist within launch_config then convert + # to group ids + if sg_index: + log.debug('security group associations found in launch config') + launch_config[sg_index]['security_groups'] = _convert_to_group_ids(launch_config[sg_index]['security_groups'], vpc_id, region, key, keyid, profile) + for d in launch_config: args.update(d) lc_ret = __salt__["state.single"]('boto_lc.present', **args) @@ -300,14 +328,16 @@ def present( # ensure that we delete scaling_policies if none are specified if scaling_policies is None: config["scaling_policies"] = [] - for key, value in config.iteritems(): + # note: do not loop using "key, value" - this can modify the value of + # the aws access key + for asg_property, value in config.iteritems(): # Only modify values being specified; introspection is difficult # otherwise since it's hard to track default values, which will # always be returned from AWS. if value is None: continue - if key in asg: - _value = asg[key] + if asg_property in asg: + _value = asg[asg_property] if not _recursive_compare(value, _value): need_update = True break @@ -351,6 +381,28 @@ def present( return ret +def _convert_to_group_ids(groups, vpc_id, region, key, keyid, profile): + ''' + given a list of security groups _convert_to_group_ids will convert all + list items in the given list to security group ids + ''' + log.debug('security group contents {0} pre-conversion'.format(groups)) + group_ids = [] + for group in groups: + if re.match('sg-.*', group): + log.debug('group {0} is a group id. get_group_id not called.' + .format(group)) + group_ids.append(group) + else: + log.debug('calling boto_secgroup.get_group_id for' + ' group name {0}'.format(group)) + group_id = __salt__['boto_secgroup.get_group_id'](group, vpc_id, region, key, keyid, profile) + log.debug('group name {0} has group id {1}'.format(group, group_id)) + group_ids.append(str(group_id)) + log.debug('security group contents {0} post-conversion'.format(group_ids)) + return group_ids + + def _recursive_compare(v1, v2): "return v1 == v2. compares list, dict, OrderedDict, recursively" if isinstance(v1, list): diff --git a/tests/unit/modules/boto_secgroup.py b/tests/unit/modules/boto_secgroup.py index 3c52a9bc47..a906a54743 100644 --- a/tests/unit/modules/boto_secgroup.py +++ b/tests/unit/modules/boto_secgroup.py @@ -1,13 +1,47 @@ # -*- coding: utf-8 -*- # import Python Libs +import random +import string from collections import OrderedDict +# import Python Third Party Libs +try: + import boto + from moto import mock_ec2 + missing_requirements = False + missing_requirements_msg = '' +except ImportError: + missing_requirements = True + missing_requirements_msg = 'boto and moto modules required for test.' + + def mock_ec2(self): + ''' + if the mock_ec2 function is not available due to import failure + this replaces the decorated function with stub_function. + Allows boto_secgroup unit tests to use the @mock_ec2 decorator + without a "NameError: name 'mock_ec2' is not defined" error. + ''' + def stub_function(self): + pass + return stub_function + # Import Salt Libs from salt.modules import boto_secgroup # Import Salt Testing Libs -from salttesting import TestCase +from salttesting import skipIf, TestCase + +vpc_id = 'vpc-mjm05d27' +region = 'us-east-1' +access_key = 'GKTADJGHEIQSXMKKRBJ08H' +secret_key = 'askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs' +conn_parameters = {'region': region, 'key': access_key, 'keyid': secret_key, 'profile': {}} + + +def _random_group_name(): + group_name = 'boto_secgroup-{0}'.format(''.join((random.choice(string.ascii_lowercase)) for char in range(12))) + return group_name class Boto_SecgroupTestCase(TestCase): @@ -24,6 +58,53 @@ class Boto_SecgroupTestCase(TestCase): {'to_port': 80, 'from_port': 80, 'ip_protocol': u'tcp', 'cidr_ip': u'0.0.0.0/0'}] self.assertEqual(boto_secgroup._split_rules(rules), split_rules) + @skipIf(missing_requirements, missing_requirements_msg) + # test can be enabled if running version of moto contains commit id + # https://github.com/spulec/moto/commit/cc0166964371f7b5247a49d45637a8f936ccbe6f + @skipIf(True, 'test skipped because of' + ' https://github.com/spulec/moto/issues/152') + @mock_ec2 + def test_get_group_id_ec2_classic(self): + ''' + tests that given a name of a group in EC2-Classic that the correct + group id will be retreived + ''' + group_name = _random_group_name() + group_description = 'test_get_group_id_ec2_classic' + conn = boto.ec2.connect_to_region(region) + group_classic = conn.create_security_group(name=group_name, + description=group_description) + # note that the vpc_id does not need to be created in order to create + # a security group within the vpc when using moto + group_vpc = conn.create_security_group(name=group_name, + description=group_description, + vpc_id=vpc_id) + retreived_group_id = boto_secgroup.get_group_id(group_name, + **conn_parameters) + self.assertEqual(group_classic.id, retreived_group_id) + + @skipIf(True, 'test skipped because moto does not yet support group' + ' filters https://github.com/spulec/moto/issues/154') + @mock_ec2 + def test_get_group_id_ec2_vpc(self): + ''' + tests that given a name of a group in EC2-VPC that the correct + group id will be retreived + ''' + group_name = _random_group_name() + group_description = 'test_get_group_id_ec2_vpc' + conn = boto.ec2.connect_to_region(region) + group_classic = conn.create_security_group(name=group_name, + description=group_description) + # note that the vpc_id does not need to be created in order to create + # a security group within the vpc when using moto + group_vpc = conn.create_security_group(name=group_name, + description=group_description, + vpc_id=vpc_id) + retreived_group_id = boto_secgroup.get_group_id(group_name, group_vpc, + **conn_parameters) + self.assertEqual(group_vpc.id, retreived_group_id) + if __name__ == '__main__': from integration import run_tests diff --git a/tests/unit/modules/boto_vpc.py b/tests/unit/modules/boto_vpc.py new file mode 100644 index 0000000000..ba13d2bf6b --- /dev/null +++ b/tests/unit/modules/boto_vpc.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# import Python Third Party Libs +try: + import boto + from moto import mock_ec2 + missing_requirements = False + missing_requirements_msg = '' +except ImportError: + missing_requirements = True + missing_requirements_msg = 'boto and moto modules required for test.' + + def mock_ec2(self): + ''' + if the mock_ec2 function is not available due to import failure + this replaces the decorated function with stub_function. + Allows boto_vpc unit tests to use the @mock_ec2 decorator + without a "NameError: name 'mock_ec2' is not defined" error. + ''' + def stub_function(self): + pass + return stub_function + +# Import Salt Libs +from salt.modules import boto_vpc + +# Import Salt Testing Libs +from salttesting import skipIf, TestCase + +region = 'us-east-1' +access_key = 'GKTADJGHEIQSXMKKRBJ08H' +secret_key = 'askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs' +conn_parameters = {'region': region, 'key': access_key, 'keyid': secret_key, 'profile': {}} + + +class Boto_VpcTestCase(TestCase): + ''' + TestCase for salt.modules.boto_vpc module + ''' + @skipIf(missing_requirements, missing_requirements_msg) + @mock_ec2 + def test_get_subnet_association_single_subnet(self): + ''' + tests that given multiple subnet ids in the same VPC that the VPC ID is + returned. The test is valuable because it uses a string as an argument + to subnets as opposed to a list. + ''' + conn = boto.vpc.connect_to_region(region) + vpc = conn.create_vpc('10.0.0.0/24') + subnet = conn.create_subnet(vpc.id, '10.0.0.0/25') + subnet_assocation = boto_vpc.get_subnet_association(subnets=subnet.id, + **conn_parameters) + self.assertEqual(vpc.id, subnet_assocation) + + @skipIf(missing_requirements, missing_requirements_msg) + @mock_ec2 + def test_get_subnet_association_multiple_subnets_same_vpc(self): + ''' + tests that given multiple subnet ids in the same VPC that the VPC ID is + returned. + ''' + conn = boto.vpc.connect_to_region(region) + vpc = conn.create_vpc('10.0.0.0/24') + subnet_a = conn.create_subnet(vpc.id, '10.0.0.0/25') + subnet_b = conn.create_subnet(vpc.id, '10.0.0.128/25') + subnet_assocation = boto_vpc.get_subnet_association([subnet_a.id, subnet_b.id], + **conn_parameters) + self.assertEqual(vpc.id, subnet_assocation) + + @skipIf(missing_requirements, missing_requirements_msg) + @mock_ec2 + def test_get_subnet_association_multiple_subnets_different_vpc(self): + ''' + tests that given multiple subnet ids in different VPCs that False is + returned. + ''' + conn = boto.vpc.connect_to_region(region) + vpc_a = conn.create_vpc('10.0.0.0/24') + vpc_b = conn.create_vpc('10.0.0.0/24') + subnet_a = conn.create_subnet(vpc_a.id, '10.0.0.0/24') + subnet_b = conn.create_subnet(vpc_b.id, '10.0.0.0/24') + subnet_assocation = boto_vpc.get_subnet_association([subnet_a.id, subnet_b.id], + **conn_parameters) + self.assertFalse(subnet_assocation) + + @skipIf(missing_requirements, missing_requirements_msg) + @mock_ec2 + def test_exists_true(self): + ''' + tests True existence of a VPC. + ''' + conn = boto.vpc.connect_to_region(region) + vpc = conn.create_vpc('10.0.0.0/24') + vpc_exists = boto_vpc.exists(vpc.id, **conn_parameters) + self.assertTrue(vpc_exists) + + +if __name__ == '__main__': + from integration import run_tests + run_tests(Boto_VpcTestCase)