add support to s3 for aws role assumption

salt/modules/s3.py

@@ -19,6 +19,10 @@ Connection module for Amazon S3
 
         s3.service_url: s3.amazonaws.com
 
+    A role_arn may also be specified in the configuration::
+
+        s3.role_arn: arn:aws:iam::111111111111:role/my-role-to-assume
+
     If a service_url is not specified, the default is s3.amazonaws.com. This
     may appear in various documentation as an "endpoint". A comprehensive list
     for Amazon S3 may be found at::
@@ -67,7 +71,7 @@ def __virtual__():
 
 
 def delete(bucket, path=None, action=None, key=None, keyid=None,
-           service_url=None, verify_ssl=None, location=None):
+           service_url=None, verify_ssl=None, location=None, role_arn=None):
     '''
     Delete a bucket, or delete an object from a bucket.
 
@@ -79,8 +83,8 @@ def delete(bucket, path=None, action=None, key=None, keyid=None,
 
         salt myminion s3.delete mybucket remoteobject
     '''
-    key, keyid, service_url, verify_ssl, location = _get_key(
-        key, keyid, service_url, verify_ssl, location)
+    key, keyid, service_url, verify_ssl, location, role_arn = _get_key(
+        key, keyid, service_url, verify_ssl, location, role_arn)
 
     return salt.utils.s3.query(method='DELETE',
                                bucket=bucket,
@@ -90,12 +94,13 @@ def delete(bucket, path=None, action=None, key=None, keyid=None,
                                keyid=keyid,
                                service_url=service_url,
                                verify_ssl=verify_ssl,
-                               location=location)
+                               location=location,
+                               role_arn=role_arn)
 
 
 def get(bucket=None, path=None, return_bin=False, action=None,
         local_file=None, key=None, keyid=None, service_url=None,
-        verify_ssl=None, location=None):
+        verify_ssl=None, location=None, role_arn=None):
     '''
     List the contents of a bucket, or return an object from a bucket. Set
     return_bin to True in order to retrieve an object wholesale. Otherwise,
@@ -147,8 +152,8 @@ def get(bucket=None, path=None, return_bin=False, action=None,
 
         salt myminion s3.get mybucket myfile.png action=acl
     '''
-    key, keyid, service_url, verify_ssl, location = _get_key(
-        key, keyid, service_url, verify_ssl, location)
+    key, keyid, service_url, verify_ssl, location, role_arn = _get_key(
+        key, keyid, service_url, verify_ssl, location, role_arn)
 
     return salt.utils.s3.query(method='GET',
                                bucket=bucket,
@@ -160,11 +165,12 @@ def get(bucket=None, path=None, return_bin=False, action=None,
                                keyid=keyid,
                                service_url=service_url,
                                verify_ssl=verify_ssl,
-                               location=location)
+                               location=location,
+                               role_arn=role_arn)
 
 
 def head(bucket, path=None, key=None, keyid=None, service_url=None,
-         verify_ssl=None, location=None):
+         verify_ssl=None, location=None, role_arn=None):
     '''
     Return the metadata for a bucket, or an object in a bucket.
 
@@ -175,8 +181,8 @@ def head(bucket, path=None, key=None, keyid=None, service_url=None,
         salt myminion s3.head mybucket
         salt myminion s3.head mybucket myfile.png
     '''
-    key, keyid, service_url, verify_ssl, location = _get_key(
-        key, keyid, service_url, verify_ssl, location)
+    key, keyid, service_url, verify_ssl, location, role_arn = _get_key(
+        key, keyid, service_url, verify_ssl, location, role_arn)
 
     return salt.utils.s3.query(method='HEAD',
                                bucket=bucket,
@@ -186,11 +192,13 @@ def head(bucket, path=None, key=None, keyid=None, service_url=None,
                                service_url=service_url,
                                verify_ssl=verify_ssl,
                                location=location,
-                               full_headers=True)
+                               full_headers=True,
+                               role_arn=role_arn)
 
 
 def put(bucket, path=None, return_bin=False, action=None, local_file=None,
-        key=None, keyid=None, service_url=None, verify_ssl=None, location=None):
+        key=None, keyid=None, service_url=None, verify_ssl=None, location=None,
+        role_arn=None):
     '''
     Create a new bucket, or upload an object to a bucket.
 
@@ -206,8 +214,8 @@ def put(bucket, path=None, return_bin=False, action=None, local_file=None,
 
         salt myminion s3.put mybucket remotepath local_file=/path/to/file
     '''
-    key, keyid, service_url, verify_ssl, location = _get_key(
-        key, keyid, service_url, verify_ssl, location)
+    key, keyid, service_url, verify_ssl, location, role_arn = _get_key(
+        key, keyid, service_url, verify_ssl, location, role_arn)
 
     return salt.utils.s3.query(method='PUT',
                                bucket=bucket,
@@ -219,10 +227,11 @@ def put(bucket, path=None, return_bin=False, action=None, local_file=None,
                                keyid=keyid,
                                service_url=service_url,
                                verify_ssl=verify_ssl,
-                               location=location)
+                               location=location,
+                               role_arn=role_arn)
 
 
-def _get_key(key, keyid, service_url, verify_ssl, location):
+def _get_key(key, keyid, service_url, verify_ssl, location, role_arn):
     '''
     Examine the keys, and populate as necessary
     '''
@@ -247,4 +256,7 @@ def _get_key(key, keyid, service_url, verify_ssl, location):
     if location is None and __salt__['config.option']('s3.location') is not None:
         location = __salt__['config.option']('s3.location')
 
-    return key, keyid, service_url, verify_ssl, location
+    if role_arn is None and __salt__['config.option']('s3.role_arn') is not None:
+        role_arn = __salt__['config.option']('s3.role_arn')
+
+    return key, keyid, service_url, verify_ssl, location, role_arn

salt/utils/aws.py

@@ -14,10 +14,12 @@ from __future__ import absolute_import
 import sys
 import time
 import binascii
-import datetime
+from datetime import datetime
 import hashlib
 import hmac
 import logging
+import salt.config
+import re
 
 # Import Salt libs
 import salt.utils.xmlutil as xml
@@ -53,6 +55,7 @@ __SecretAccessKey__ = ''
 __Token__ = ''
 __Expiration__ = ''
 __Location__ = ''
+__AssumeCache__ = {}
 
 
 def creds(provider):
@@ -70,7 +73,7 @@ def creds(provider):
     if provider['id'] == IROLE_CODE or provider['key'] == IROLE_CODE:
         # Check to see if we have cache credentials that are still good
         if __Expiration__ != '':
-            timenow = datetime.datetime.utcnow()
+            timenow = datetime.utcnow()
             timestamp = timenow.strftime('%Y-%m-%dT%H:%M:%SZ')
             if timestamp < __Expiration__:
                 # Current timestamp less than expiration fo cached credentials
@@ -114,7 +117,7 @@ def sig2(method, endpoint, params, provider, aws_api_version):
 
     http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
     '''
-    timenow = datetime.datetime.utcnow()
+    timenow = datetime.utcnow()
     timestamp = timenow.strftime('%Y-%m-%dT%H:%M:%SZ')
 
     # Retrieve access credentials from meta-data, or use provided
@@ -147,9 +150,58 @@ def sig2(method, endpoint, params, provider, aws_api_version):
     return params_with_headers
 
 
+def assumed_creds(prov_dict, role_arn, location=None):
+    valid_session_name_re = re.compile("[^a-z0-9A-Z+=,.@-]")
+
+    now = (datetime.utcnow() - datetime(1970, 1, 1)).total_seconds()
+    for key, creds in __AssumeCache__.items():
+        if (creds["Expiration"] - now) <= 120:
+            __AssumeCache__.delete(key)
+
+    if role_arn in __AssumeCache__:
+        c = __AssumeCache__[role_arn]
+        return c["AccessKeyId"], c["SecretAccessKey"], c["SessionToken"]
+
+    version = "2011-06-15"
+    session_name = valid_session_name_re.sub('', salt.config.get_id({"root_dir": None})[0])[0:63]
+
+    headers, requesturl = sig4(
+        'GET',
+        'sts.amazonaws.com',
+        params={
+            "Version": version,
+            "Action": "AssumeRole",
+            "RoleSessionName": session_name,
+            "RoleArn": role_arn,
+            "Policy": '{"Version":"2012-10-17","Statement":[{"Sid":"Stmt1", "Effect":"Allow","Action":"*","Resource":"*"}]}',
+            "DurationSeconds": "3600"
+        },
+        aws_api_version=version,
+        data='',
+        uri='/',
+        prov_dict=prov_dict,
+        product='sts',
+        location=location,
+        requesturl="https://sts.amazonaws.com/"
+    )
+    headers["Accept"] = "application/json"
+    result = requests.request('GET', requesturl, headers=headers,
+                              data='',
+                              verify=True)
+
+    if result.status_code >= 400:
+        LOG.info('AssumeRole response: {0}'.format(result.content))
+    result.raise_for_status()
+    resp = result.json()
+
+    data = resp["AssumeRoleResponse"]["AssumeRoleResult"]["Credentials"]
+    __AssumeCache__[role_arn] = data
+    return data["AccessKeyId"], data["SecretAccessKey"], data["SessionToken"]
+
+
 def sig4(method, endpoint, params, prov_dict,
          aws_api_version=DEFAULT_AWS_API_VERSION, location=None,
-         product='ec2', uri='/', requesturl=None, data=''):
+         product='ec2', uri='/', requesturl=None, data='', role_arn=None):
     '''
     Sign a query against AWS services using Signature Version 4 Signing
     Process. This is documented at:
@@ -158,10 +210,13 @@ def sig4(method, endpoint, params, prov_dict,
     http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
     http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
     '''
-    timenow = datetime.datetime.utcnow()
+    timenow = datetime.utcnow()
 
     # Retrieve access credentials from meta-data, or use provided
-    access_key_id, secret_access_key, token = creds(prov_dict)
+    if role_arn is None:
+        access_key_id, secret_access_key, token = creds(prov_dict)
+    else:
+        access_key_id, secret_access_key, token = assumed_creds(prov_dict, role_arn, location=location)
 
     if location is None:
         location = get_region_from_metadata()

salt/utils/s3.py

@@ -28,7 +28,7 @@ log = logging.getLogger(__name__)
 def query(key, keyid, method='GET', params=None, headers=None,
           requesturl=None, return_url=False, bucket=None, service_url=None,
           path='', return_bin=False, action=None, local_file=None,
-          verify_ssl=True, location=None, full_headers=False):
+          verify_ssl=True, location=None, full_headers=False, role_arn=None):
     '''
     Perform a query against an S3-like API. This function requires that a
     secret key and the id for that key are passed in. For instance:
@@ -106,6 +106,7 @@ def query(key, keyid, method='GET', params=None, headers=None,
             data=data,
             uri='/{0}'.format(path),
             prov_dict={'id': keyid, 'key': key},
+            role_arn=role_arn,
             location=location,
             product='s3',
             requesturl=requesturl,

tests/unit/modules/s3_test.py

@@ -33,7 +33,7 @@ class S3TestCase(TestCase):
         '''
         with patch.object(s3, '_get_key',
                           return_value=('key', 'keyid', 'service_url',
-                                        'verify_ssl', 'location')):
+                                        'verify_ssl', 'location', 'role_arn')):
             with patch.object(salt.utils.s3, 'query', return_value='A'):
                 self.assertEqual(s3.delete('bucket'), 'A')
 
@@ -44,7 +44,7 @@ class S3TestCase(TestCase):
         '''
         with patch.object(s3, '_get_key',
                           return_value=('key', 'keyid', 'service_url',
-                                        'verify_ssl', 'location')):
+                                        'verify_ssl', 'location', 'role_arn')):
             with patch.object(salt.utils.s3, 'query', return_value='A'):
                 self.assertEqual(s3.get(), 'A')
 
@@ -54,7 +54,7 @@ class S3TestCase(TestCase):
         '''
         with patch.object(s3, '_get_key',
                           return_value=('key', 'keyid', 'service_url',
-                                        'verify_ssl', 'location')):
+                                        'verify_ssl', 'location', 'role_arn')):
             with patch.object(salt.utils.s3, 'query', return_value='A'):
                 self.assertEqual(s3.head('bucket'), 'A')
 
@@ -64,7 +64,7 @@ class S3TestCase(TestCase):
         '''
         with patch.object(s3, '_get_key',
                           return_value=('key', 'keyid', 'service_url',
-                                        'verify_ssl', 'location')):
+                                        'verify_ssl', 'location', 'role_arn')):
             with patch.object(salt.utils.s3, 'query', return_value='A'):
                 self.assertEqual(s3.put('bucket'), 'A')