valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-07 17:09:03 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #46155 from kojiromike/kms-renderer

Browse Source 
Add a KMS Envelope-Encryption Renderer
This commit is contained in:
Nicole Thomas 2018-03-12 14:42:22 -04:00 committed by GitHub
commit f5abf1bc95
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 464 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

259
salt/renderers/aws_kms.py Normal file
View File

@ -0,0 +1,259 @@
# -*- coding: utf-8 -*-
r'''
Renderer that will decrypt ciphers encrypted using `AWS KMS Envelope Encryption`_.
.. _`AWS KMS Envelope Encryption`: https://docs.aws.amazon.com/kms/latest/developerguide/workflow.html
Any key in the data to be rendered can be a urlsafe_b64encoded string, and this renderer will attempt
to decrypt it before passing it off to Salt. This allows you to safely store secrets in
source control, in such a way that only your Salt master can decrypt them and
distribute them only to the minions that need them.
The typical use-case would be to use ciphers in your pillar data, and keep the encrypted
data key on your master. This way developers with appropriate AWS IAM privileges can add new secrets
quickly and easily.
This renderer requires the boto3_ Python library.
.. _boto3: https://boto3.readthedocs.io/
Setup
-----
First, set up your AWS client. For complete instructions on configuration the AWS client,
please read the `boto3 configuration documentation`_. By default, this renderer will use
the default AWS profile. You can override the profile name in salt configuration.
For example, if you have a profile in your aws client configuration named "salt",
you can add the following salt configuration:
.. code-block:: yaml
aws_kms:
profile_name: salt
.. _boto3 configuration documentation: https://boto3.readthedocs.io/en/latest/guide/configuration.html
The rest of these instructions assume that you will use the default profile for key generation
and setup. If not, export AWS_PROFILE and set it to the desired value.
Once the aws client is configured, generate a KMS customer master key and use that to generate
a local data key.
.. code-block:: bash
# data_key=$(aws kms generate-data-key --key-id your-key-id --key-spec AES_256
--query 'CiphertextBlob' --output text)
# echo 'aws_kms:'
# echo ' data_key: !!binary "%s"\n' "$data_key" >> config/master
To apply the renderer on a file-by-file basis add the following line to the
top of any pillar with gpg data in it:
.. code-block:: yaml
#!yaml|aws_kms
Now with your renderer configured, you can include your ciphers in your pillar
data like so:
.. code-block:: yaml
#!yaml|aws_kms
a-secret: gAAAAABaj5uzShPI3PEz6nL5Vhk2eEHxGXSZj8g71B84CZsVjAAtDFY1mfjNRl-1Su9YVvkUzNjI4lHCJJfXqdcTvwczBYtKy0Pa7Ri02s10Wn1tF0tbRwk=
'''
# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import logging
import base64
# Import salt libs
import salt.utils.stringio
# Import 3rd-party libs
from salt.ext import six
try:
import botocore.exceptions
import boto3
logging.getLogger('boto3').setLevel(logging.CRITICAL)
except ImportError:
pass
try:
import cryptography.fernet as fernet
HAS_FERNET = True
except ImportError:
HAS_FERNET = False
def __virtual__():
'''
Only load if boto libraries exist and if boto libraries are greater than
a given version.
'''
return HAS_FERNET and salt.utils.versions.check_boto_reqs()
log = logging.getLogger(__name__)
def _cfg(key, default=None):
'''
Return the requested value from the aws_kms key in salt configuration.
If it's not set, return the default.
'''
root_cfg = __salt__.get('config.get', __opts__.get)
kms_cfg = root_cfg('aws_kms', {})
return kms_cfg.get(key, default)
def _cfg_data_key():
'''
Return the encrypted KMS data key from configuration.
Raises SaltConfigurationError if not set.
'''
data_key = _cfg('data_key', '')
if data_key:
return data_key
raise salt.exceptions.SaltConfigurationError('aws_kms:data_key is not set')
def _session():
'''
Return the boto3 session to use for the KMS client.
If aws_kms:profile_name is set in the salt configuration, use that profile.
Otherwise, fall back on the default aws profile.
We use the boto3 profile system to avoid having to duplicate
individual boto3 configuration settings in salt configuration.
'''
profile_name = _cfg('profile_name')
if profile_name:
log.info('Using the "%s" aws profile.', profile_name)
else:
log.info('aws_kms:profile_name is not set in salt. Falling back on default profile.')
try:
return boto3.Session(profile_name=profile_name)
except botocore.exceptions.ProfileNotFound as orig_exc:
err_msg = 'Boto3 could not find the "{}" profile configured in Salt.'.format(
profile_name or 'default')
config_error = salt.exceptions.SaltConfigurationError(err_msg)
six.raise_from(config_error, orig_exc)
except botocore.exceptions.NoRegionError as orig_exc:
err_msg = ('Boto3 was unable to determine the AWS '
'endpoint region using the {} profile.').format(profile_name or 'default')
config_error = salt.exceptions.SaltConfigurationError(err_msg)
six.raise_from(config_error, orig_exc)
def _kms():
'''
Return the boto3 client for the KMS API.
'''
session = _session()
return session.client('kms')
def _api_decrypt():
'''
Return the response dictionary from the KMS decrypt API call.
'''
kms = _kms()
data_key = _cfg_data_key()
try:
return kms.decrypt(CiphertextBlob=data_key)
except botocore.exceptions.ClientError as orig_exc:
error_code = orig_exc.response.get('Error', {}).get('Code', '')
if error_code != 'InvalidCiphertextException':
raise
err_msg = 'aws_kms:data_key is not a valid KMS data key'
config_error = salt.exceptions.SaltConfigurationError(err_msg)
six.raise_from(config_error, orig_exc)
def _plaintext_data_key():
'''
Return the configured KMS data key decrypted and encoded in urlsafe base64.
Cache the result to minimize API calls to AWS.
'''
response = getattr(_plaintext_data_key, 'response', None)
cache_hit = response is not None
if not cache_hit:
response = _api_decrypt()
setattr(_plaintext_data_key, 'response', response)
key_id = response['KeyId']
plaintext = response['Plaintext']
if hasattr(plaintext, 'encode'):
plaintext = plaintext.encode(__salt_system_encoding__)
log.debug('Using key %s from %s', key_id, 'cache' if cache_hit else 'api call')
return plaintext
def _base64_plaintext_data_key():
'''
Return the configured KMS data key decrypted and encoded in urlsafe base64.
'''
plaintext_data_key = _plaintext_data_key()
return base64.urlsafe_b64encode(plaintext_data_key)
def _decrypt_ciphertext(cipher, translate_newlines=False):
'''
Given a blob of ciphertext as a bytestring, try to decrypt
the cipher and return the decrypted string. If the cipher cannot be
decrypted, log the error, and return the ciphertext back out.
'''
if translate_newlines:
cipher = cipher.replace(r'\n', '\n')
if hasattr(cipher, 'encode'):
cipher = cipher.encode(__salt_system_encoding__)
# Decryption
data_key = _base64_plaintext_data_key()
plain_text = fernet.Fernet(data_key).decrypt(cipher)
if hasattr(plain_text, 'decode'):
plain_text = plain_text.decode(__salt_system_encoding__)
return six.text_type(plain_text)
def _decrypt_object(obj, translate_newlines=False):
'''
Recursively try to decrypt any object.
Recur on objects that are not strings.
Decrypt strings that are valid Fernet tokens.
Return the rest unchanged.
'''
if salt.utils.stringio.is_readable(obj):
return _decrypt_object(obj.getvalue(), translate_newlines)
if isinstance(obj, six.string_types):
try:
return _decrypt_ciphertext(obj,
translate_newlines=translate_newlines)
except (fernet.InvalidToken, TypeError):
return obj
elif isinstance(obj, dict):
for key, value in six.iteritems(obj):
obj[key] = _decrypt_object(value,
translate_newlines=translate_newlines)
return obj
elif isinstance(obj, list):
for key, value in enumerate(obj):
obj[key] = _decrypt_object(value,
translate_newlines=translate_newlines)
return obj
else:
return obj
def render(data, saltenv='base', sls='', argline='', **kwargs): # pylint: disable=unused-argument
'''
Decrypt the data to be rendered that was encrypted using AWS KMS envelope encryption.
'''
translate_newlines = kwargs.get('translate_newlines', False)
return _decrypt_object(data, translate_newlines=translate_newlines)

205
tests/unit/renderers/test_aws_kms.py Normal file
View File

@ -0,0 +1,205 @@
# -*- coding: utf-8 -*-
'''
Unit tests for AWS KMS Decryption Renderer.
'''
# pylint: disable=protected-access
# Import Python Libs
from __future__ import absolute_import, print_function, unicode_literals
# Import Salt Testing libs
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.unit import skipIf, TestCase
from tests.support.mock import (
NO_MOCK,
NO_MOCK_REASON,
MagicMock,
patch
)
# Import Salt libs
import salt.exceptions
import salt.renderers.aws_kms as aws_kms
try:
import botocore.exceptions
import botocore.session
import botocore.stub
NO_BOTOCORE = False
except ImportError:
NO_BOTOCORE = True
try:
import cryptography.fernet as fernet
NO_FERNET = False
except ImportError:
NO_FERNET = True
PLAINTEXT_SECRET = 'Use more salt.'
ENCRYPTED_DATA_KEY = 'encrypted-data-key'
PLAINTEXT_DATA_KEY = b'plaintext-data-key'
BASE64_DATA_KEY = b'cGxhaW50ZXh0LWRhdGEta2V5'
AWS_PROFILE = 'test-profile'
REGION_NAME = 'us-test-1'
@skipIf(NO_MOCK, NO_MOCK_REASON)
@skipIf(NO_BOTOCORE, 'Unable to import botocore libraries')
class AWSKMSTestCase(TestCase, LoaderModuleMockMixin):
'''
unit test AWS KMS renderer
'''
def setup_loader_modules(self):
return {aws_kms: {}}
def test__cfg_data_key(self):
'''
_cfg_data_key returns the aws_kms:data_key from configuration.
'''
config = {'aws_kms': {'data_key': ENCRYPTED_DATA_KEY}}
with patch.dict(aws_kms.__salt__, {'config.get': config.get}): # pylint: disable=no-member
self.assertEqual(aws_kms._cfg_data_key(), ENCRYPTED_DATA_KEY,
'_cfg_data_key did not return the data key configured in __salt__.')
with patch.dict(aws_kms.__opts__, config): # pylint: disable=no-member
self.assertEqual(aws_kms._cfg_data_key(), ENCRYPTED_DATA_KEY,
'_cfg_data_key did not return the data key configured in __opts__.')
def test__cfg_data_key_no_key(self):
'''
When no aws_kms:data_key is configured,
calling _cfg_data_key should raise a SaltConfigurationError
'''
self.assertRaises(salt.exceptions.SaltConfigurationError, aws_kms._cfg_data_key)
def test__session_profile(self): # pylint: disable=no-self-use
'''
_session instantiates boto3.Session with the configured profile_name
'''
with patch.object(aws_kms, '_cfg', lambda k: AWS_PROFILE):
with patch('boto3.Session') as session:
aws_kms._session()
session.assert_called_with(profile_name=AWS_PROFILE)
def test__session_noprofile(self):
'''
_session raises a SaltConfigurationError
when boto3 raises botocore.exceptions.ProfileNotFound.
'''
with patch('boto3.Session') as session:
session.side_effect = botocore.exceptions.ProfileNotFound(profile=AWS_PROFILE)
self.assertRaises(salt.exceptions.SaltConfigurationError, aws_kms._session)
def test__session_noregion(self):
'''
_session raises a SaltConfigurationError
when boto3 raises botocore.exceptions.NoRegionError
'''
with patch('boto3.Session') as session:
session.side_effect = botocore.exceptions.NoRegionError
self.assertRaises(salt.exceptions.SaltConfigurationError, aws_kms._session)
def test__kms(self): # pylint: disable=no-self-use
'''
_kms calls boto3.Session.client with 'kms' as its only argument.
'''
with patch('boto3.Session.client') as client:
aws_kms._kms()
client.assert_called_with('kms')
def test__kms_noregion(self):
'''
_kms raises a SaltConfigurationError
when boto3 raises a NoRegionError.
'''
with patch('boto3.Session') as session:
session.side_effect = botocore.exceptions.NoRegionError
self.assertRaises(salt.exceptions.SaltConfigurationError, aws_kms._kms)
def test__api_decrypt(self): # pylint: disable=no-self-use
'''
_api_decrypt_response calls kms.decrypt with the
configured data key as the CiphertextBlob kwarg.
'''
kms_client = MagicMock()
with patch.object(aws_kms, '_kms') as kms_getter:
kms_getter.return_value = kms_client
with patch.object(aws_kms, '_cfg_data_key', lambda: ENCRYPTED_DATA_KEY):
aws_kms._api_decrypt()
kms_client.decrypt.assert_called_with(CiphertextBlob=ENCRYPTED_DATA_KEY) # pylint: disable=no-member
def test__api_decrypt_badkey(self):
'''
_api_decrypt_response raises SaltConfigurationError
when kms.decrypt raises a botocore.exceptions.ClientError
with an error_code of 'InvalidCiphertextException'.
'''
kms_client = MagicMock()
kms_client.decrypt.side_effect = botocore.exceptions.ClientError( # pylint: disable=no-member
error_response={'Error': {'Code': 'InvalidCiphertextException'}},
operation_name='Decrypt',
)
with patch.object(aws_kms, '_kms') as kms_getter:
kms_getter.return_value = kms_client
with patch.object(aws_kms, '_cfg_data_key', lambda: ENCRYPTED_DATA_KEY):
self.assertRaises(salt.exceptions.SaltConfigurationError, aws_kms._api_decrypt)
def test__plaintext_data_key(self):
'''
_plaintext_data_key returns the 'Plaintext' value from the response.
It caches the response and only calls _api_decrypt exactly once.
'''
with patch.object(aws_kms, '_api_decrypt', return_value={'KeyId': 'key-id', 'Plaintext': PLAINTEXT_DATA_KEY}) as api_decrypt:
self.assertEqual(aws_kms._plaintext_data_key(), PLAINTEXT_DATA_KEY)
aws_kms._plaintext_data_key()
api_decrypt.assert_called_once()
def test__base64_plaintext_data_key(self):
'''
_base64_plaintext_data_key returns the urlsafe base64 encoded plain text data key.
'''
with patch.object(aws_kms, '_plaintext_data_key', return_value=PLAINTEXT_DATA_KEY):
self.assertEqual(aws_kms._base64_plaintext_data_key(), BASE64_DATA_KEY)
@skipIf(NO_FERNET, 'Failed to import cryptography.fernet')
def test__decrypt_ciphertext(self):
'''
test _decrypt_ciphertext
'''
test_key = fernet.Fernet.generate_key()
crypted = fernet.Fernet(test_key).encrypt(PLAINTEXT_SECRET.encode())
with patch.object(aws_kms, '_base64_plaintext_data_key', return_value=test_key):
self.assertEqual(aws_kms._decrypt_ciphertext(crypted), PLAINTEXT_SECRET)
@skipIf(NO_FERNET, 'Failed to import cryptography.fernet')
def test__decrypt_object(self):
'''
Test _decrypt_object
'''
test_key = fernet.Fernet.generate_key()
crypted = fernet.Fernet(test_key).encrypt(PLAINTEXT_SECRET.encode())
secret_map = {'secret': PLAINTEXT_SECRET}
crypted_map = {'secret': crypted}
secret_list = [PLAINTEXT_SECRET]
crypted_list = [crypted]
with patch.object(aws_kms, '_base64_plaintext_data_key', return_value=test_key):
self.assertEqual(aws_kms._decrypt_object(PLAINTEXT_SECRET), PLAINTEXT_SECRET)
self.assertEqual(aws_kms._decrypt_object(crypted), PLAINTEXT_SECRET)
self.assertEqual(aws_kms._decrypt_object(crypted_map), secret_map)
self.assertEqual(aws_kms._decrypt_object(crypted_list), secret_list)
self.assertEqual(aws_kms._decrypt_object(None), None)
@skipIf(NO_FERNET, 'Failed to import cryptography.fernet')
def test_render(self):
'''
Test that we can decrypt some data.
'''
test_key = fernet.Fernet.generate_key()
crypted = fernet.Fernet(test_key).encrypt(PLAINTEXT_SECRET.encode())
with patch.object(aws_kms, '_base64_plaintext_data_key', return_value=test_key):
self.assertEqual(aws_kms.render(crypted), PLAINTEXT_SECRET)