From ccacbaed027bd4a959d9e9fccfa14ffdd00736d7 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 2 Apr 2013 19:25:45 +0100 Subject: [PATCH] Enforce the new providers configuration. Refs #239. * Updated the `old_to_new()` function to produce a more salt like syntax. * Added a cli option argument to allow providing a different file which would contain the several cloud providers configuration. * Besides the separate file for the could providers configuration, we also added support to include any `*.conf` file from `cloud.providers.d/` which is relative to the parent directory of the provided cloud providers configuration file. * Update the several documentation files in order to include examples of both the old and new cloud providers configuration syntax side-to-side. --- doc/topics/aws.rst | 391 ++++++++++++++++++++++++++++++------- doc/topics/config.rst | 255 +++++++++++++++++++++--- doc/topics/parallels.rst | 77 ++++++-- doc/topics/profiles.rst | 25 ++- doc/topics/rackspace.rst | 64 +++++- saltcloud/config.py | 102 +++++++--- saltcloud/utils/parsers.py | 50 ++++- 7 files changed, 792 insertions(+), 172 deletions(-) diff --git a/doc/topics/aws.rst b/doc/topics/aws.rst index 5e3af2ba2b..59a9864ba1 100644 --- a/doc/topics/aws.rst +++ b/doc/topics/aws.rst @@ -7,6 +7,8 @@ platforms Salt Cloud has been built to support. Set up the cloud config at ``/etc/salt/cloud``: +* Using the old format: + .. code-block:: yaml # Set up the location of the salt master @@ -51,23 +53,113 @@ Set up the cloud config at ``/etc/salt/cloud``: AWS.ssh_username: ec2-user +* Using the new configuration format: + +.. code-block:: yaml + + aws-southeast-public-ips: + # Set up the location of the salt master + # + minion: + master: saltmaster.example.com + + # Specify whether to use public or private IP for deploy script. + # + # Valid options are: + # private_ips - The salt-master is also hosted with AWS + # public_ips - The salt-master is hosted outside of AWS + # + ssh_interface: public_ips + + # Set the AWS access credentials (see below) + # + id: HJGRYCILJLKJYG + key: 'kdjgfsgm;woormgl/aserigjksjdhasdfgn' + + # Make sure this key is owned by root with permissions 0400. + # + private_key: /etc/salt/my_test_key.pem + keyname: my_test_key + securitygroup: default + + # Optionally configure default region + # + location: ap-southeast-1 + availability_zone: ap-southeast-1b + + # Configure which user to use to run the deploy script. This setting is + # dependent upon the AMI that is used to deploy. It is usually safer to + # configure this individually in a profile, than globally. Typical users + # are: + # + # Amazon Linux -> ec2-user + # RHEL -> ec2-user + # CentOS -> ec2-user + # Ubuntu -> ubuntu + # + ssh_username: ec2-user + + + aws-southeast-private-ips: + # Set up the location of the salt master + # + minion: + master: saltmaster.example.com + + # Specify whether to use public or private IP for deploy script. + # + # Valid options are: + # private_ips - The salt-master is also hosted with AWS + # public_ips - The salt-master is hosted outside of AWS + # + ssh_interface: private_ips + + # Set the AWS access credentials (see below) + # + id: HJGRYCILJLKJYG + key: 'kdjgfsgm;woormgl/aserigjksjdhasdfgn' + + # Make sure this key is owned by root with permissions 0400. + # + private_key: /etc/salt/my_test_key.pem + keyname: my_test_key + securitygroup: default + + # Optionally configure default region + # + location: ap-southeast-1 + availability_zone: ap-southeast-1b + + # Configure which user to use to run the deploy script. This setting is + # dependent upon the AMI that is used to deploy. It is usually safer to + # configure this individually in a profile, than globally. Typical users + # are: + # + # Amazon Linux -> ec2-user + # RHEL -> ec2-user + # CentOS -> ec2-user + # Ubuntu -> ubuntu + # + ssh_username: ec2-user + + Access Credentials ================== -The ``id`` and ``key`` settings may be found in the Security Credentials area +The ``id`` and ``key`` settings may be found in the Security Credentials area of the AWS Account page: https://portal.aws.amazon.com/gp/aws/securityCredentials -Both are located in the Access Credentials area of the page, under the Access -Keys tab. The ``id`` setting is labeled Access Key ID, and the ``key`` setting +Both are located in the Access Credentials area of the page, under the Access +Keys tab. The ``id`` setting is labeled Access Key ID, and the ``key`` setting is labeled Secret Access Key. Key Pairs ========= -In order to create an instance with Salt installed and configured, a key pair -will need to be created. This can be done in the EC2 Management Console, in the -Key Pairs area. These key pairs are unique to a specific region. Keys in the +In order to create an instance with Salt installed and configured, a key pair +will need to be created. This can be done in the EC2 Management Console, in the +Key Pairs area. These key pairs are unique to a specific region. Keys in the us-east-1 region can be configured at: https://console.aws.amazon.com/ec2/home?region=us-east-1#s=KeyPairs @@ -76,23 +168,24 @@ Keys in the us-west-1 region can be configured at https://console.aws.amazon.com/ec2/home?region=us-west-1#s=KeyPairs -...and so on. When creating a key pair, the browser will prompt to download a -pem file. This file must be placed in a directory accessable by Salt Cloud, +...and so on. When creating a key pair, the browser will prompt to download a +pem file. This file must be placed in a directory accessable by Salt Cloud, with permissions set to either 0400 or 0600. Security Groups =============== -An instance on AWS needs to belong to a security group. Like key pairs, these -are unique to a specific region. These are also configured in the EC2 Management -Console. Security groups for the us-east-1 region can be configured at: +An instance on AWS needs to belong to a security group. Like key pairs, these +are unique to a specific region. These are also configured in the EC2 +Management Console. Security groups for the us-east-1 region can be configured +at: https://console.aws.amazon.com/ec2/home?region=us-east-1#s=SecurityGroups ...and so on. -A security group defines firewall rules which an instance will adhere to. If -the salt-master is configured outside of AWS, the security group must open the +A security group defines firewall rules which an instance will adhere to. If +the salt-master is configured outside of AWS, the security group must open the SSH port (usually port 22) in order for Salt Cloud to install Salt. @@ -100,26 +193,49 @@ Cloud Profiles ============== Set up an initial profile at ``/etc/salt/cloud.profiles``: +* Using the old cloud providers configuration format: + .. code-block:: yaml base_aws: - provider: aws - image: ami-e565ba8c - size: Micro Instance - ssh-user: ec2-user + provider: aws + image: ami-e565ba8c + size: Micro Instance + ssh-user: ec2-user + + +* Using the new cloud providers configuration format and the example + configuration above: + +.. code-block:: yaml + + base_aws_private: + provider: aws-southeast-private-ips + image: ami-e565ba8c + size: Micro Instance + ssh-user: ec2-user + + base_aws_public: + provider: aws-southeast-public-ips + image: ami-e565ba8c + size: Micro Instance + ssh-user: ec2-user + The profile can be realized now with a salt command: .. code-block:: bash # salt-cloud -p base_aws ami.example.com + # salt-cloud -p base_aws_public ami.example.com + # salt-cloud -p base_aws_private ami.example.com -This will create an instance named ``ami.example.com`` in EC2. The minion that -is installed on this instance will have an ``id`` of ``ami.example.com``. If -the command was executed on the salt-master, its Salt key will automatically be +This will create an instance named ``ami.example.com`` in EC2. The minion that +is installed on this instance will have an ``id`` of ``ami.example.com``. If +the command was executed on the salt-master, its Salt key will automatically be signed on the master. -Once the instance has been created with salt-minion installed, connectivity to +Once the instance has been created with salt-minion installed, connectivity to it can be verified with Salt: .. code-block:: bash @@ -131,6 +247,8 @@ Required Settings ================= The following settings are always required for AWS: +* Using the old cloud configuration format: + .. code-block:: yaml # Set the AWS login data @@ -141,21 +259,51 @@ The following settings are always required for AWS: AWS.private_key: /root/test.pem +* Using the new cloud configuration format: + +.. code-block:: yaml + + # Set the AWS login data + aws-config: + id: HJGRYCILJLKJYG + key: 'kdjgfsgm;woormgl/aserigjksjdhasdfgn' + keyname: test + securitygroup: quick-start + private_key: /root/test.pem + provider: aws + + Optional Settings ================= -AWS allows a location to be set for servers to be deployed in. Availability + +AWS allows a location to be set for servers to be deployed in. Availability zones exist inside regions, and may be added to increase specificity. +* Using the old cloud configuration format: + .. code-block:: yaml # Optionally configure default region AWS.location: ap-southeast-1 AWS.availability_zone: ap-southeast-1b -AWS instances can have a public or private IP, or both. When an instance is + +* Using the new cloud configuration format: + +.. code-block:: yaml + + aws-config: + # Optionally configure default region + location: ap-southeast-1 + availability_zone: ap-southeast-1b + + +AWS instances can have a public or private IP, or both. When an instance is deployed, Salt Cloud needs to log into it via SSH to run the deploy script. -By default, the public IP will be used for this. If the salt-cloud command -is run from another AWS instance, the private IP should be used. +By default, the public IP will be used for this. If the salt-cloud command is +run from another AWS instance, the private IP should be used. + +* Using the old cloud configuration format: .. code-block:: yaml @@ -163,21 +311,45 @@ is run from another AWS instance, the private IP should be used. # private_ips or public_ips AWS.ssh_interface: public_ips + +* Using the new cloud configuration format: + +.. code-block:: yaml + + aws-config: + # Specify whether to use public or private IP for deploy script + # private_ips or public_ips + ssh_interface: public_ips + + Many AWS instances do not allow remote access to the root user by default. -Instead, another user must be used to run the deploy script using sudo. Some -common usernames include ec2-user (for Amazon Linux), ubuntu (for Ubuntu -instances), admin (official Debian) and bitnami (for images provided by +Instead, another user must be used to run the deploy script using sudo. Some +common usernames include ec2-user (for Amazon Linux), ubuntu (for Ubuntu +instances), admin (official Debian) and bitnami (for images provided by Bitnami). +* Using the old cloud configuration format: + .. code-block:: yaml # Configure which user to use to run the deploy script AWS.ssh_username: ec2-user -Multiple usernames can be provided, in which case Salt Cloud will attempt to -guess the correct username. This is mostly useful in the main configuration +* Using the new cloud configuration format: + +.. code-block:: yaml + + aws-config + # Configure which user to use to run the deploy script + ssh_username: ec2-user + + +Multiple usernames can be provided, in which case Salt Cloud will attempt to +guess the correct username. This is mostly useful in the main configuration file: +* Using the old cloud configuration format: + .. code-block:: yaml AWS.ssh_username: @@ -186,8 +358,23 @@ file: - admin - bitnami + +* Using the new cloud configuration format: + +.. code-block:: yaml + + aws-config: + ssh_username: + - ec2-user + - ubuntu + - admin + - bitnami + + Multiple security groups can also be specified in the same fashion: +* Using the old cloud configuration format: + .. code-block:: yaml AWS.securitygroup: @@ -207,10 +394,20 @@ each cloud profile: VirtualName: ephemeral1 +* Using the old cloud configuration format: + +.. code-block:: yaml + + aws-config: + securitygroup: + - default + - extra + + Modify AWS Tags =============== -One of the features of AWS is the ability to tag resources. In fact, under the -hood, the names given to EC2 instances by salt-cloud are actually just stored +One of the features of AWS is the ability to tag resources. In fact, under the +hood, the names given to EC2 instances by salt-cloud are actually just stored as a tag called Name. Salt Cloud has the ability to manage these tags: .. code-block:: bash @@ -222,8 +419,8 @@ as a tag called Name. Salt Cloud has the ability to manage these tags: Rename AWS Instances ==================== -As mentioned above, AWS instances are named via a tag. However, renaming an -instance by renaming its tag will cause the salt keys to mismatch. A rename +As mentioned above, AWS instances are named via a tag. However, renaming an +instance by renaming its tag will cause the salt keys to mismatch. A rename function exists which renames both the instance, and the salt keys. .. code-block:: bash @@ -233,7 +430,7 @@ function exists which renames both the instance, and the salt keys. AWS Termination Protection ========================== -AWS allows the user to enable and disable termination protection on a specific +AWS allows the user to enable and disable termination protection on a specific instance. An instance with this protection enabled cannot be destroyed. .. code-block:: bash @@ -244,31 +441,43 @@ instance. An instance with this protection enabled cannot be destroyed. Rename on Destroy ================= -When instances on AWS are destroyed, there will be a lag between the time that -the action is sent, and the time that Amazon cleans up the instance. During this -time, the instance still retails a Name tag, which will cause a collision if the -creation of an instance with the same name is attempted before the cleanup -occurs. In order to avoid such collisions, Salt Cloud can be configured to -rename instances when they are destroyed. The new name will look something like: +When instances on AWS are destroyed, there will be a lag between the time that +the action is sent, and the time that Amazon cleans up the instance. During +this time, the instance still retails a Name tag, which will cause a collision +if the creation of an instance with the same name is attempted before the +cleanup occurs. In order to avoid such collisions, Salt Cloud can be configured +to rename instances when they are destroyed. The new name will look something +like: .. code-block:: bash myinstance-DEL20f5b8ad4eb64ed88f2c428df80a1a0c -In order to enable this, add AWS.rename_on_destroy line to the main + +In order to enable this, add AWS.rename_on_destroy line to the main configuration file: +* Using the old cloud configuration format: + .. code-block:: yaml AWS.rename_on_destroy: True +* Using the new cloud configuration format: + +.. code-block:: yaml + + aws-config: + rename_on_destroy: True + + EC2 Images ========== -The following are lists of available AMI images, generally sorted by OS. These -lists are on 3rd-party websites, are not managed by Salt Stack in any way. They -are provided here as a reference for those who are interested, and contain no -warranty (express or implied) from anyone affiliated with Salt Stack. Most of +The following are lists of available AMI images, generally sorted by OS. These +lists are on 3rd-party websites, are not managed by Salt Stack in any way. They +are provided here as a reference for those who are interested, and contain no +warranty (express or implied) from anyone affiliated with Salt Stack. Most of them have never been used, much less tested, by the Salt Stack team. * `Arch Linux`__ @@ -298,10 +507,12 @@ them have never been used, much less tested, by the Salt Stack team. Experimental EC2 Driver ======================= -An experimental driver has been added to Salt Cloud called EC2. The -configuration for this driver is the same as for AWS, but with EC2 in the +An experimental driver has been added to Salt Cloud called EC2. The +configuration for this driver is the same as for AWS, but with EC2 in the argument names: +* Using the old cloud configuration format: + .. code-block:: yaml # Set the EC2 login data @@ -311,21 +522,35 @@ argument names: EC2.securitygroup: quick-start EC2.private_key: /root/test.pem -This driver contains optimizations over the old AWS driver, which increase -speed and functionality. However, because this is a new driver, it is currently -considered to be experimental, and as such, the old AWS driver may still be + +* Using the new cloud configuration format: + +.. code-block:: yaml + + ec2-config: + # Set the EC2 login data + id: HJGRYCILJLKJYG + key: 'kdjgfsgm;woormgl/aserigjksjdhasdfgn' + keyname: test + securitygroup: quick-start + private_key: /root/test.pem + + +This driver contains optimizations over the old AWS driver, which increase +speed and functionality. However, because this is a new driver, it is currently +considered to be experimental, and as such, the old AWS driver may still be used as before. -IMPORTANT: Because this driver is in experimental status, its usage and +IMPORTANT: Because this driver is in experimental status, its usage and configuration should be expected to change. -The remainder of this document describes settings which may be used with the +The remainder of this document describes settings which may be used with the EC2 driver. show_image ========== -This is a function that describes an AMI on EC2. This will give insight as to +This is a function that describes an AMI on EC2. This will give insight as to the defaults that will be applied to an instance using a particular AMI. .. code-block:: bash @@ -335,9 +560,9 @@ the defaults that will be applied to an instance using a particular AMI. show_instance ============= -This action is a thin wrapper around --full-query, which displays details on a -single instance only. In an environment with several machines, this will save a -user from having to sort through all instance data, just to examine a single +This action is a thin wrapper around --full-query, which displays details on a +single instance only. In an environment with several machines, this will save a +user from having to sort through all instance data, just to examine a single instance. .. code-block:: bash @@ -347,10 +572,10 @@ instance. delvol_on_destroy ================= -This argument overrides the default DeleteOnTermination setting in the AMI for -the root EBS volume for an instance. Many AMIs contain 'false' as a default, -resulting in orphaned volumes in the EC2 account, which may unknowingly be -charged to the account. This setting can be added to the profile or map file +This argument overrides the default DeleteOnTermination setting in the AMI for +the root EBS volume for an instance. Many AMIs contain 'false' as a default, +resulting in orphaned volumes in the EC2 account, which may unknowingly be +charged to the account. This setting can be added to the profile or map file for an instance. .. code-block:: yaml @@ -358,14 +583,25 @@ for an instance. delvol_on_destroy: True -This can also be set as a global setting in the EC2 cloud configuration: +This can also be set as a cloud provider setting in the EC2 cloud +configuration: + +* Using the old cloud configuration format: .. code-block:: yaml EC2.delvol_on_destroy: True +* Using the new cloud configuration format: -The setting for this may be changed on an existing instance using one of the +.. code-block:: yaml + + ec2-config: + delvol_on_destroy: True + + + +The setting for this may be changed on an existing instance using one of the following commands: .. code-block:: bash @@ -376,8 +612,9 @@ following commands: EC2 Termination Protection ========================== -AWS allows the user to enable and disable termination protection on a specific -instance. An instance with this protection enabled cannot be destroyed. The EC2 + +AWS allows the user to enable and disable termination protection on a specific +instance. An instance with this protection enabled cannot be destroyed. The EC2 driver adds a show_term_protect action to the regular AWS functionality. .. code-block:: bash @@ -389,21 +626,23 @@ driver adds a show_term_protect action to the regular AWS functionality. Alternate Endpoint ================== -Normally, ec2 endpoints are build using the region and the service_url. The +Normally, EC2 endpoints are build using the region and the service_url. The resulting endpoint would follow this pattern: .. code-block:: ec2.. + This results in an endpoint that looks like: .. code-block:: ec2.us-east-1.amazonaws.com -There are other projects that support an EC2 compatibility layer, which this -scheme does not account for. This can be overridden by specifying the endpoint + +There are other projects that support an EC2 compatibility layer, which this +scheme does not account for. This can be overridden by specifying the endpoint directly in the main cloud configuration file: .. code-block:: yaml @@ -419,8 +658,8 @@ The EC2 driver has several functions and actions for management of EBS volumes. Creating Volumes ---------------- A volume may be created, independent of an instance. A zone must be specified. -A size or a snapshot may be specified (in GiB). If neither is given, a default -size of 10 GiB will be used. If a snapshot is given, the size of the snapshot +A size or a snapshot may be specified (in GiB). If neither is given, a default +size of 10 GiB will be used. If a snapshot is given, the size of the snapshot will be used. .. code-block:: bash @@ -432,8 +671,8 @@ will be used. Attaching Volumes ----------------- -Unattached volumes may be attached to an instance. The following values are -required: name or instance_id, volume_id and device. +Unattached volumes may be attached to an instance. The following values are +required; name or instance_id, volume_id and device. .. code-block:: bash @@ -442,7 +681,7 @@ required: name or instance_id, volume_id and device. Show a Volume ------------- -The details about an existing volume may be retreived. +The details about an existing volume may be retrieved. .. code-block:: bash @@ -475,9 +714,9 @@ The EC2 driver has the ability to manage key pairs. Creating a Key Pair ------------------- -A key pair is required in order to create an instance. When creating a key pair +A key pair is required in order to create an instance. When creating a key pair with this function, the return data will contain a copy of the private key. -This private key is not stored by Amazon, and will not be obtainable past this +This private key is not stored by Amazon, and will not be obtainable past this point, and should be stored immediately. .. code-block:: bash @@ -487,7 +726,7 @@ point, and should be stored immediately. Show a Key Pair --------------- -This function will show the details related to a key pair, not including the +This function will show the details related to a key pair, not including the private key itself (which is not stored by Amazon). .. code-block:: bash diff --git a/doc/topics/config.rst b/doc/topics/config.rst index 4cde0814d1..367f03d35e 100644 --- a/doc/topics/config.rst +++ b/doc/topics/config.rst @@ -2,15 +2,15 @@ Core Configuration ================== -A number of core configuration options and some options that are global to the -VM profiles can be set in the cloud config file. By default this file is +A number of core configuration options and some options that are global to the +VM profiles can be set in the cloud configuration file. By default this file is located at ``/etc/salt/cloud``. Minion Configuration ==================== -The default minion configuration is set up in this file. This is where the +The default minion configuration is set up in this file. This is where the minions that are created derive their configuration. .. code-block:: yaml @@ -18,6 +18,7 @@ minions that are created derive their configuration. minion: master: saltmaster.example.com + This is the location in particular to specify the location of the salt master. @@ -26,21 +27,58 @@ Cloud Configurations The data specific to interacting with public clouds is set up here. +**ATTENTION**: Since version 0.8.7 a new cloud provider configuration syntax +was implemented. It will allow for multiple configurations of the same cloud +provider where only minor details can change, for example, the region for an +EC2 instance. While the old format is still supported and automatically +migrated every time salt-cloud configuration is parsed, a choice was made to +warn the user or even exit with an error if both formats are mixed. + +While moving towards an improved and extensible configuration handling +regarding the cloud providers, ``--providers-config``, which defaults to +``/etc/salt/cloud.providers`` was added to the cli parser. It allows for the +cloud providers configuration to be provided in a different file, and/or even +any matching file on a sub-directory, ``cloud.providers.d/*.conf`` which is +relative to the providers configuration file(with the above configuration file +as an example, ``/etc/salt/cloud.providers.d/*.conf``). + + Rackspace --------- Rackspace cloud requires two configuration options: +* Using the old format: + .. code-block:: yaml RACKSPACE.user: example_user RACKSPACE.apikey: 123984bjjas87034 + + +* Using the new configuration format: + +.. code-block:: yaml + + rackspace-config: + user: example_user + apikey: 123984bjjas87034 + provider: rackspace + + +**NOTE**: With the new providers configuration syntax you would have ``provider: +rackspace-config`` instead of ``provider: rackspace`` on a profile +configuration. + + Amazon AWS ---------- A number of configuration options are required for Amazon AWS: +* Using the old format: + .. code-block:: yaml AWS.id: HJGRYCILJLKJYG @@ -49,27 +87,72 @@ A number of configuration options are required for Amazon AWS: AWS.securitygroup: quick-start AWS.private_key: /root/test.pem + +* Using the new configuration format: + +.. code-block:: yaml + + aws-quick-start: + id: HJGRYCILJLKJYG + key: 'kdjgfsgm;woormgl/aserigjksjdhasdfgn' + keyname: test + securitygroup: quick-start + private_key: /root/test.pem + provider: aws + + aws-default: + id: HJGRYCILJLKJYG + key: 'kdjgfsgm;woormgl/aserigjksjdhasdfgn' + keyname: test + securitygroup: default + private_key: /root/test.pem + provider: aws + + +**NOTE**: With the new providers configuration syntax you would have +``provider: aws-quick-start`` or ``provider: aws-default`` instead of +``provider: aws`` on a profile configuration. + Linode ------ -Linode requires a single api key, but the default root password also needs -to be set: +Linode requires a single API key, but the default root password also needs to +be set: + +* Using the old format: .. code-block:: yaml LINODE.apikey: asldkgfakl;sdfjsjaslfjaklsdjf;askldjfaaklsjdfhasldsadfghdkf LINODE.password: F00barbaz -The password needs to be 8 characters and contain lowercase, uppercase and + +* Using the new configuration format: + +.. code-block:: yaml + + linode-foo: + apikey: asldkgfakl;sdfjsjaslfjaklsdjf;askldjfaaklsjdfhasldsadfghdkf + password: F00barbaz + provider: linode + + +**NOTE**: With the new providers configuration syntax you would have ``provider: +linode-foo`` instead of ``provider: linode`` on a profile configuration. + +The password needs to be 8 characters and contain lowercase, uppercase and numbers. + Joyent Cloud ------------ -The Joyent cloud requires three configuration paramaters. The user name and -password that are used to log into the Joyent system, and the location of -the private ssh key associated with the Joyent account. The ssh key is needed -to send the provisioning commands up to the freshly created virtual machine, +The Joyent cloud requires three configuration parameters. The user name and +password that are used to log into the Joyent system, and the location of the +private ssh key associated with the Joyent account. The ssh key is needed to +send the provisioning commands up to the freshly created virtual machine, + +* Using the old format: .. code-block:: yaml @@ -77,29 +160,65 @@ to send the provisioning commands up to the freshly created virtual machine, JOYENT.password: saltybacon JOYENT.private_key: /root/joyent.pem + +* Using the new configuration format: + +.. code-block:: yaml + + joyent-config: + user: fred + password: saltybacon + private_key: /root/joyent.pem + provider: joyent + + +**NOTE**: With the new providers configuration syntax you would have ``provider: +joyent-config`` instead of ``provider: joyent`` on a profile configuration. + + GoGrid ------ -To use Salt Cloud with GoGrid log into the GoGrid web interface and -create an api key. Do this by clicking on "My Account" and then going to the -API Keys tab. +To use Salt Cloud with GoGrid log into the GoGrid web interface and create an +API key. Do this by clicking on "My Account" and then going to the API Keys +tab. -The GOGRID.apikey and the GOGRID.sharedsecret configuration paramaters need to -be set in the config file to enable interfacing with GoGrid: +The GOGRID.apikey and the GOGRID.sharedsecret configuration parameters need to +be set in the configuration file to enable interfacing with GoGrid: + +* Using the old format: .. code-block:: yaml GOGRID.apikey: asdff7896asdh789 GOGRID.sharedsecret: saltybacon + +* Using the new configuration format: + +.. code-block:: yaml + + gogrid-config: + apikey: asdff7896asdh789 + sharedsecret: saltybacon + provider: gogrid + + +**NOTE**: With the new providers configuration syntax you would have +``provider: gogrid-config`` instead of ``provider: gogrid`` on a profile +configuration. + + OpenStack --------- -OpenStack configuration differs between providers, and at the moment several -options need to be specified. This module has been officially tested against -the HP and the Rackspace implementations, and some examples are provided for +OpenStack configuration differs between providers, and at the moment several +options need to be specified. This module has been officially tested against +the HP and the Rackspace implementations, and some examples are provided for both. +* Using the old format: + .. code-block:: yaml # For HP @@ -122,33 +241,94 @@ both. OPENSTACK.tenant: 5555555 OPENSTACK.password: mypass -If you have an API key for your provider, it may be specified instead of a + +If you have an API key for your provider, it may be specified instead of a password: .. code-block:: yaml OPENSTACK.apikey: 901d3f579h23c8v73q9 -You will certainly need to configure the ``user``, ``tenant`` and either -``password`` or ``apikey``. -If your OpenStack instances only have private IP addresses and a CIDR range of -private addresses are not reachable from the salt-master, you may set your -preference to have Salt ignore it: +* Using the new configuration format: .. code-block:: yaml - OPENSTACK.ignore_cidr: 192.168.0.0/16 + # For HP + openstack-hp-config: + identity_url: + 'https://region-a.geo-1.identity.hpcloudsvc.com:35357/v2.0/' + compute_name: Compute + compute_region: 'az-1.region-a.geo-1' + tenant: myuser-tenant1 + user: myuser + ssh_key_name: mykey + ssh_key_file: '/etc/salt/hpcloud/mykey.pem' + password: mypass + provider: openstack + + # For Rackspace + openstack-rackspace-config: + identity_url: 'https://identity.api.rackspacecloud.com/v2.0/tokens' + compute_name: cloudServersOpenStack + protocol: ipv4 + compute_region: DFW + protocol: ipv4 + user: myuser + tenant: 5555555 + password: mypass + provider: openstack + + +If you have an API key for your provider, it may be specified instead of a +password: + +.. code-block:: yaml + + openstack-hp-config: + apikey: 901d3f579h23c8v73q9 + + openstack-rackspace-config: + apikey: 901d3f579h23c8v73q9 + + +**NOTE**: With the new providers configuration syntax you would have +``provider: openstack-hp-config`` or ``provider: openstack-rackspace-config`` +instead of ``provider: openstack`` on a profile configuration. + + +You will certainly need to configure the ``user``, ``tenant`` and either +``password`` or ``apikey``. + + +If your OpenStack instances only have private IP addresses and a CIDR range of +private addresses are not reachable from the salt-master, you may set your +preference to have Salt ignore it. Using the old could configurations syntax: + +.. code-block:: yaml + + OPENSTACK.ignore_cidr: 192.168.0.0/16 + + +Using the new syntax: + +.. code-block:: yaml + + openstack-config: + ignore_cidr: 192.168.0.0/16 + IBM SmartCloud Enterprise ------------------------- -In addition to a username and password, the IBM SCE module requires an SSH key, -which is currently configured inside IBM's web interface. A location is also -required to create instances, but not to query their cloud. This is important, -because you need to use salt-cloud --list-locations (with the other options +In addition to a username and password, the IBM SCE module requires an SSH key, +which is currently configured inside IBM's web interface. A location is also +required to create instances, but not to query their cloud. This is important, +because you need to use salt-cloud --list-locations (with the other options already set) in order to find the name of the location that you want to use. +* Using the old format: + .. code-block:: yaml IBMSCE.user: myuser@mycorp.com @@ -158,3 +338,20 @@ already set) in order to find the name of the location that you want to use. IBMSCE.location: Raleigh + +* Using the new configuration format: + +.. code-block:: yaml + + ibmsce-config: + user: myuser@mycorp.com + password: mypass + ssh_key_name: mykey + ssh_key_file: '/etc/salt/ibm/mykey.pem' + location: Raleigh + provider: ibmsce + + +**NOTE**: With the new providers configuration syntax you would have +``provider: imbsce-config`` instead of ``provider: ibmsce`` on a profile +configuration. diff --git a/doc/topics/parallels.rst b/doc/topics/parallels.rst index 103013b1dd..904a07e98b 100644 --- a/doc/topics/parallels.rst +++ b/doc/topics/parallels.rst @@ -2,13 +2,13 @@ Getting Started With Parallels ============================== -Parallels Cloud Server is a product by Parallels that delivers a cloud hosting -solution. The PARALLELS module for Salt Cloud enables you to manage instances +Parallels Cloud Server is a product by Parallels that delivers a cloud hosting +solution. The PARALLELS module for Salt Cloud enables you to manage instances hosted by a provider using PCS. Further information can be found at: http://www.parallels.com/products/pcs/ -Set up the cloud config at ``/etc/salt/cloud``: +* Using the old format, set up the cloud configuration at ``/etc/salt/cloud``: .. code-block:: yaml @@ -27,15 +27,42 @@ Set up the cloud config at ``/etc/salt/cloud``: PARALLELS.url: https://api.cloud.xmission.com:4465/paci/v1.0/ +* Using the new format, set up the cloud configuration at + ``/etc/salt/cloud.providers`` or + ``/etc/salt/cloud.providers.d/parallels.conf``: + +.. code-block:: yaml + + parallels-config: + # Set up the location of the salt master + # + minion: + master: saltmaster.example.com + + # Set the PARALLELS access credentials (see below) + # + user: myuser + password: badpass + + # Set the access URL for your PARALLELS provider + # + url: https://api.cloud.xmission.com:4465/paci/v1.0/ + + + Access Credentials ================== -The ``user``, ``password`` and ``url`` will be provided to you by your cloud +The ``user``, ``password`` and ``url`` will be provided to you by your cloud provider. These are all required in order for the PARALLELS driver to work. Cloud Profiles ============== -Set up an initial profile at ``/etc/salt/cloud.profiles``: +Set up an initial profile at ``/etc/salt/cloud.profiles`` or +``/etc/salt/cloud.profiles.d/parallels.conf``: + + +* Using the old cloud configuration format: .. code-block:: yaml @@ -43,18 +70,30 @@ Set up an initial profile at ``/etc/salt/cloud.profiles``: provider: parallels image: ubuntu-12.04-x86_64 + +* Using the new cloud configuration format and the cloud configuration example + from above: + +.. code-block:: yaml + + parallels-ubuntu: + provider: parallels-config + image: ubuntu-12.04-x86_64 + + + The profile can be realized now with a salt command: .. code-block:: bash # salt-cloud -p parallels-ubuntu myubuntu -This will create an instance named ``myubuntu`` on the cloud provider. The +This will create an instance named ``myubuntu`` on the cloud provider. The minion that is installed on this instance will have an ``id`` of ``myubuntu``. -If the command was executed on the salt-master, its Salt key will automatically +If the command was executed on the salt-master, its Salt key will automatically be signed on the master. -Once the instance has been created with salt-minion installed, connectivity to +Once the instance has been created with salt-minion installed, connectivity to it can be verified with Salt: .. code-block:: bash @@ -66,6 +105,9 @@ Required Settings ================= The following settings are always required for PARALLELS: + +* Using the old cloud configuration format: + .. code-block:: yaml PARALLELS.user: myuser @@ -73,12 +115,23 @@ The following settings are always required for PARALLELS: PARALLELS.url: https://api.cloud.xmission.com:4465/paci/v1.0/ +* Using the new cloud configuration format: + +.. code-block:: yaml + + parallels-config: + user: myuser + password: badpass + url: https://api.cloud.xmission.com:4465/paci/v1.0/ + + + Optional Settings ================= -Unlike other cloud providers in Salt Cloud, Parallels does not utilize a -``size`` setting. This is because Parallels allows the end-user to specify a -more detailed configuration for their instances, than is allowed by many other -cloud providers. The following options are available to be used in a profile, +Unlike other cloud providers in Salt Cloud, Parallels does not utilize a +``size`` setting. This is because Parallels allows the end-user to specify a +more detailed configuration for their instances, than is allowed by many other +cloud providers. The following options are available to be used in a profile, with their default settings listed. .. code-block:: yaml diff --git a/doc/topics/profiles.rst b/doc/topics/profiles.rst index fce02de9b3..3124015e96 100644 --- a/doc/topics/profiles.rst +++ b/doc/topics/profiles.rst @@ -2,7 +2,7 @@ VM Profiles =========== Salt cloud designates virtual machines inside the profile configuration file. -The profile configuration file defaults to ``/etc/salt/cloud.profiles`` and is +The profile configuration file defaults to ``/etc/salt/cloud.profiles`` and is a yaml configuration. The syntax for declaring profiles is simple: .. code-block:: yaml @@ -13,7 +13,8 @@ a yaml configuration. The syntax for declaring profiles is simple: size: 256 server script: Fedora -A few key peices of information need to be declared and can change based on the + +A few key pieces of information need to be declared and can change based on the public cloud provider. A number of additional parameters can also be inserted: .. code-block:: yaml @@ -29,6 +30,7 @@ public cloud provider. A number of additional parameters can also be inserted: grains: role: webserver + The image must be selected from available images. Similarly, sizes must be selected from the list of sizes. To get a list of available images and sizes use the following command: @@ -38,19 +40,24 @@ use the following command: salt-cloud --list-images openstack salt-cloud --list-sizes openstack -Some parameters can be specified in the main Salt cloud config file and then -are applied to all cloud profiles. For instance if only a single cloud provider -is being used then the provider option can be declared in the Salt cloud config -file. -Multiple Config Files ---------------------- +Some parameters can be specified in the main Salt cloud configuration file and +then are applied to all cloud profiles. For instance if only a single cloud +provider is being used then the provider option can be declared in the Salt +cloud configuration file. + + +Multiple Configuration Files +---------------------------- In addition to ``/etc/salt/cloud.profiles``, profiles can also be specified in -any file matching ``/etc/salt/cloud.profiles.d/*conf``. This allows for more +any file matching ``cloud.profiles.d/*conf`` which is a sub-directory relative +to the profiles configuration file(with the above configuration file as an +example, ``/etc/salt/cloud.profiles.d/*.conf``). This allows for more extensible configuration, and plays nicely with various configuration management tools as well as version control systems. + Larger Example -------------- diff --git a/doc/topics/rackspace.rst b/doc/topics/rackspace.rst index 7a87631e2f..071a88065f 100644 --- a/doc/topics/rackspace.rst +++ b/doc/topics/rackspace.rst @@ -2,10 +2,10 @@ Getting Started With Rackspace ============================== -Rackspace is a major public cloud platform and is one of the core platforms +Rackspace is a major public cloud platform and is one of the core platforms that Salt Cloud has been built to support. -Set up the cloud config at ``/etc/salt/cloud``: +* Using the old format, set up the cloud configuration at ``/etc/salt/cloud``: .. code-block:: yaml @@ -31,8 +31,39 @@ Set up the cloud config at ``/etc/salt/cloud``: OPENSTACK.apikey: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +* Using the new format, set up the cloud configuration at + ``/etc/salt/cloud.providers`` or + ``/etc/salt/cloud.providers.d/rackspace.conf``: + +.. code-block:: yaml + + rackspace-config: + # Set the location of the salt-master + # + minion: + master: saltmaster.example.com + + # Configure Rackspace using the OpenStack plugin + # + identity_url: 'https://identity.api.rackspacecloud.com/v2.0/tokens' + compute_name: cloudServersOpenStack + protocol: ipv4 + + # Set the compute region: + # + compute_region: DFW + + # Configure Rackspace authentication credentials + # + user: myname + tenant: 123456 + apikey: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + + + Compute Region ============== + Rackspace currently has three compute regions which may be used: .. code-block:: @@ -49,12 +80,17 @@ Note: if you are using LON with a UK account, you must use the following identit Authentication ============== -The ``user`` is the same user as is used to log into the Rackspace Control -Panel. The ``tenant`` and ``apikey`` can be found in the API Keys area of the -Control Panel. The ``apikey`` will be labeled as API Key (and may need to be + +The ``user`` is the same user as is used to log into the Rackspace Control +Panel. The ``tenant`` and ``apikey`` can be found in the API Keys area of the +Control Panel. The ``apikey`` will be labeled as API Key (and may need to be generated), and ``tenant`` will be labeled as Cloud Account Number. -An initial profile will be configured in ``/etc/salt/cloud.profiles``: +An initial profile can be configured in ``/etc/salt/cloud.profiles`` or +``/etc/salt/cloud.profiles.d/openstack.conf``: + + +* Using the old cloud configuration format: .. code-block:: yaml @@ -63,6 +99,18 @@ An initial profile will be configured in ``/etc/salt/cloud.profiles``: size: 512MB Standard Instance image: Ubuntu 12.04 LTS (Precise Pangolin) + +* Using the new cloud configuration format and the example configuration from + above: + +.. code-block:: yaml + + openstack_512: + provider: openstack-config + size: 512MB Standard Instance + image: Ubuntu 12.04 LTS (Precise Pangolin) + + To instantiate a machine based on this profile: .. code-block:: bash @@ -70,10 +118,10 @@ To instantiate a machine based on this profile: # salt-cloud -p openstack_512 myinstance This will create a virtual machine at Rackspace with the name ``myinstance``. -This operation may take several minutes to complete, depending on the current +This operation may take several minutes to complete, depending on the current load at the Rackspace data center. -Once the instance has been created with salt-minion installed, connectivity to +Once the instance has been created with salt-minion installed, connectivity to it can be verified with Salt: .. code-block:: bash diff --git a/saltcloud/config.py b/saltcloud/config.py index a853d57a47..fb0ad978e6 100644 --- a/saltcloud/config.py +++ b/saltcloud/config.py @@ -4,6 +4,7 @@ Manage configuration files in salt-cloud # Import python libs import os +import logging # Import salt libs import salt.config @@ -40,6 +41,13 @@ VM_CONFIG_DEFAULTS = { 'default_include': 'cloud.profiles.d/*.conf', } +PROVIDER_CONFIG_DEFAULTS = { + 'default_include': 'cloud.providers.d/*.conf', +} + + +log = logging.getLogger(__name__) + def cloud_config(path, env_var='SALT_CLOUD_CONFIG', defaults=None): ''' @@ -108,13 +116,11 @@ def apply_cloud_config(overrides, defaults=None): # Migrate old configuration opts = old_to_new(opts) - opts = prov_dict(opts) return opts def old_to_new(opts): - optskeys = opts.keys() providers = ('AWS', 'EC2', 'GOGRID', @@ -123,36 +129,24 @@ def old_to_new(opts): 'LINODE', 'OPENSTACK', 'RACKSPACE') - for opt in optskeys: - for provider in providers: - if opt.startswith(provider): - if provider.lower() not in opts: - opts[provider.lower()] = {} - comps = opt.split('.') - opts[provider.lower()][comps[1]] = opts[opt] - return opts - -def prov_dict(opts): - providers = ('AWS', - 'EC2', - 'GOGRID', - 'IBMSCE', - 'JOYENT', - 'LINODE', - 'OPENSTACK', - 'RACKSPACE') - optskeys = opts.keys() - opts['providers'] = {} for provider in providers: - lprov = provider.lower() - opts['providers'][lprov] = {} - for opt in optskeys: - if opt == lprov: - opts['providers'][lprov][lprov] = opts[opt] - elif type(opts[opt]) is dict and 'provider' in opts[opt]: - if opts[opt]['provider'] == lprov: - opts['providers'][lprov][opt] = opts[opt] + + provider_config = {} + for opt in opts.keys(): + if not opt.startswith(provider): + continue + value = opts.pop(opt) + name = opt.split('.', 1)[1] + provider_config[name] = value + + if provider_config: + provider_config['provider'] = provider.lower() + opts.setdefault('providers', {}).setdefault( + provider.lower(), []).append( + provider_config + ) + return opts @@ -195,3 +189,51 @@ def apply_vm_profiles_config(overrides, defaults=None): vms.append(val) return vms + + +def cloud_providers_config(path, + env_var='SALT_CLOUD_PROVIDERS_CONFIG', + defaults=None): + ''' + Read in the salt cloud providers configuration file + ''' + if defaults is None: + defaults = PROVIDER_CONFIG_DEFAULTS + + overrides = salt.config.load_config(path, env_var) + default_include = overrides.get( + 'default_include', defaults['default_include'] + ) + include = overrides.get('include', []) + + overrides.update( + salt.config.include_config(default_include, path, verbose=False) + ) + overrides.update( + salt.config.include_config(include, path, verbose=True) + ) + return apply_cloud_providers_config(overrides, defaults) + + +def apply_cloud_providers_config(overrides, defaults=None): + if defaults is None: + defaults = PROVIDER_CONFIG_DEFAULTS + + opts = defaults.copy() + if overrides: + opts.update(overrides) + + # Is the user still using the old format in the new configuration file?! + converted_opts = old_to_new(opts.copy()) + if opts != converted_opts: + log.warn('Please switch to the new providers configuration syntax') + opts = converted_opts + + providers = {} + + for key, val in opts.items(): + if key in ('conf_file', 'include', 'default_include'): + continue + providers[key] = val + + return providers diff --git a/saltcloud/utils/parsers.py b/saltcloud/utils/parsers.py index 8d01c4a163..53a60480b8 100644 --- a/saltcloud/utils/parsers.py +++ b/saltcloud/utils/parsers.py @@ -27,6 +27,7 @@ class CloudConfigMixIn(object): self.master_config = {} self.cloud_config = {} self.profiles_config = {} + self.providers_config = {} group = self.config_group = optparse.OptionGroup( self, "Configuration Options", @@ -50,6 +51,12 @@ class CloudConfigMixIn(object): help='The location of the saltcloud VM config file. ' 'Default: /etc/salt/cloud.profiles' ) + group.add_option( + '--providers-config', + default=None, + help='The location of the salt cloud VM providers ' + 'configuration file. Default: /etc/salt/cloud.providers' + ) self.add_option_group(group) def __assure_absolute_paths(self, name): @@ -84,7 +91,20 @@ class CloudConfigMixIn(object): # Loaded in CloudConfigMixIn.process_vm_config() self.config['vm'] = self.profiles_config - # 4th - Override config with cli options + # 4th - Include Cloud Providers + if 'providers' in self.config and self.providers_config: + self.error( + 'Do not mix the old cloud providers configuration with ' + 'the new one. The providers configuration should now go in ' + 'the file `/etc/salt/cloud.providers` or a separate `*.conf` ' + 'file within `cloud.providers.d/` which is relative to ' + '`/etc/salt/cloud.providers`. To provide another location ' + 'for the providers configuration file, please use ' + '`--providers-config`.' + ) + self.config['providers'] = self.providers_config + + # 5th - Override config with cli options # Done in parsers.MergeConfigMixIn.__merge_config_with_cli() # Remove log_level_logfile from config if set to None so it can be @@ -120,6 +140,13 @@ class CloudConfigMixIn(object): self.options.vm_config = self.cloud_config.get( 'vm_config', '/etc/salt/cloud.profiles' ) + if self.options.providers_config is None: + # No providers config was provided from cli + # Set the profiles configuration file path to the one provided in + # the cloud's configuration or the default path. + self.options.providers_config = self.cloud_config.get( + 'providers_config', '/etc/salt/cloud.providers' + ) def process_master_config(self): self.master_config = salt.config.master_config( @@ -132,9 +159,16 @@ class CloudConfigMixIn(object): self.profiles_config = config.vm_profiles_config( self.options.vm_config ) - # Force process_vm_config to run AFTER process_cloud_config + # Force process_vm_config to run AFTER process_master_config process_vm_config._mixin_prio_ = -998 + def process_providers_config(self): + self.providers_config = config.cloud_providers_config( + self.options.providers_config + ) + # Force process_providers_config to run AFTER process_vm_config + process_providers_config._mixin_prio_ = -997 + class ExecutionOptionsMixIn(object): __metaclass__ = parsers.MixInMeta @@ -154,18 +188,18 @@ class ExecutionOptionsMixIn(object): group.add_option( '-a', '--action', default='', - help=('Perform an action that may be specific to this cloud ' - 'provider. This argument requires one or more instance ' - 'names to be specified.') + help='Perform an action that may be specific to this cloud ' + 'provider. This argument requires one or more instance ' + 'names to be specified.' ) group.add_option( '-f', '--function', nargs=2, default='', metavar=' ', - help=('Perform an function that may be specific to this cloud ' - 'provider, that does not apply to an instance. This ' - 'argument requires a provider to be specified (i.e.: nova).') + help='Perform an function that may be specific to this cloud ' + 'provider, that does not apply to an instance. This ' + 'argument requires a provider to be specified (i.e.: nova).' ) group.add_option( '-p', '--profile',