Extend ec2 module to support spot instances

library/cloud/ec2

@@ -67,6 +67,12 @@ options:
     required: true
     default: null
     aliases: []
+  spot_price:
+    description:
+      - Maximum spot price to bid, If not set a regular on-demand instance is requested. A spot request is made with this maximum bid. When it is filled, the instance is started.
+    required: false
+    default: null
+    aliases: []
   image:
     description:
        - I(emi) (or I(ami)) to use for the instance
@@ -97,6 +103,11 @@ options:
       - how long before wait gives up, in seconds
     default: 300
     aliases: []
+  spot_wait_timeout:
+    description:
+      - how long to wait for the spot instance request to be fulfilled
+    default: 600
+    aliases: []
   ec2_url:
     description:
       - Url to use to connect to EC2 or your Eucalyptus cloud (by default the module will use EC2 endpoints).  Must be specified if region is not used. If not set then the value of the EC2_URL environment variable, if any, is used
@@ -247,6 +258,19 @@ local_action:
     vpc_subnet_id: subnet-29e63245
     assign_public_ip: yes
 
+# Spot instance example
+- local_action:
+    module: ec2
+    spot_price: 0.24
+    spot_wait_timeout: 600
+    keypair: mykey
+    group_id: sg-1dc53f72
+    instance_type: m1.small
+    image: ami-6e649707
+    wait: yes
+    vpc_subnet_id: subnet-29e63245
+    assign_public_ip: yes
+
 # Launch instances, runs some tasks
 # and then terminate them
 
@@ -392,6 +416,7 @@ def create_instances(module, ec2):
     group_id = module.params.get('group_id')
     zone = module.params.get('zone')
     instance_type = module.params.get('instance_type')
+    spot_price = module.params.get('spot_price')
     image = module.params.get('image')
     count = module.params.get('count')
     monitoring = module.params.get('monitoring')
@@ -399,6 +424,7 @@ def create_instances(module, ec2):
     ramdisk = module.params.get('ramdisk')
     wait = module.params.get('wait')
     wait_timeout = int(module.params.get('wait_timeout'))
+    spot_wait_timeout = int(module.params.get('spot_wait_timeout'))
     placement_group = module.params.get('placement_group')
     user_data = module.params.get('user_data')
     instance_tags = module.params.get('instance_tags')
@@ -456,16 +482,12 @@ def create_instances(module, ec2):
         try:
             params = {'image_id': image,
                       'key_name': key_name,
-                      'client_token': id,
-                      'min_count': count_remaining,
-                      'max_count': count_remaining,
                       'monitoring_enabled': monitoring,
                       'placement': zone,
                       'placement_group': placement_group,
                       'instance_type': instance_type,
                       'kernel_id': kernel,
                       'ramdisk_id': ramdisk,
-                      'private_ip_address': private_ip,
                       'user_data': user_data}
 
             if boto_supports_profile_name_arg(ec2):
@@ -498,22 +520,55 @@ def create_instances(module, ec2):
                 else:
                     params['security_groups'] = group_name
 
-            res = ec2.run_instances(**params)
+            if not spot_price:
+                params.update({
+                    'min_count': count_remaining,
+                    'max_count': count_remaining,
+                    'client_token': id,
+                    'private_ip_address': private_ip,
+                })
+                res = ec2.run_instances(**params)
+                instids = [ i.id for i in res.instances ]
+                while True:
+                    try:
+                        ec2.get_all_instances(instids)
+                        break
+                    except boto.exception.EC2ResponseError as e:
+                        if "<Code>InvalidInstanceID.NotFound</Code>" in str(e):
+                            # there's a race between start and get an instance
+                            continue
+                        else:
+                            module.fail_json(msg = str(e))
+            else:
+                if private_ip:
+                    module.fail_json(
+                        msg='private_ip only available with on-demand (non-spot) instances')
+                params.update({
+                    'count': count_remaining,
+                })
+                res = ec2.request_spot_instances(spot_price, **params)
+                #Now we have to do the intermediate waiting
+                if wait:
+                    spot_req_inst_ids = dict()
+                    spot_wait_timeout = time.time() + spot_wait_timeout
+                    while spot_wait_timeout > time.time():
+                        reqs = ec2.get_all_spot_instance_requests()
+                        for sirb in res:
+                            if sirb.id in spot_req_inst_ids:
+                                continue
+                            for sir in reqs:
+                                if sir.id == sirb.id and sir.instance_id is not None:
+                                    spot_req_inst_ids[sirb.id] = sir.instance_id
+                        if len(spot_req_inst_ids) < count:
+                            time.sleep(5)
+                        else:
+                            break
+                    if spot_wait_timeout <= time.time():
+                        module.fail_json(msg = "wait for spot requests timeout on %s" % time.asctime())
+                    instids = spot_req_inst_ids.values()
         except boto.exception.BotoServerError, e:
             module.fail_json(msg = "%s: %s" % (e.error_code, e.error_message))
 
-        instids = [ i.id for i in res.instances ]
-        while True:
-            try:
-                res.connection.get_all_instances(instids)
-                break
-            except boto.exception.EC2ResponseError as e:
-                if "<Code>InvalidInstanceID.NotFound</Code>" in str(e):
-                    # there's a race between start and get an instance
-                    continue
-                else:
-                    module.fail_json(msg = str(e))
-
         if instance_tags:
             try:
                 ec2.create_tags(instids, instance_tags)
@@ -521,15 +576,14 @@ def create_instances(module, ec2):
                 module.fail_json(msg = "%s: %s" % (e.error_code, e.error_message))
 
         # wait here until the instances are up
-        this_res = []
         num_running = 0
         wait_timeout = time.time() + wait_timeout
         while wait_timeout > time.time() and num_running < len(instids):
-            res_list = res.connection.get_all_instances(instids)
+            res_list = ec2.get_all_instances(instids)
-            if len(res_list) > 0:
+            num_running = 0
-                this_res = res_list[0]
+            for res in res_list:
-                num_running = len([ i for i in this_res.instances if i.state=='running' ])
+                num_running += len([ i for i in res.instances if i.state=='running' ])
-            else:
+            if len(res_list) <= 0:
                 # got a bad response of some sort, possibly due to 
                 # stale/cached data. Wait a second and then try again
                 time.sleep(1)
@@ -543,8 +597,9 @@ def create_instances(module, ec2):
             # waiting took too long
             module.fail_json(msg = "wait for instances running timeout on %s" % time.asctime())
 
-        for inst in this_res.instances:
+        #We do this after the loop ends so that we end up with one list
-            running_instances.append(inst)
+        for res in res_list:
+            running_instances.extend(res.instances)
 
     instance_dict_array = []
     created_instance_ids = []
@@ -631,6 +686,7 @@ def main():
             region = dict(aliases=['aws_region', 'ec2_region'], choices=AWS_REGIONS),
             zone = dict(aliases=['aws_zone', 'ec2_zone']),
             instance_type = dict(aliases=['type']),
+            spot_price = dict(),
             image = dict(),
             kernel = dict(),
             count = dict(default='1'),
@@ -638,6 +694,7 @@ def main():
             ramdisk = dict(),
             wait = dict(type='bool', default=False),
             wait_timeout = dict(default=300),
+            spot_wait_timeout = dict(default=600),
             ec2_url = dict(),
             ec2_secret_key = dict(aliases=['aws_secret_key', 'secret_key'], no_log=True),
             ec2_access_key = dict(aliases=['aws_access_key', 'access_key']),