Merge branch 'issue26-await-backup-creation'

Conflicts:
	botocross/__init__.py

botocross/__init__.py

@@ -91,6 +91,23 @@ def build_filter_parser(resource_name, add_ids=True):
                                           help="A {0} id. [can be used multiple times]".format(resource_name))
     return parser
 
+def build_backup_parser(resource_name, expire_only=False, backup_retention=None):
+    parser = argparse.ArgumentParser(add_help=False)
+    if not expire_only:
+        parser.add_argument("-d", "--description",
+                            help="A description for the {0} [default: <provided>]".format(resource_name))
+    default_retention_help = str(backup_retention) if backup_retention else "None, i.e. don't expire"
+    parser.add_argument("-br", "--backup_retention", type=int, default=backup_retention,
+                        help="The number of backups to retain (correlated via backup_set). [default: {0}]".format(default_retention_help))
+    parser.add_argument("-bs", "--backup_set", default='default',
+                        help="A backup set name (determines retention correlation). [default: 'default'")
+    if not expire_only:
+        parser.add_argument("-bt", "--backup_timeout", default=None,
+                            help="Maximum duration to await successful resource creation - an ISO 8601 duration, e.g. 'PT8H' (8 hours). [default: None, i.e. don't await]")
+    parser.add_argument("-ns", "--no_origin_safeguard", action="store_true",
+                        help="Allow deletion of images originating from other tools. [default: False]")
+    return parser
+
 def parse_credentials(args):
     return {'aws_access_key_id': args.aws_access_key_id, 'aws_secret_access_key': args.aws_secret_access_key}
 
@@ -122,3 +139,10 @@ def filter_regions_s3(regions, region):
 # pylint: disable=R0903
 class ExitCodes:
     (OK, FAIL) = range(0, 2)
+
+class BotocrossAwaitTimeoutError(StandardError):
+    """
+    Timeout error when awaiting a resource transition.
+    """
+    def __init__(self, message):
+        Exception.__init__(self, message)

botocross/ec2/__init__.py

@@ -21,12 +21,22 @@
 
 from datetime import datetime
 from operator import attrgetter
+import botocross as bc
+import isodate
 import logging
+import time
 ec2_log = logging.getLogger('botocross.ec2')
 
-DEFAULT_BACKUP_SET = "default"
+AWAIT_TRANSITION_DELAY = 4
+AWAIT_TRANSITION_TIMEOUT = "P1D"
 CREATED_BY_BOTO_EBS_SNAPSHOT_SCRIPT_SIGNATURE = "Created by Botocross EBS Snapshot Script from "
 CREATED_BY_BOTO_EC2_IMAGE_SCRIPT_SIGNATURE = "Created by Botocross EC2 Image Script from "
+IMAGE_STATES_PROGRESSING = { 'pending' }
+IMAGE_STATES_SUCCEEDED = { 'available' }
+IMAGE_STATES_FAILED = { 'failed', 'deregistered' }
+SNAPSHOT_STATES_PROGRESSING = { 'pending' }
+SNAPSHOT_STATES_SUCCEEDED = { 'completed' }
+SNAPSHOT_STATES_FAILED = { 'error' }
 TAG_NAME = "Name"
 TAG_BACKUP_POLICY = "Backup Policy"
 
@@ -45,17 +55,63 @@ def derive_name(ec2, resource_id, iso_datetime=None, id_only=False):
     name += "." + iso_datetime.strftime("%Y%m%dT%H%M%SZ")
     return name
 
+def format_states(states):
+    return ', '.join(["{0}: {1}".format(k, len(v)) for (k, v) in states.iteritems()])
+
+def await_resources(ec2, resource_function, resource_type, state_field, timeout, delay):
+    log = ec2_log
+    duration = None
+    end = None
+    states = {}
+
+    if timeout:
+        duration = isodate.parse_duration(timeout)
+        end = datetime.now() + duration
+
+    while True:
+        states = {}
+        resources = resource_function()
+        for resource in resources:
+            states.setdefault(getattr(resource, state_field), []).append(resource)
+
+        # TODO: this should allow arbitrary resources via respective target state(s).
+        if not "pending" in states:
+            log.info("... {0} transitioned ({1})".format(resource_type, format_states(states)))
+            break
+
+        if end and datetime.now() > end:
+            message = "FAILED to transition all {0} after {2} ({1})!".format(resource_type, format_states(states),
+                                                                             isodate.duration_isoformat(duration))
+            log.info("... " + message)
+            raise bc.BotocrossAwaitTimeoutError(message)
+
+        log.info("... {0} still transitioning ({1}) ...".format(resource_type, format_states(states)))
+        time.sleep(delay)
+
+    return states
+
+def await_snapshots(ec2, snapshots, timeout=AWAIT_TRANSITION_TIMEOUT, delay=AWAIT_TRANSITION_DELAY):
+    def list_snapshots():
+        return ec2.get_all_snapshots(snapshot_ids=sorted(snapshots))
+
+    return await_resources(ec2, list_snapshots, "snapshots", "status", timeout=timeout, delay=delay)
+
 def create_snapshots(ec2, volumes, backup_set, description):
     log = ec2_log
+    snapshots = []
     for volume in volumes:
         signature = description if description else CREATED_BY_BOTO_EBS_SNAPSHOT_SCRIPT_SIGNATURE + volume.id
         log.debug("Description: " + signature)
-        response = ec2.create_snapshot(volume.id, description=signature)
+        snapshot = ec2.create_snapshot(volume.id, description=signature)
+        # NOTE: create_image() currently (boto 2.6.0) returns just the id rather than the resource as create_snapshots() does!
+        snapshots.append(snapshot.id)
 
         name = derive_name(ec2, volume.id)
         log.debug(TAG_NAME + ": " + name)
         tags = {TAG_NAME: name, TAG_BACKUP_POLICY: backup_set}
-        ec2.create_tags([response.id], tags)
+        ec2.create_tags([snapshot.id], tags)
+
+    return snapshots
 
 def expire_snapshots(ec2, volume_ids, backup_set, backup_retention, no_origin_safeguard=False):
     log = ec2_log
@@ -79,21 +135,32 @@ def expire_snapshots(ec2, volume_ids, backup_set, backup_retention, no_origin_sa
             log.info("... deleting snapshot '" + snapshot.id + "' ...")
             ec2.delete_snapshot(snapshot.id)
 
+def await_images(ec2, images, timeout=AWAIT_TRANSITION_TIMEOUT, delay=AWAIT_TRANSITION_DELAY):
+    def list_images():
+        return ec2.get_all_images(image_ids=sorted(images), owners=['self'])
+
+    return await_resources(ec2, list_images, "images", "state", timeout=timeout, delay=delay)
+
 def create_images(ec2, instances, backup_set, description, no_reboot=False):
     log = ec2_log
+    images = []
     for instance in instances:
         signature = description if description else CREATED_BY_BOTO_EC2_IMAGE_SCRIPT_SIGNATURE + instance.id
         log.debug("Description: " + signature)
         iso_datetime = datetime.utcnow()
         ami_name = derive_name(ec2, instance.id, iso_datetime, True)
         log.debug("AMI name: " + ami_name)
         image = ec2.create_image(instance.id, name=ami_name, description=signature, no_reboot=no_reboot)
+        # NOTE: create_image() currently (boto 2.6.0) returns just the id rather than the resource as create_snapshots() does!
+        images.append(image)
 
         name = derive_name(ec2, instance.id, iso_datetime)
         log.debug(TAG_NAME + ": " + name)
         tags = {TAG_NAME: name, TAG_BACKUP_POLICY: backup_set}
         ec2.create_tags([image], tags)
 
+    return images
+
 def expire_images(ec2, instance_ids, backup_set, backup_retention, no_origin_safeguard=False):
     log = ec2_log
     for instance_id in instance_ids:

scripts/create-images.py

@@ -27,13 +27,13 @@
 import boto.ec2
 import botocross as bc
 import logging
+import sys
 
 # configure command line argument parsing
 parser = argparse.ArgumentParser(description='Create images of EC2 instances in all/some available EC2 regions',
-                                 parents=[bc.build_region_parser(), bc.build_filter_parser('EC2 instance'), bc.build_common_parser()])
-parser.add_argument("-d", "--description", help="A description for the EC2 image [default: <provided>]")
+                                 parents=[bc.build_region_parser(), bc.build_filter_parser('EC2 instance'),
+                                          bc.build_backup_parser('EC2 instance'), bc.build_common_parser()])
 parser.add_argument("-nr", "--no_reboot", action="store_true", help="Prevent shut down of instance before creating the image. [default: False]")
-parser.add_argument("-bs", "--backup_set", default=DEFAULT_BACKUP_SET, help="A backup set name (determines retention correlation). [default: 'default'")
 args = parser.parse_args()
 
 # process common command line arguments
@@ -42,14 +42,12 @@
 credentials = bc.parse_credentials(args)
 regions = bc.filter_regions(boto.ec2.regions(), args.region)
 filter = bc.build_filter(args.filter, args.exclude)
-log.info(args.resource_ids)
 
 # execute business logic
 log.info("Imaging EC2 instances:")
 
-backup_set = args.backup_set if args.backup_set else DEFAULT_BACKUP_SET
-log.debug(backup_set)
-
+# REVIEW: For backup purposes it seems reasonable to only consider all OK vs. FAIL?!
+exit_code = bc.ExitCodes.OK
 for region in regions:
     try:
         ec2 = boto.connect_ec2(region=region, **credentials)
@@ -58,7 +56,27 @@
             exclusions = ec2.get_all_instances(filters=filter['excludes'])
             reservations = bc.filter_list_by_attribute(reservations, exclusions, 'id')
         instances = [instance for reservation in reservations for instance in reservation.instances]
+
         print region.name + ": " + str(len(instances)) + " instances"
-        create_images(ec2, instances, backup_set, args.description, no_reboot=args.no_reboot)
+        images = create_images(ec2, instances, args.backup_set, args.description, no_reboot=args.no_reboot)
+
+        if args.backup_timeout:
+            try:
+                states = await_images(ec2, images, timeout=args.backup_timeout)
+                if not bc.ec2.IMAGE_STATES_SUCCEEDED.issuperset(states):
+                    message = "FAILED to create some images: {0}!".format(format_states(states))
+                    log.error(message)
+                    exit_code = bc.ExitCodes.FAIL
+            except bc.BotocrossAwaitTimeoutError, e:
+                log.error(e.message)
+                exit_code = bc.ExitCodes.FAIL
+
+        if args.backup_retention:
+            instance_ids = [instance.id for instance in instances]
+            expire_images(ec2, instance_ids, args.backup_set, args.backup_retention, args.no_origin_safeguard)
+
     except boto.exception.BotoServerError, e:
         log.error(e.error_message)
+        exit_code = bc.ExitCodes.FAIL
+
+sys.exit(exit_code)