[cs615asa] [git commit] CS615 EBS-BACKUP; backup a directory into Elastic Block Storage (EBS) branch main updated. 0e0bceae43d7551cb1beb6f726cc2b3e121d0d6b
Git Owner
jschauma at stevens.edu
Sat May 8 18:40:49 EDT 2021
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "CS615 EBS-BACKUP; backup a directory into Elastic Block Storage (EBS)".
The branch, main has been updated
via 0e0bceae43d7551cb1beb6f726cc2b3e121d0d6b (commit)
via 3fa9d6e5266615ac6e3b9238baf4c5b6a91ad769 (commit)
via 67796d5828ae13afcfbcfe2122015c111203fc00 (commit)
via 4f044ddc207b2c437ba6c6d566fdf107eb5b68a1 (commit)
via 9d99736bb784e97e77bfae5fc53118919a208f4c (commit)
via cf4536739697aae956ea315575e0002384cd522c (commit)
from 721aa150ffd8f463e53b30f317b81553f8fa766f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 0e0bceae43d7551cb1beb6f726cc2b3e121d0d6b
Author: Justin Ho <jho1 at stevens.edu>
Date: Sat May 8 17:52:38 2021 -0400
Added most of the documentation for ec2.py
diff --git a/src/ec2.py b/src/ec2.py
index b68d05d..21e0fce 100644
--- a/src/ec2.py
+++ b/src/ec2.py
@@ -1,3 +1,9 @@
+"""Facilitates working with the AWS EC2 instance API to create instances.
+
+Todo:
+ *
+"""
+
import json
import os
import yaml
@@ -5,8 +11,35 @@ import re
from botocore.exceptions import ClientError
-
class EC2(object):
+ """Spins up an AWS EC2 instance.
+
+ You can instead specify an already existing instance instead of spinning
+ up an entirely new instance. To do so, you may specify in `config` an
+ instance ID in use in the specified AWS session.
+
+ Otherwise, a default instance will be spun up: a t2.micro NetBSD instance
+ (ID: `ami-0018b2d98332ba7e3`) named `ebs-backup` set in the us-east-1a
+ availability zone.
+
+ You may also override the default configuration details for spinning up
+ an instance via the `EBS_BACKUP_FLAGS_AWS` environment variable.
+
+ Args:
+ session (boto3.session.Session): The AWS session generated by the
+ boto3 module.
+ config (dict): Optional. The configuration details to spin up the
+ instance - this should contain an instance ID if specified.
+
+ Attributes:
+ session (boto3.session.Session): The AWS session generated by the
+ boto3 module.
+ ec2_client (boto3.session.Session.client): The AWS client assoc-
+ iated with the AWS session.
+ instance (boto3.EC2.Instance): The spun up EC2 instance.
+ instance_id (str): The specified instance ID to spin up.
+ """
+
session = None
ec2_client = None
instance = None
@@ -24,7 +57,8 @@ class EC2(object):
self.instance = i
self.instance_id = i.instance_id
if self.instance is None:
- raise Exception('Could not find an instance with ID: ' + config['instance_id'])
+ raise Exception('Could not find an instance with ID: '
+ + config['instance_id'])
else:
args = {
'ImageId': 'ami-0018b2d98332ba7e3',
@@ -32,13 +66,15 @@ class EC2(object):
'MaxCount': 1,
'InstanceType': 't2.micro',
'KeyName': 'ebs-backup',
- 'Placement': {"AvailabilityZone": config.get('zone_id', 'us-east-1a')}
+ 'Placement': {"AvailabilityZone": config.get('zone_id',
+ 'us-east-1a')}
}
# TODO: need to parse 'EBS_BACKUP_FLAGS_AWS' in
# order to override defaults...
if os.getenv('EBS_BACKUP_FLAGS_AWS') is not None:
- args = EC2.parse_overrides(os.getenv('EBS_BACKUP_FLAGS_AWS'), args)
+ args = EC2.parse_overrides(os.getenv('EBS_BACKUP_FLAGS_AWS'),
+ args)
instance = self.ec2_client.run_instances(**args)
@@ -51,12 +87,36 @@ class EC2(object):
@staticmethod
def primitive_check(value):
+ """Checks if the specified value is a bare value.
+
+ This function facilitates parsing the `EBS_BACKUP_FLAGS` environment
+ variables, made to check the values passed to each flag. A value is
+ considered a primitive if it is "bare," meaning the value is not a
+ list, dictionary, or a key-value pair.
+
+ Args:
+ value (str): The string to check for primitiveness.
+
+ Returns:
+ True if the value is a primitive; false otherwise.
+ """
+
if '[' not in value and '{' not in value and '=' not in value and ',' not in value:
return True
return False
@staticmethod
def parse_values(value, pl):
+ """
+
+ Args:
+ value:
+ pl:
+
+ Returns:
+
+ """
+
if EC2.primitive_check(value):
return { 'pl': value, 'n': len(value) }
cursor = 0
@@ -129,6 +189,22 @@ class EC2(object):
@staticmethod
def parse_overrides(override_string, args):
+ """
+
+ This function is specifically built to parse the configuration over-
+ rides present in the `EBS_BACKUP_FLAGS_AWS` environment variable. In
+ particular, this function parses the flags and their associated values
+ and subsequently adds them to a dictionary of arguments
+
+ Args:
+ override_string (str):
+
+ args (dict):
+
+ Returns:
+
+ """
+
if '--cli-input-json' in override_string:
file = override_string.split('--cli-input-json')[1].trim().split(' ')[0]
return json.loads(file)
@@ -164,6 +240,25 @@ class EC2(object):
return args
def get_a_zone(self):
+ """Gets a suitable availability zone available to the client.
+
+ If the client has the permissions to describe the availability zones
+ available to it, the first availability zone the client finds in the
+ `available` state is the suitable zone returned.
+
+ If the verbose flag is in the environment, a JSON of the availability
+ zones is printed.
+
+ Returns:
+ The first availability zone in the `available` state.
+
+ Raises:
+ UnauthorizedOperation: The client does not have the permissions
+ to describe the availability zones available to it.
+ Exception: The client could not find an availability zone in the
+ `available` state.
+ """
+
try:
self.ec2_client.describe_availability_zones(DryRun=True)
except ClientError as e:
@@ -188,12 +283,14 @@ class EC2(object):
def get_ip(self):
# Seems like we need to call reload otherwise the DNS name is blank
# https://stackoverflow.com/a/60880894/1613023
+
self.instance.reload()
return self.instance.public_ip_address
def get_dns(self):
# Seems like we need to call reload otherwise the DNS name is blank
# https://stackoverflow.com/a/60880894/1613023
+
self.instance.reload()
return self.instance.public_dns_name
@@ -205,15 +302,13 @@ class EC2(object):
)
def cleanup(self):
+ """Terminates the instance.
+
+ Foregoes stopping then terminating instances, since non-graceful
+ exits can unintentially build up useless stopped instances.
+ """
+
try:
- # Terminate rather than stopping then terminating
- # otherwise non-graceful exits could result in a
- # build-up of useless stopped instances
-
- # res1 = self.ec2_client.stop_instances(
- # InstanceIds=[self.instance_id],
- # DryRun=False
- # )
res = self.instance.terminate()
if 'EBS_BACKUP_VERBOSE' in os.environ and os.environ['EBS_BACKUP_VERBOSE']:
commit 3fa9d6e5266615ac6e3b9238baf4c5b6a91ad769
Author: Justin Ho <jho1 at stevens.edu>
Date: Fri May 7 21:09:27 2021 -0400
Fixed volume size logic, added more class explanation, fitted text to 80-character width
diff --git a/src/volume.py b/src/volume.py
index b9e417d..f0ea10f 100644
--- a/src/volume.py
+++ b/src/volume.py
@@ -1,10 +1,12 @@
"""Facilitates working with the AWS EC2 volume API to create backup volumes.
Todo:
- * Must be able to determine a suitable raw disk device (`device_name`); it may differ
- depending on the instance type
- * The size of the volume should be at least 2x as large as the directory to be backed up
- * Should check if an existing volume is not large enough to fit the above constraint
+ * Must be able to determine a suitable raw disk device (`device_name`);
+ it may differ depending on the instance type
+ * The size of the volume should be checked to be at least 2x as large as
+ the directory to be backed up
+ * Should check if an existing volume is not large enough to fit the above
+ constraint
* Should validate that `volume_id` is set in `delete()`
"""
@@ -12,21 +14,41 @@ from botocore.exceptions import ClientError
class Volume(object):
"""Spins up an AWS EBS volume for attaching to an EC2 instance.
+
+ You can configure the volume you wish to spin up. If you wish to spin up
+ a specific volume, you may specify a volume ID.
+
+ Otherwise, you may also specify the size of the directory you wish to
+ backup (in KiB) - the resulting size of the volume will be double the
+ size in GiB, floored. If the resulting size is less than 1 GiB, the size
+ of the volume will automatically be set to 1 GiB. Additionally, the volume
+ will be set in the us-east-1a availability zone.
+
+ If no configuration details are specified, a default volume of 1 GiB in
+ the us-east-1a availability zone is spun up.
Args:
- session (boto3.session.Session): The AWS session generated by the boto3 module.
- config (dict): Optional. The configuration details to create the volume - this should
- contain a volume ID or the volume size if specified.
+ session (boto3.session.Session):
+ The AWS session generated by the boto3 module.
+ config (dict):
+ Optional. The configuration details to create the volume - this
+ should contain a volume ID or the volume size (in KiB) if specified.
Attributes:
- session (boto3.session.Session): The AWS session generated by the boto3 module.
- ec2_client (boto3.session.Session.client): The AWS client associated with the AWS session.
- ec2_volume (EC2.Volume): The volume to be attached to the client instance.
- volume_id (str): The ID of the volume.
- device_name (str): The name of the block device to attach the volume to.
+ session (boto3.session.Session):
+ The AWS session generated by the boto3 module.
+ ec2_client (boto3.session.Session.client):
+ The AWS client associated with the AWS session.
+ ec2_volume (EC2.Volume):
+ The volume to be attached to the client instance.
+ volume_id (str):
+ The ID of the volume.
+ device_name (str):
+ The name of the block device to attach the volume to.
"""
- # From the session instance we can access all services via the either `client` or `resource`
+ # From the session instance we can access all services via the either
+ # `client` or `resource`
session = None
ec2_client = None
@@ -34,8 +56,8 @@ class Volume(object):
volume_id = None
# Chosen from: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
- # Note: the name of the device on the machine doesn't necessarily match the volume.
- #
+ # Note: the name of the device on the machine doesn't necessarily match
+ # the volume.
# device_name in main is the device in NetBSD after it is attached.
# This is the device name for the AWS function parameter.
@@ -50,9 +72,9 @@ class Volume(object):
# Creates a new volume if no volume ID is specified.
if not ('volume_id' in config) or config['volume_id'] is None:
- # Volume sizes are in gigabytes
+ # Volume sizes are in gibibytes
- true_size = (config['size'] / 1000 / 1000) * 2
+ true_size = (config['size'] // 1024 // 1024) * 2
true_size = 1 if true_size < 1 else true_size
res = self.ec2_client.create_volume(
@@ -72,12 +94,12 @@ class Volume(object):
def attach_to_instance(self, instance_id):
"""Attaches the EBS volume to a given instance.
- The volume can be attached to the instance regardless if the instance is running or
- stopped. If attaching an encrypted EBS volume, the instance must support Amazon EBS
- encryption.
+ The volume can be attached to the instance regardless if the instance
+ is running or stopped. If attaching an encrypted EBS volume, the
+ instance must support Amazon EBS encryption.
- After attaching the volume, you must make the volume available to use on the
- instance.
+ After attaching the volume, you must make the volume available to use
+ on the instance.
Args:
instance_id (str): The instance to attach the EBS volume to.
@@ -89,18 +111,20 @@ class Volume(object):
DryRun=False
)
- res['ResponseMetadata'] # Note: potential metadata we might be interested in.
+ res['ResponseMetadata'] # Note: metadata we might be interested in.
def cleanup(self, instance_id):
"""Detaches the EBS volume from a given instance.
- Any file systems on the device should be unmounted before detaching the volume -
- otherwise, the volume may become stuck in the busy state while detaching,
- indefinitely delaying the detachment until manual intervention.
+ Any file systems on the device should be unmounted before detaching
+ the volume - otherwise, the volume may become stuck in the busy state
+ while detaching, indefinitely delaying the detachment until manual
+ intervention.
Args:
instance_id (str): The instance to detach the EBS volume from.
"""
+
self.ec2_volume.detach_from_instance(
Device=self.device_name,
InstanceId=instance_id,
@@ -112,13 +136,13 @@ class Volume(object):
def delete(self):
"""Deletes the EBS volume.
- The volume must be in the available state - meaning the volume must not be attached to
- an instance - to be deleted.
+ The volume must be in the available state - meaning the volume must
+ not be attached to an instance - to be deleted.
"""
res = self.ec2_client.delete_volume(
VolumeId=self.volume_id
)
- res['ResponseMetadata'] # Note: potential metadata we might be interested in.
+ res['ResponseMetadata'] # Note: metadata we might be interested in.
commit 67796d5828ae13afcfbcfe2122015c111203fc00
Merge: 4f044dd cf45367
Author: Justin Ho <jho1 at stevens.edu>
Date: Fri May 7 17:14:47 2021 -0400
Merge contributors
diff --cc CONTRIBUTORS
index 563e4b6,25e7dc6..4eae7a7
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@@ -5,9 -5,7 +5,10 @@@ Username Account Added Confirmed Ac
-------------------------------------------
jschauma 2021-02-07 yes
cmagyar 2021-02-13 yes
-hdaly1 2021-02-14
+hdaly1 2021-02-14 yes
ngoldste 2021-02-16 yes
+hdevlin 2021-02-22 yes
ajin1 2021-02-22 yes
-jho1 2021-03-08 yes
+camatang 2021-02-20 yes
+ddelaus 2021-05-05 yes
++jho1 2021-03-08 yes
commit 4f044ddc207b2c437ba6c6d566fdf107eb5b68a1
Merge: 9d99736 721aa15
Author: Justin Ho <jho1 at stevens.edu>
Date: Fri May 7 16:28:53 2021 -0400
Merged device_name comment
diff --cc src/volume.py
index 7c2390c,b7e1801..b9e417d
--- a/src/volume.py
+++ b/src/volume.py
@@@ -1,43 -1,23 +1,45 @@@
-# WIP: "Backup Volume" class for working with ec2 volume API
-from botocore.exceptions import ClientError
+"""Facilitates working with the AWS EC2 volume API to create backup volumes.
+
+Todo:
- * `device_name` should be a global variable (it is used in `main.py` as well)
+ * Must be able to determine a suitable raw disk device (`device_name`); it may differ
+ depending on the instance type
+ * The size of the volume should be at least 2x as large as the directory to be backed up
+ * Should check if an existing volume is not large enough to fit the above constraint
+ * Should validate that `volume_id` is set in `delete()`
+"""
+from botocore.exceptions import ClientError
class Volume(object):
+ """Spins up an AWS EBS volume for attaching to an EC2 instance.
+
+ Args:
+ session (boto3.session.Session): The AWS session generated by the boto3 module.
+ config (dict): Optional. The configuration details to create the volume - this should
+ contain a volume ID or the volume size if specified.
+
+ Attributes:
+ session (boto3.session.Session): The AWS session generated by the boto3 module.
+ ec2_client (boto3.session.Session.client): The AWS client associated with the AWS session.
+ ec2_volume (EC2.Volume): The volume to be attached to the client instance.
+ volume_id (str): The ID of the volume.
+ device_name (str): The name of the block device to attach the volume to.
+ """
# From the session instance we can access all services via the either `client` or `resource`
+
session = None
ec2_client = None
ec2_volume = None
volume_id = None
- # chosen from: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
- # note: the name of the device on the machine doesn't necessarily match the volume
-
- #
- # TODO - should be global variable (used in main.py too)
- # -> Not a global variable. Device_name in main is the device in NetBSD after it's attached.
- # -> This is the device name for the AWS function parameter.
- device_name = "/dev/sdf" # Just for example. Should determine based on instance type
+ # Chosen from: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
- # Note: the name of the device on the machine doesn't necessarily match the volume
-
- device_name = "/dev/sdf" # Just for example. Should determine based on instance type.
++ # Note: the name of the device on the machine doesn't necessarily match the volume.
++ #
++ # device_name in main is the device in NetBSD after it is attached.
++ # This is the device name for the AWS function parameter.
++
++ device_name = "/dev/sdf"
def __init__(self, session, config=None):
if config is None:
commit 9d99736bb784e97e77bfae5fc53118919a208f4c
Author: Justin Ho <jho1 at stevens.edu>
Date: Fri May 7 16:17:09 2021 -0400
Added documentation to volume module
diff --git a/src/volume.py b/src/volume.py
index c0826b9..7c2390c 100644
--- a/src/volume.py
+++ b/src/volume.py
@@ -1,20 +1,43 @@
-# WIP: "Backup Volume" class for working with ec2 volume API
-from botocore.exceptions import ClientError
+"""Facilitates working with the AWS EC2 volume API to create backup volumes.
+
+Todo:
+ * `device_name` should be a global variable (it is used in `main.py` as well)
+ * Must be able to determine a suitable raw disk device (`device_name`); it may differ
+ depending on the instance type
+ * The size of the volume should be at least 2x as large as the directory to be backed up
+ * Should check if an existing volume is not large enough to fit the above constraint
+ * Should validate that `volume_id` is set in `delete()`
+"""
+from botocore.exceptions import ClientError
class Volume(object):
+ """Spins up an AWS EBS volume for attaching to an EC2 instance.
+
+ Args:
+ session (boto3.session.Session): The AWS session generated by the boto3 module.
+ config (dict): Optional. The configuration details to create the volume - this should
+ contain a volume ID or the volume size if specified.
+
+ Attributes:
+ session (boto3.session.Session): The AWS session generated by the boto3 module.
+ ec2_client (boto3.session.Session.client): The AWS client associated with the AWS session.
+ ec2_volume (EC2.Volume): The volume to be attached to the client instance.
+ volume_id (str): The ID of the volume.
+ device_name (str): The name of the block device to attach the volume to.
+ """
# From the session instance we can access all services via the either `client` or `resource`
+
session = None
ec2_client = None
ec2_volume = None
volume_id = None
- # chosen from: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
- # note: the name of the device on the machine doesn't necessarily match the volume
+ # Chosen from: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
+ # Note: the name of the device on the machine doesn't necessarily match the volume
- # TODO - should be global variable (used in main.py too)
- device_name = "/dev/sdf" # Just for example. Should determine based on instance type
+ device_name = "/dev/sdf" # Just for example. Should determine based on instance type.
def __init__(self, session, config=None):
if config is None:
@@ -22,41 +45,60 @@ class Volume(object):
self.session = session
self.ec2_client = session.client('ec2')
- # for new volumes
+ # Creates a new volume if no volume ID is specified.
+
if not ('volume_id' in config) or config['volume_id'] is None:
-
- # volume sizes are in gigabytes
+ # Volume sizes are in gigabytes
+
true_size = (config['size'] / 1000 / 1000) * 2
true_size = 1 if true_size < 1 else true_size
res = self.ec2_client.create_volume(
AvailabilityZone='us-east-1a',
- # TODO: this needs to be 2x the dir that is being backed up
Size=true_size,
)
self.volume_id = res['VolumeId']
else:
self.volume_id = config['volume_id']
- # for existing volumes
- # TODO: what if the existing volume is not large enough
+
+ # Associates the newly-created or existing volume to the session.
+
self.ec2_volume = session.resource('ec2').Volume(self.volume_id)
if self.ec2_volume is None:
raise Exception("The provided volume ID does not exist")
- # Instance doesn't need to be running, according to boto3 documentation for volume.attach_volume
- # Encrypted EBS volumes must be attached to instances that support Amazon EBS encryption
- # After volume is attached, it must be made available
- # TODO: Must find out how to determine suitable raw disk device, may differ depending on the instance type
def attach_to_instance(self, instance_id):
+ """Attaches the EBS volume to a given instance.
+
+ The volume can be attached to the instance regardless if the instance is running or
+ stopped. If attaching an encrypted EBS volume, the instance must support Amazon EBS
+ encryption.
+
+ After attaching the volume, you must make the volume available to use on the
+ instance.
+
+ Args:
+ instance_id (str): The instance to attach the EBS volume to.
+ """
+
res = self.ec2_volume.attach_to_instance(
Device=self.device_name,
InstanceId=instance_id,
DryRun=False
)
- # Note: potential metadata we might be interested in
- res['ResponseMetadata']
+
+ res['ResponseMetadata'] # Note: potential metadata we might be interested in.
def cleanup(self, instance_id):
+ """Detaches the EBS volume from a given instance.
+
+ Any file systems on the device should be unmounted before detaching the volume -
+ otherwise, the volume may become stuck in the busy state while detaching,
+ indefinitely delaying the detachment until manual intervention.
+
+ Args:
+ instance_id (str): The instance to detach the EBS volume from.
+ """
self.ec2_volume.detach_from_instance(
Device=self.device_name,
InstanceId=instance_id,
@@ -66,10 +108,15 @@ class Volume(object):
return self.volume_id
def delete(self):
- # TODO: ensure `volume_id` is set
+ """Deletes the EBS volume.
+
+ The volume must be in the available state - meaning the volume must not be attached to
+ an instance - to be deleted.
+ """
+
res = self.ec2_client.delete_volume(
VolumeId=self.volume_id
)
- # Note: potential metadata we might be interested in
- res['ResponseMetadata']
+
+ res['ResponseMetadata'] # Note: potential metadata we might be interested in.
commit cf4536739697aae956ea315575e0002384cd522c
Author: Justin Ho <jho1 at stevens.edu>
Date: Mon Mar 8 19:37:04 2021 -0500
added Justin to contributors
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index ff25fcf..25e7dc6 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -8,3 +8,4 @@ cmagyar 2021-02-13 yes
hdaly1 2021-02-14
ngoldste 2021-02-16 yes
ajin1 2021-02-22 yes
+jho1 2021-03-08 yes
\ No newline at end of file
-----------------------------------------------------------------------
Summary of changes:
CONTRIBUTORS | 1 +
src/ec2.py | 119 ++++++++++++++++++++++++++++++++++++++++++++++++++------
src/volume.py | 123 +++++++++++++++++++++++++++++++++++++++++++++-------------
3 files changed, 205 insertions(+), 38 deletions(-)
hooks/post-receive
--
CS615 EBS-BACKUP; backup a directory into Elastic Block Storage (EBS)
More information about the cs615asa
mailing list