[cs615asa] [git commit] CS615 EBS-BACKUP; backup a directory into Elastic Block Storage (EBS) branch merge created. d680152bbd86bc7109d59044ed421811bcc7c5ba
Git Owner
jschauma at stevens.edu
Wed Apr 21 15:04:16 EDT 2021
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "CS615 EBS-BACKUP; backup a directory into Elastic Block Storage (EBS)".
The branch, merge has been created
at d680152bbd86bc7109d59044ed421811bcc7c5ba (commit)
- Log -----------------------------------------------------------------
commit d680152bbd86bc7109d59044ed421811bcc7c5ba
Author: Charles Magyar IV <cmagyar at stevens.edu>
Date: Tue Apr 20 23:51:05 2021 -0400
volume not 'existing' (it was, just in a different place) fixed. Added a delay to make sure the volume is attached before attempting to write to it. \n\nAdded enum and constants for clarrity. \n\nNOTE: I commented out a lot of stuff, we should clean it up. \nPrint statements should only be available in verbose mode.\n\n ***In your .ssh/config file, please allow *.amazonaws.com to use IdentityFile ebs-backup
diff --git a/src/main.py b/src/main.py
index 988cfea..58495c4 100644
--- a/src/main.py
+++ b/src/main.py
@@ -9,21 +9,30 @@ import time
from subprocess import Popen, PIPE
from argument_parsing import parse_args
from ec2 import EC2
+from enum import Enum
from volume import Volume
#### TODO's ####
-# [sub ] determine whether to use subprocess or python logic for various operations
+# [sub] determine whether to use subprocess or python logic for various operations
# [ ] create central logging utility around EBS_BACKUP_VERBOSE
-# [soon ] integrate local and remote filter logic
-# [plz no ] potentially reuse the same instance or recreate each time?
-# [ ] clean-up of NetBSD Ami leaves a 10 Gib volume (/dev/sda1), should delete.
+# [soon] integrate local and remote filter logic
+# [plz no] potentially reuse the same instance or recreate each time?
+
### Questions's ###
# [yes] can we have both local and remote filters?
ZONE = 'us-east-1'
KEY_FLAGS = os.getenv('EBS_BACKUP_FLAGS_SSH') or ''
+DEVICE_NAME = 'xbd1' #NetBSD labels devices sequentially.
+MAX_WAIT_TRIES = 12
+TRY_WAIT_TIME = 10 #Seconds
+
+class VolumeStatus(Enum):
+ DETACHED = 'detached'
+ ATTACHED = 'attached'
+ ATTACHING = 'attaching'
# Windows alternative to ~/.aws/credentials & ~/.aws/config (for local/testing purposes)
# achieves the same result of boto3.[resource|client]('service') via
@@ -36,26 +45,22 @@ session = boto3.Session(
client = boto3.client('ec2')
-def pipe_everything(args, target_ip):
-
- device_name = 'sdf'
+def pipe_backup(args, target_ip):
local_filter = args.l
remote_filter = args.r
target_dir = args.dir
-
# specifically root for the NetBSD ami we are using
user = 'root'
host = target_ip
url = user+'@'+host
# key = KEY_FLAGS
local_filter = args.l
- print(url)
tar_process = Popen(['tar', 'cf', '-', target_dir], stdout=PIPE)
if( local_filter is not None ):
local_filter_process = Popen(local_filter.split(' '), stdin=tar_process.stdout, stdout=PIPE)
- ssh_process = Popen(['ssh', '-v', '-o', 'StrictHostKeyChecking=no', url, '"dd', 'of=/dev/'+device_name+'"'], stdin=local_filter_process.stdout)
+ ssh_process = Popen(['ssh', '-v', '-o', 'StrictHostKeyChecking=no', url, '"dd', 'of=/dev/'+DEVICE_NAME+'"'], stdin=local_filter_process.stdout)
else:
- ssh_process = Popen(['ssh', '-v', '-o', 'StrictHostKeyChecking=no', url, 'dd', 'of=/dev/'+device_name+''], stdin=tar_process.stdout)
+ ssh_process = Popen(['ssh', '-v', '-o', 'StrictHostKeyChecking=no', url, 'dd', 'of=/dev/'+DEVICE_NAME+''], stdin=tar_process.stdout)
#TODO - Remote filter.
@@ -68,12 +73,39 @@ def wait_for_instance_ok(instance_id, tries):
instance_status_info = status_dictionary.get('InstanceStatuses')
status = instance_status_info[0].get('InstanceStatus').get('Status')
while(status != 'ok' and i < tries):
- print(status)
- time.sleep(10)
+ print(('Instance {id} Status: ' + status).format(id=instance_id))
+ time.sleep(TRY_WAIT_TIME)
status_dictionary = client.describe_instance_status(InstanceIds=[instance_id])
instance_status_info = status_dictionary.get('InstanceStatuses')
status = instance_status_info[0].get('InstanceStatus').get('Status')
i = i + 1
+ if(i == tries):
+ print('Max tries reached. Moving on...')
+ else:
+ print('Instace {id} Status: ok'.format(id=instance_id)
+
+def wait_for_volume_status(volume_id, target_status, tries):
+ i = 1
+ status_dictionary = client.describe_volumes(VolumeIds=[volume_id])
+ volume_status = status_dictionary.get('Volumes')[0].get('Attachments')[0].get('State')
+ while(volume_status != target_status and i < tries):
+ print('Volume status: ' + volume_status)
+ print('Target status: ' + target_status)
+ time.sleep(TRY_WAIT_TIME)
+ status_dictionary = client.describe_volumes(VolumeIds=[volume_id])
+ if(status_dictionary is not None):
+ volumes = status_dictionary.get('Volumes')
+ if(volumes is not None and len(volumes) > 0):
+ attachments = volumes[0].get('Attachments')
+ if(attachments is not None and len(attachments) > 0):
+ volume_status = attachments[0].get('State')
+ else:
+ break;
+ i = i + 1
+ if(i == tries):
+ print('Max tries completed. Moving on...')
+ else:
+ print('Volume status: ' + volume_status)
def upload(user, host, _dir):
# scp directory to be backed up to ec2 instance
@@ -140,10 +172,15 @@ def backup(args):
'volume_id': args.v,
'size': size,
})
+ # IF VERBOSE...
+ print('Instance spinning up...')
ec2.wait_for_instance()
- # Allow 12 times (2 minutes) for instance to be fully available
- wait_for_instance_ok(ec2.instance_id, 12)
vol.attach_to_instance(ec2.instance_id)
+ wait_for_volume_status(vol.id(), VolumeStatus.ATTACHED.value, MAX_WAIT_TRIES)
+
+ # Allow for instance to be ok to ssh to
+ wait_for_instance_ok(ec2.instance_id, MAX_WAIT_TRIES)
+
# 3. tar ...
# tar(args.dir)
@@ -152,21 +189,21 @@ def backup(args):
# pipe_everything(args, ec2.get_ip())
# pass
- pipe_everything(args, ec2.get_dns())
+ pipe_backup(args, ec2.get_dns())
# 5. scp/ssh ...
#upload('root', ec2.get_ip(), args.dir)
- if args.r is not None:
+# if args.r is not None:
# 6. apply remote-filter ...
- pass
+# pass
# 7. copy to volume ...
#backup_data('root', ec2.get_ip(), args.dir)
- # vol.cleanup(ec2.get_id())
-
+ vol.cleanup(ec2.get_id())
+ wait_for_volume_status(vol.id(), VolumeStatus.DETACHED.value, MAX_WAIT_TRIES)
# 8. teardown ...
- # ec2.cleanup()
+ ec2.cleanup()
# remove local tar after upload
#subprocess.call('rm {dir}.tar')
@@ -177,8 +214,8 @@ def backup(args):
except Exception as ex:
# print meaningful errors to STDERR
print(ex, file=sys.stderr)
- # if ec2 is not None:
- # ec2.cleanup()
+ if ec2 is not None:
+ ec2.cleanup()
if __name__ == '__main__':
commit e7ea0c37739ca675f7698e0348ac1099ad6b0702
Author: Charles Magyar IV <cmagyar at stevens.edu>
Date: Sun Apr 18 23:07:28 2021 -0400
testing shows that a NetBSD instance dd's a tarfile to /dev/xfs. That tarfile can be retrieved from /dev/xfs. However, if the volume is detached and re-attached, it's no longer discernible.
diff --git a/src/main.py b/src/main.py
index 4eeaecc..988cfea 100644
--- a/src/main.py
+++ b/src/main.py
@@ -38,6 +38,7 @@ client = boto3.client('ec2')
def pipe_everything(args, target_ip):
+ device_name = 'sdf'
local_filter = args.l
remote_filter = args.r
target_dir = args.dir
@@ -52,9 +53,9 @@ def pipe_everything(args, target_ip):
tar_process = Popen(['tar', 'cf', '-', target_dir], stdout=PIPE)
if( local_filter is not None ):
local_filter_process = Popen(local_filter.split(' '), stdin=tar_process.stdout, stdout=PIPE)
- ssh_process = Popen(['ssh', '-v', '-o', 'StrictHostKeyChecking=no', url, '"dd', 'of=/dev/sda2"'], stdin=local_filter_process.stdout)
+ ssh_process = Popen(['ssh', '-v', '-o', 'StrictHostKeyChecking=no', url, '"dd', 'of=/dev/'+device_name+'"'], stdin=local_filter_process.stdout)
else:
- ssh_process = Popen(['ssh', '-v', '-o', 'StrictHostKeyChecking=no', url, '"dd', 'of=/dev/sda2"'], stdin=tar_process.stdout)
+ ssh_process = Popen(['ssh', '-v', '-o', 'StrictHostKeyChecking=no', url, 'dd', 'of=/dev/'+device_name+''], stdin=tar_process.stdout)
#TODO - Remote filter.
diff --git a/src/volume.py b/src/volume.py
index b3f97f7..c0826b9 100644
--- a/src/volume.py
+++ b/src/volume.py
@@ -14,7 +14,7 @@ class Volume(object):
# note: the name of the device on the machine doesn't necessarily match the volume
# TODO - should be global variable (used in main.py too)
- device_name = "/dev/sda2" # Just for example. Should determine based on instance type
+ device_name = "/dev/sdf" # Just for example. Should determine based on instance type
def __init__(self, session, config=None):
if config is None:
commit 23434c2c122c0cc6cbe1e9faf759caf617ead763
Author: Charles Magyar IV <cmagyar at stevens.edu>
Date: Sun Apr 18 21:11:21 2021 -0400
Stuck. Piping commands is ok. Tar is created (in local testing). I'm having issues confirming the tar is created on the remote volume. Lots of things commented out of main.py in order to test (all things that deconstruct the instance and detach the volume currently)
diff --git a/src/ec2.py b/src/ec2.py
index 0e45e72..a351225 100644
--- a/src/ec2.py
+++ b/src/ec2.py
@@ -74,6 +74,12 @@ class EC2(object):
self.instance.reload()
return self.instance.public_ip_address
+ def get_dns(self):
+ # Seems like we need to call reload otherwise the DNS name is blank
+ # https://stackoverflow.com/a/60880894/1613023
+ self.instance.reload()
+ return self.instance.public_dns_name
+
def wait_for_instance(self):
waiter = self.ec2_client.get_waiter('instance_running')
waiter.wait(
diff --git a/src/main.py b/src/main.py
index 445b8d1..4eeaecc 100644
--- a/src/main.py
+++ b/src/main.py
@@ -1,23 +1,26 @@
+import atexit
+import boto3
import os
+import pipes
import subprocess
import sys
-import atexit
-
-import boto3
+import time
+from subprocess import Popen, PIPE
+from argument_parsing import parse_args
from ec2 import EC2
from volume import Volume
-from argument_parsing import parse_args
#### TODO's ####
-# [ ] determine whether to use subprocess or python logic for various operations
+# [sub ] determine whether to use subprocess or python logic for various operations
# [ ] create central logging utility around EBS_BACKUP_VERBOSE
-# [ ] integrate local and remote filter logic
-# [ ] potentially reuse the same instance or recreate each time?
+# [soon ] integrate local and remote filter logic
+# [plz no ] potentially reuse the same instance or recreate each time?
+# [ ] clean-up of NetBSD Ami leaves a 10 Gib volume (/dev/sda1), should delete.
### Questions's ###
-# [ ] can we have both local and remote filters?
+# [yes] can we have both local and remote filters?
ZONE = 'us-east-1'
KEY_FLAGS = os.getenv('EBS_BACKUP_FLAGS_SSH') or ''
@@ -31,6 +34,46 @@ session = boto3.Session(
region_name=ZONE,
)
+client = boto3.client('ec2')
+
+def pipe_everything(args, target_ip):
+
+ local_filter = args.l
+ remote_filter = args.r
+ target_dir = args.dir
+
+ # specifically root for the NetBSD ami we are using
+ user = 'root'
+ host = target_ip
+ url = user+'@'+host
+ # key = KEY_FLAGS
+ local_filter = args.l
+ print(url)
+ tar_process = Popen(['tar', 'cf', '-', target_dir], stdout=PIPE)
+ if( local_filter is not None ):
+ local_filter_process = Popen(local_filter.split(' '), stdin=tar_process.stdout, stdout=PIPE)
+ ssh_process = Popen(['ssh', '-v', '-o', 'StrictHostKeyChecking=no', url, '"dd', 'of=/dev/sda2"'], stdin=local_filter_process.stdout)
+ else:
+ ssh_process = Popen(['ssh', '-v', '-o', 'StrictHostKeyChecking=no', url, '"dd', 'of=/dev/sda2"'], stdin=tar_process.stdout)
+
+ #TODO - Remote filter.
+
+ out, err = ssh_process.communicate()
+
+
+def wait_for_instance_ok(instance_id, tries):
+ i = 1
+ status_dictionary = client.describe_instance_status(InstanceIds=[instance_id])
+ instance_status_info = status_dictionary.get('InstanceStatuses')
+ status = instance_status_info[0].get('InstanceStatus').get('Status')
+ while(status != 'ok' and i < tries):
+ print(status)
+ time.sleep(10)
+ status_dictionary = client.describe_instance_status(InstanceIds=[instance_id])
+ instance_status_info = status_dictionary.get('InstanceStatuses')
+ status = instance_status_info[0].get('InstanceStatus').get('Status')
+ i = i + 1
+
def upload(user, host, _dir):
# scp directory to be backed up to ec2 instance
# flags = os.environ['EBS_BACKUP_VERBOSE']
@@ -69,7 +112,6 @@ def calculate_dir_size(dir):
def exit(ec2):
def handle():
ec2.cleanup()
-
return handle
@@ -85,7 +127,7 @@ def backup(args):
})
# If the EC2 instance gets created let's be sure to cleanup
# regardless of manner of exit
- atexit.register(exit(ec2))
+ # atexit.register(exit(ec2))
else:
ec2 = EC2(session, {
'instance_id': args.i,
@@ -98,42 +140,44 @@ def backup(args):
'size': size,
})
ec2.wait_for_instance()
-
+ # Allow 12 times (2 minutes) for instance to be fully available
+ wait_for_instance_ok(ec2.instance_id, 12)
vol.attach_to_instance(ec2.instance_id)
-
# 3. tar ...
- tar(args.dir)
+ # tar(args.dir)
- if args.l is not None:
+ #if args.l is not None:
# 4. apply local-filter ...
- pass
+ # pipe_everything(args, ec2.get_ip())
+ # pass
+ pipe_everything(args, ec2.get_dns())
# 5. scp/ssh ...
- upload('root', ec2.get_ip(), args.dir)
+ #upload('root', ec2.get_ip(), args.dir)
if args.r is not None:
# 6. apply remote-filter ...
pass
# 7. copy to volume ...
- backup_data('root', ec2.get_ip(), args.dir)
+ #backup_data('root', ec2.get_ip(), args.dir)
- vol.cleanup(ec2.get_id())
+ # vol.cleanup(ec2.get_id())
# 8. teardown ...
- ec2.cleanup()
+ # ec2.cleanup()
# remove local tar after upload
- subprocess.call('rm {dir}.tar')
+ #subprocess.call('rm {dir}.tar')
# If successful, ebs - backup will print the volume - id of the volume
# to which it backed up the data as the only output.
- vol.id()
+ print(vol.id())
except Exception as ex:
# print meaningful errors to STDERR
print(ex, file=sys.stderr)
- if ec2 is not None:
- ec2.cleanup()
+ # if ec2 is not None:
+ # ec2.cleanup()
if __name__ == '__main__':
diff --git a/src/volume.py b/src/volume.py
index 546d97f..b3f97f7 100644
--- a/src/volume.py
+++ b/src/volume.py
@@ -12,7 +12,9 @@ class Volume(object):
# chosen from: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
# note: the name of the device on the machine doesn't necessarily match the volume
- device_name = "/dev/xbd1" # Just for example. Should determine based on instance type
+
+ # TODO - should be global variable (used in main.py too)
+ device_name = "/dev/sda2" # Just for example. Should determine based on instance type
def __init__(self, session, config=None):
if config is None:
-----------------------------------------------------------------------
hooks/post-receive
--
CS615 EBS-BACKUP; backup a directory into Elastic Block Storage (EBS)
More information about the cs615asa
mailing list