Coverage for libs/sdc_etl_libs/aws_helpers/aws_helpers.py : 15%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import json
2import logging
3import os
4import re
5import sys
6from datetime import datetime
8import boto3
9import botocore
10from boto3.dynamodb.conditions import Key
11from botocore.exceptions import ClientError
12from sdc_etl_libs.aws_helpers import aws_exceptions
13from sdc_etl_libs.sdc_credentials.sdc_endpoint_credentials import SDCEndpointCredentials
14from sdc_etl_libs.sdc_s3.SDCS3 import SDCS3
17class AWSHelpers:
19 @staticmethod
20 def get_account_id() -> str:
21 """
22 Grab the AWS account id. Can be used to create resource ARN strings
23 for use in other methods so that account id/ARN is not hard-coded.
24 :return: Account ID as string
25 """
27 account_id = boto3.client("sts").get_caller_identity()["Account"]
29 return str(account_id)
31 @staticmethod
32 def iterate_on_s3_response(response_: dict, bucket_name_: str, prefix_: str, files_: list, give_full_path_):
33 """
34 Iterate over an S3 List Objects result and adds object file/object
35 names to list.
36 :param response_: Response from List Objects func.
37 :param bucket_name_: Name of S3 bucket that was searched.
38 :param prefix_: Prefix used to find files.
39 :param files_: List append S3 URLs to.
40 :return: None
41 """
43 for item in response_["Contents"]:
44 if prefix_ in item["Key"]:
45 if give_full_path_:
46 files_.append("s3://" + bucket_name_ + "/" + item["Key"])
47 else:
48 files_.append(os.path.basename(item["Key"]))
50 @staticmethod
51 def get_file_list_s3(bucket_name_,
52 prefix_,
53 access_key_=None,
54 secret_key_=None,
55 region_='us-east-2',
56 file_prefix_: str = None,
57 file_suffix_: str = None,
58 file_regex_: str = None,
59 give_full_path_=False):
60 """
61 Creates a list of items in an S3 bucket.
62 :param bucket_name_: Name of S3 bucket to search
63 :param prefix_: Prefix used to find files.
64 :param access_key_: AWS Access Key for S3 bucket (if applicable)
65 :param secret_key_: AWS Secret Key for S3 bucket (if applicable)
66 :param region_: AWS region (Default = 'us-east-2')
67 :param file_prefix_: If used, function will return files that start
68 with this (case-sensitive). Can be used in tandem with file_suffix_
69 :param file_suffix_: If used, function will return files that end
70 with this (case-sensitive). Can be used in tandem with file_prefix_
71 :param file_regex_: If used, will return all files that match this
72 regex pattern. file_prefix_ & file_suffix_ will be ignored.
73 :param give_full_path_: If False, only file name will be returned. If
74 true, full path & file name will be returned.
75 :return: List of S3 file/object names as strings
76 """
78 client = boto3.client(
79 's3', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
81 response = client.list_objects_v2(Bucket=bucket_name_, Prefix=prefix_)
83 all_files = []
85 if "Contents" in response:
86 AWSHelpers.iterate_on_s3_response(response, bucket_name_, prefix_, all_files, give_full_path_)
87 while response["IsTruncated"]:
88 print(response["NextContinuationToken"])
89 response = client.list_objects_v2(
90 Bucket=bucket_name_, Prefix=prefix_, ContinuationToken=response["NextContinuationToken"])
91 AWSHelpers.iterate_on_s3_response(response, bucket_name_, prefix_, all_files, give_full_path_)
93 if file_regex_ or file_prefix_ or file_suffix_:
94 pattern = file_regex_ if file_regex_ else \
95 f"{file_prefix_ if file_prefix_ else ''}.*{file_suffix_ if file_suffix_ else ''}"
97 files = [x for x in all_files if re.search(pattern, x)]
99 else:
100 files = all_files
102 return files
104 @staticmethod
105 def query_dynamodb_etl_table_for_files(table_name_, job_name_, region_='us-east-2'):
106 """
107 Queries DynamoDB ETL table for list of file names for a job.
108 :param table_name_: String. The name of the DynamoDB table.
109 :param job_name_: String. THe name of the job. Used as partition key in table.
110 :param region_: String. AWS Region. Default = 'us-east-2'.
111 :return: Dict. Dictionary of file names ex {"s3://file":1}.
112 """
114 dynamodb = boto3.resource('dynamodb', region_name=region_)
115 table = dynamodb.Table(table_name_)
117 response = table.query(KeyConditionExpression=Key('job_name').eq(job_name_))
118 data = response['Items']
119 out = {}
121 while 'LastEvaluatedKey' in response:
122 try:
123 response = table.query(ExclusiveStartKey=response['LastEvaluatedKey'])
124 data.append(response['Items'])
125 except Exception as e:
126 logging.error(e)
127 logging.error(f"Failed to scan table {table_name_}")
129 for file in data:
130 out[file["file_name"]] = 1
132 return out
134 @staticmethod
135 def insert_processed_file_list_into_dynamo(table_name_, job_name_, file_list_, region_='us-east-2'):
136 """
137 Inserts file name sinto DynamoDB ETL table.
138 :param table_name_: String. The name of the DynamoDB table.
139 :param file_list_: List. List of file names.
140 :param job_name_: String. THe name of the job. Used as partition key in table.
141 :param region_: String. AWS Region. Default = 'us-east-2'.
142 :return: None.
143 """
145 insert_data = []
147 for filename in file_list_:
148 insert_data.append({"job_name": job_name_, "file_name": filename})
150 dynamodb = boto3.resource('dynamodb', region_name=region_)
151 table = dynamodb.Table(table_name_)
153 try:
154 with table.batch_writer() as batch:
155 for item in insert_data:
156 batch.put_item(Item=item)
157 except Exception as e:
158 logging.error(e)
159 logging.error(f"Failed to put {item} to table {table_name_}")
161 @staticmethod
162 def add_item_to_dynamodb_table(table_name_, item_, region_='us-east-2', access_key_=None, secret_key_=None):
163 """
164 Adds a single item to a DynamoDB table.
165 :param table_name_: Name of the DynamoDB Table.
166 :param item_: String. Item to add to table.
167 :param region_: String. AWS Region. Default = 'us-east-2'.
168 :param access_key_: AWS access key. Default = None.
169 :param secret_key_: AWS secret key. Default = NOne.
170 :return: Dict. Response from DynamoDB.
171 """
173 dynamodb = boto3.resource(
174 'dynamodb', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
175 table = dynamodb.Table(table_name_)
177 try:
178 response = table.put_item(Item=item_)
179 except ClientError as e:
180 logging.error(f"It's dead, Jimmy! {e}")
181 raise
182 except Exception as e:
183 logging.error(f"Something terrible happened! All we know is {e}")
184 raise
186 return response["ResponseMetadata"]["HTTPStatusCode"]
188 @staticmethod
189 def get_item_from_dynamodb_table(table_name_,
190 key_,
191 sort_key_="all",
192 select_="all",
193 consistent_read_=False,
194 limit_=500,
195 access_key_=None,
196 secret_key_=None,
197 region_='us-east-2'):
198 """
199 Get an item from a DynamoDB table.
200 :param table_name_: Name of the DynamoDB Table.
201 :param key_: String. Value to query table for.
202 :param sort_key_: String. Filtering methodology for the table query. Default = "all".
203 Available options:
204 ALL: Returns all items for a given partition key.
205 {VALUE}: Returns items with a sort key matching passed value.
206 FIRST: Returns item with first sort key (as sorted in the table)
207 LAST: Returns item with last sort key (as sorted in the table)
208 :param select_: String. Used to limit which item's attributes to return.
209 Available options:
210 ALL: returns all attributes
211 {VALUE}: Returns one or more attributions specified in the parameter (e.g. "field1" or "field1,field2")
212 COUNT: Returns number of matching items instead of matching items themselves.
213 :param consistent_read_: Boolean. Specifies whether the read should be performed with strong or eventual consistency.
214 Default = False.
215 Available options:
216 False: Eventually consistent.
217 True: Strongly consistent.
218 :param limit_: Int or 'ALL'. Specifies number of items to be evaluated(!) before returning the results.
219 Default = 500.
220 Available options:
221 ALL: Evaluates all records.
222 {VALUE}: Stops evaluating records at value.
223 :param region_: String. AWS Region. Default = 'us-east-2'.
224 :param access_key_: AWS access key. Default = None.
225 :param secret_key_: AWS secret key. Default = None.
226 :return: Dict. DynamoDB response with results of query.
227 """
229 dynamodb = boto3.resource(
230 'dynamodb', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
232 table = dynamodb.Table(table_name_)
233 key_schema = table.key_schema
234 for k in key_schema:
235 if k["KeyType"] == "HASH":
236 hash_value = k["AttributeName"]
237 elif k["KeyType"] == "RANGE":
238 range_value = k["AttributeName"]
240 condition = Key(hash_value).eq(key_)
241 scan_index_forward = True
243 if sort_key_.lower() == "all":
244 pass
245 elif sort_key_.lower() == "first":
246 limit_ = 1
247 elif sort_key_.lower() == "last":
248 limit_ = 1
249 scan_index_forward = False
250 elif (sort_key_.lower()).isnumeric():
251 condition = Key(hash_value).eq(key_) & Key(range_value).eq(int(sort_key_))
252 else:
253 condition = Key(hash_value).eq(key_) & Key(range_value).eq(sort_key_)
255 attributes_to_get = []
256 if select_.lower() == "all":
257 select = "ALL_ATTRIBUTES"
258 elif select_.lower() == "count":
259 select = "COUNT"
260 else:
261 select = "SPECIFIC_ATTRIBUTES"
262 attributes_to_get = select_.split(",")
264 try:
265 if select != "SPECIFIC_ATTRIBUTES":
266 response = table.query(
267 KeyConditionExpression=condition,
268 Limit=limit_,
269 ScanIndexForward=scan_index_forward,
270 Select=select,
271 ConsistentRead=consistent_read_)
272 else:
273 response = table.query(
274 KeyConditionExpression=condition,
275 Limit=limit_,
276 ScanIndexForward=scan_index_forward,
277 Select=select,
278 ProjectionExpression=select_,
279 ConsistentRead=consistent_read_)
281 except ClientError as e:
282 logging.error(e.response['Error']['Message'])
283 resp = {"statusCode": 400, "Description": e.response['Error']['Message']}
284 return resp
285 else:
286 resp = {}
287 resp["statusCode"] = response["ResponseMetadata"]["HTTPStatusCode"]
288 if select == "COUNT":
289 resp["Payload"] = response["Count"]
290 else:
291 resp["Payload"] = response['Items']
292 return resp
294 @staticmethod
295 def delete_item_from_dynamodb_table(table_name_,
296 key_,
297 sort_key_="all",
298 region_='us-east-2',
299 access_key_=None,
300 secret_key_=None):
301 """
302 Deletes item(s) from a DynamoDB table.
303 :param table_name_: Name of the DynamoDB Table.
304 :param key_: String. Value to delete from table.
305 :param sort_key_: String. Filtering methodology for the table delete. Default = "all".
306 Available options:
307 ALL: Returns all items for a given partition key.
308 {VALUE}: Returns items with a sort key matching passed value.
309 :param region_: String. AWS Region. Default = 'us-east-2'.
310 :param access_key_: AWS access key. Default = None.
311 :param secret_key_: AWS secret key. Default = None.
312 :return: Dict. DynamoDB response with results of operation.
313 """
315 dynamodb = boto3.resource(
316 'dynamodb', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
318 table = dynamodb.Table(table_name_)
319 key_schema = table.key_schema
320 for k in key_schema:
321 if k["KeyType"] == "HASH":
322 hash_value = k["AttributeName"]
323 elif k["KeyType"] == "RANGE":
324 range_value = k["AttributeName"]
326 key = {hash_value: key_}
328 if sort_key_.lower() == "all":
329 pass
330 elif (sort_key_.lower()).isnumeric():
331 key[range_value] = int(sort_key_)
332 else:
333 key[range_value] = sort_key_
335 try:
336 response = table.delete_item(Key=key)
338 except ClientError as e:
339 logging.error(e.response['Error']['Message'])
340 raise
341 else:
342 return response["ResponseMetadata"]["HTTPStatusCode"]
344 @staticmethod
345 def get_secrets(secret_id_, access_key_=None, secret_key_=None, region_='us-east-2', decode_=True):
346 """
347 Retrieves AWS Secrets data.
348 :param secret_id_: ID of secret
349 :param access_key_: AWS Access Key (Optional)
350 :param secret_key_: AWS Secret Key (Optional)
351 :param region_: AWS Region
352 :param decode_: If True, decode Base64 private keys in secrets_dict
353 :return: Dictionary containing results of secrets
354 """
356 try:
357 secrets = boto3.client(
358 'secretsmanager', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
360 response = secrets.get_secret_value(SecretId=secret_id_)
361 secret_dict = json.loads(response["SecretString"])
363 if decode_:
364 # Decode Base64 private keys in secrets_dict
365 secret_dict = SDCEndpointCredentials.get_decoded_credentials(secret_dict)
367 return secret_dict
369 except botocore.exceptions.ClientError:
370 ex_type, ex_value, ex_traceback = sys.exc_info()
371 if "ExpiredTokenException" in str(ex_value):
372 raise aws_exceptions.ExpiredTokenException(ex_value)
373 # can add more here if needed
374 else:
375 raise
377 except Exception as e:
378 raise
380 @staticmethod
381 def create_secrets(secret_name_,
382 secret_desc_=None,
383 secret_string_=None,
384 access_key_=None,
385 secret_key_=None,
386 region_='us-east-2'):
387 """
388 Creates an AWS Secret.
390 :param secret_name_: Name for the secret
391 :param secret_desc_: Description for the secret (Optional)
392 :param secret_string_: String in dictionary format containing the
393 secret key/value pairs
394 :param access_key_: AWS access key (Optional)
395 :param secret_key_: AWS secret key (Optional)
396 :param region_: AWS region (Default = 'us-east-2'
397 :return: Dictionary containing results of secrets creation
399 Example of secret_string_:
400 {
401 "username":"atticus",
402 "password":"abc123" # pragma: allowlist secret
403 }
404 """
406 try:
407 secrets = boto3.client(
408 'secretsmanager', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
410 response = secrets.create_secret(Name=secret_name_, Description=secret_desc_, SecretString=secret_string_)
412 return response
414 except Exception as e:
415 logging.error("Failed to create secret. {}".format(e))
417 @staticmethod
418 def sns_create_topic(topic_name_, access_key_=None, secret_key_=None, region_='us-east-2'):
419 """
420 Create an SNS topic where messages can be sent.
422 :param topic_name_: Desired name of SNS topic
423 :param access_key_: AWS access key (Optional)
424 :param secret_key_: AWS secret key (Optional)
425 :param region_: AWS region
426 :return: Dictionary of created SNS topic details
427 """
429 sns = boto3.client('sns', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
430 response = sns.create_topic(Name=topic_name_)
431 return response
433 @staticmethod
434 def sns_subscribe(topic_name_,
435 protocol_='email',
436 endpoint_='',
437 topic_arn_=None,
438 access_key_=None,
439 secret_key_=None,
440 region_='us-east-2'):
441 """
442 Subscribe to an SNS topic.
444 :param topic_name_: Topic name (ex. 'prod-glue-tnt-events')
445 :param protocol_: Method of delivery (Default = 'email')
446 :param endpoint_: Where to send to (Delivery default is 'email',
447 so email address here)
448 :param access_key_: AWS access key (Optional)
449 :param secret_key_: AWS secret key (Optional)
450 :param region_: AWS region (Default us-east-2)
451 :param topic_arn_: Full Topic ARN, if not supplying topic_name_
452 (ex. arn:aws:sns:us-east-2:{account_id}:prod-glue-tnt-alerts)
453 :return: Dictionary of subscribe results
454 """
455 sns = boto3.client('sns', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
457 if topic_arn_:
458 topic_arn = topic_arn_
459 elif topic_name_:
460 account_id = AWSHelpers.get_account_id()
461 topic_arn = 'arn:aws:sns:{}:{}:{}'.format(region_, account_id, topic_name_)
462 else:
463 raise Exception("Must provide a topic name or topic ARN.")
465 response = sns.subscribe(TopicArn=topic_arn, Protocol=protocol_, Endpoint=endpoint_)
467 return response
469 @staticmethod
470 def sns_publish(topic_name_,
471 message_=None,
472 subject_=None,
473 topic_arn_=None,
474 access_key_=None,
475 secret_key_=None,
476 region_='us-east-2'):
477 """
478 Simple method for sending a message via SNS.
479 Note: a topic needs to have already been set up to utilize
480 this function.
482 :param topic_name_: Topic name (ex. 'prod-glue-tnt-events')
483 :param subject_: Subject for e-mail. If None, will default
484 to 'AWS Notification for Data Engineering'.
485 :param message_: Message to send to topic. Will be converted to string.
486 :param access_key_: AWS access key (Optional)
487 :param secret_key_: AWS secret key (Optional)
488 :param region_: AWS region. 'us-east-2' default if None.
489 :param topic_arn_: Full Topic ARN, if not supplying topic_name_
490 (ex. arn:aws:sns:us-east-2:{account_id}:prod-glue-tnt-alerts)
491 :return: Logging to console of message sent.
492 """
494 sns = boto3.client('sns', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
496 if topic_arn_:
497 topic_arn = topic_arn_
498 elif topic_name_:
499 account_id = AWSHelpers.get_account_id()
500 topic_arn = 'arn:aws:sns:{}:{}:{}'.format(region_, account_id, topic_name_)
501 else:
502 raise Exception("Must provide a topic name or topic ARN.")
504 subject = subject_ if subject_ else "AWS Notification for " \
505 "Data Engineering"
507 response = sns.publish(TopicArn=topic_arn, Subject=str(subject), Message=str(message_))
509 print(response)
510 logging.info("Sent the following message to SNS topic:\n{}".format(message_))
512 @staticmethod
513 def cloudwatch_put_rule(event_name_,
514 event_pattern_,
515 event_desc_: str = '',
516 state_="ENABLED",
517 access_key_=None,
518 secret_key_=None,
519 region_='us-east-2'):
520 """
521 Creates a Cloudwatch Event rule.
522 :param event_name_: String. Name for Cloudwatch Event.
523 :param event_desc_:String. Description for Cloudwatch Event.
524 :param event_pattern_: Dict. Pattern for event (Represented as JSON). More info at:
525 https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/CloudWatchEventsandEventPatterns.html
526 :param state_: String. State of Event. Default = 'ENABLED'
527 Available options:
528 'ENABLED': Event is enabled.
529 'DISABLED': Event is disabled.
530 :param access_key_: String. AWS access key (Optional)
531 :param secret_key_: String. AWS secret key (Optional)
532 :param region_: String. AWS region. Default = 'us-east-2'.
533 :return: Dict. Response from CloudWatch.
534 """
536 events = boto3.client(
537 'events', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
539 response = events.put_rule(Name=event_name_, Description=event_desc_, EventPattern=event_pattern_, State=state_)
541 return response
543 @staticmethod
544 def cloudwatch_put_target(event_name_,
545 target_name_,
546 target_id_=None,
547 input_map_=None,
548 input_template_=None,
549 access_key_=None,
550 secret_key_=None,
551 region_='us-east-2'):
552 """
553 Assigns a target for a Cloudwatch Event rule.
554 Note: We currently use SNS as a target with e-mail subscription. This
555 function is setup for that purpose, but, can be modified later on for
556 other targets/subs.
558 :param event_name_: String. Name of Cloudwatch Event we want target assigned to
559 :param target_name_: String. Name of target (For SNS, topic name. Will be
560 converted to ARN).
561 :param target_id_: String. Name of ID for target (For SNS, leave as None.
562 Will be converted to topic name from target_name_ value).
563 :param input_map_: Dict. JSON string of key/value pairs
564 from event details (Optional)
565 :param input_template_: String. Message layout (Optional).
566 :param access_key_: String. AWS access key (Optional)
567 :param secret_key_: String. AWS secret key (Optional)
568 :param region_: String. AWS region. Default = 'us-east-2'.
569 :return: Dict. Response from CloudWatch.
570 """
572 events = boto3.client(
573 'events', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
575 account_id = AWSHelpers.get_account_id()
576 topic_arn = 'arn:aws:sns:{}:{}:{}'.format(region_, account_id, target_name_)
578 response = events.put_targets(
579 Rule=event_name_,
580 Targets=[{
581 'Arn': topic_arn,
582 'Id': target_name_,
583 'InputTransformer': {
584 'InputPathsMap': input_map_,
585 'InputTemplate': input_template_
586 }
587 }])
589 return response
591 @staticmethod
592 def cloud_watch_generate_query(id_, **params):
593 """
594 Generates a query to run against CLoudWatch.
595 :param id_: String. A short name used to tie this object to the results in the response.
596 :param **params: Dict. Various pieces of data used to generate the query.
597 'Service': String. Service name.
598 'MetricName': String. Name of the metric.
599 'Dimensions': Dict. The dimensions for the metric associated with the alarm.
600 'Period': Int. The granularity, in seconds, of the returned data points.
601 'MetricStat': Dict. The metric to be returned, along with statistics, period, and units.
602 'Label': String. A human-readable label for this metric or expression.
603 :return: String. Query.
604 """
606 service_ = params.get('Service')
608 if not service_:
609 logging.error(f'"Service" keyword was not provided')
610 raise ValueError(f'"Service" keyword was not provided')
612 metric_name_ = params.get('MetricName')
613 if not metric_name_:
614 logging.error(f'"MetricName" keyword was not provided')
615 raise ValueError(f'"MetricName" keyword was not provided')
617 dims_ = params.get('Dimensions')
618 dimensions_ = []
619 if isinstance(dims_, dict):
620 for dim in dims_:
621 dim_val = dict()
622 dim_val['Name'] = dim
623 dim_val['Value'] = dims_[dim]
624 dimensions_.append(dim_val)
625 else:
626 logging.error(f'"Dimensions" keyword must be a dictionary')
627 raise ValueError(f'"Dimensions" keyword must be a dictionary')
629 query_ = {
630 'Id': id_,
631 'MetricStat': {
632 'Metric': {
633 'Namespace': f'AWS/{service_}',
634 'MetricName': metric_name_,
635 'Dimensions': dimensions_
636 }
637 },
638 'ReturnData': True
639 }
641 period_ = params.get('Period')
642 if not period_:
643 logging.error(f'"Period" keyword was not provided')
644 raise ValueError(f'"Period" keyword was not provided')
645 elif period_:
646 query_['MetricStat']['Period'] = period_
648 stat_ = params.get('Stat')
649 if not period_:
650 logging.error(f'"Stat" keyword was not provided')
651 raise ValueError(f'"Stat" keyword was not provided')
652 elif stat_:
653 query_['MetricStat']['Stat'] = stat_
655 unit_ = params.get('Unit')
656 if unit_:
657 query_['MetricStat']['Unit'] = unit_
659 label_ = params.get('Label')
660 if label_:
661 query_['Label'] = label_
663 return query_
665 @staticmethod
666 def cloudwatch_get_metric_data(start_time_,
667 end_time_,
668 access_key_=None,
669 secret_key_=None,
670 region_='us-east-2',
671 **params):
672 """
673 Retrieves metric data from CloudWatch.
674 :param start_time_: Datetime. The time stamp indicating the earliest data to be returned.
675 :param end_time_: Datetime. The time stamp indicating the latest data to be returned.
676 :param access_key_: AWS access key (Optional).
677 :param secret_key_: AWS secret key (Optional).
678 :param region_: AWS region (Default = 'us-east-2'.
679 :param **params: Dict. Various pieces of information used to generate query for CloudWatch.
680 See cloud_watch_generate_query() for more information.
681 :return: Dict. Response from CloudWatch.
682 """
684 cw_client = boto3.client(
685 'cloudwatch', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
687 metric_data_queries_ = []
688 for key in params:
689 query_id = key
690 query_params = params[key]
691 query_ = AWSHelpers.cloud_watch_generate_query(query_id, **query_params)
692 metric_data_queries_.append(query_)
694 if isinstance(start_time_, str):
695 start_time_ = datetime.strptime(start_time_, "%Y-%m-%d %H:%M:%S")
696 elif isinstance(start_time_, datetime):
697 pass
698 else:
699 raise ValueError(f'Something not right with start_time_ data type')
701 if isinstance(end_time_, str):
702 end_time_ = datetime.strptime(end_time_, "%Y-%m-%d %H:%M:%S")
703 elif isinstance(end_time_, datetime):
704 pass
705 else:
706 raise ValueError(f'Something not right with end_time_ data type')
708 next_token_ = None
709 while True:
710 kwargs = {'MetricDataQueries': metric_data_queries_, 'StartTime': start_time_, 'EndTime': end_time_}
712 if next_token_:
713 kwargs['NextToken'] = next_token_
714 resp = cw_client.get_metric_data(**kwargs)
715 next_token_ = resp.get('NextToken')
716 if not next_token_:
717 break
719 return resp
721 @staticmethod
722 def create_ses_template(template_name_,
723 subject_,
724 body_=None,
725 overwrite_=False,
726 src_="string",
727 html_=True,
728 s3_params_=None,
729 filepath_=None,
730 access_key_=None,
731 secret_key_=None,
732 region_='us-east-2'):
733 """
734 Creates an SES template.
735 :param template_name_: String. Name of SES template (mandatory).
736 :param subject_: String. email subject portion of the template (mandatory).
737 :param body_: String. E-mail body portion of the template. Default = None.
738 Note. Required parameter when src_="string"
739 :param overwrite_: Boolean. Flag which indicates whether existing template with the same name should be
740 overwritten. Default = False.
741 :param src_: String. Indicates the source of template data.
742 Available options:
743 's3': Template data is provided in s3, S3_params_ must be provided.
744 'file': Template data is provided in local file, filepath_ must be provided.
745 'string': Template data is passed in string, body_ must be provided
746 :param html_: Boolean. Specifies whether the email body is html or text. Default = True.
747 :param s3_params_: Dict. Dictionary of s3 information/options. NOTE: Required parameter when src_="s3".
748 Default = None.
749 :param filepath_: String. File path to template file. Required parameter when src_="file". Default = None.
750 :param access_key_: String. AWS access key (optional). Default = None.
751 :param secret_key_: String. AWS secret key (optional). Default = None.
752 :param region_: String. AWS region. Default = 'us-east-2'.
753 :return: String. Status Code of template generation from SES.
754 """
756 ses_client = boto3.client(
757 'ses', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
759 if src_.lower() not in ["s3", "file", "string"]:
760 logging.error('Parameter "src_" must be one if these options: "s3", "file", "string"')
761 raise
763 if subject_ is None:
764 logging.error('Parameter subject_ must be provided.')
765 raise
767 if src_.lower() == "s3":
768 if s3_params_ is None:
769 raise Exception('Must supply S3 parameters as dict in s3_params_')
770 if not isinstance(s3_params_, dict):
771 raise Exception('Parameter "s3_params_" must be of type Dict')
772 if "bucket_name" not in s3_params_ or "filename" not in s3_params_:
773 raise Exception('Both "bucket_name" and "filename" must be provided when S3 selected for src_')
775 filename = s3_params_["filename"]
776 bucket_name = s3_params_["bucket_name"]
777 file = filename.split(".")[0]
778 prefix = "" if "prefix" not in s3_params_ else s3_params_["prefix"]
779 compression_type = None if "compression_type" not in s3_params_ else s3_params_["compression_type"]
780 decode = 'utf-8' if "decode" not in s3_params_ else s3_params_["decode"]
782 bucket = SDCS3()
783 bucket.connect()
784 body = bucket.get_file_as_file_object(
785 bucket_name_=bucket_name,
786 prefix_=prefix,
787 file_name_=file,
788 decode_=decode,
789 compression_type_=compression_type)
791 elif src_.lower() == "file":
792 if filepath_ is None:
793 raise Exception('Must supply filepath to local file with template in "filepath_" when '
794 '"file" selected for src_')
795 else:
796 with open(filepath_) as f:
797 body = f.read()
799 elif src_.lower() == "string":
800 if body_ is None:
801 raise Exception('Parameter body_ must be provided when "string" selected for src_')
802 body = body_
804 else:
805 raise Exception(f'{src_} is not a valid source for SES template.')
807 body_type = "HtmlPart" if html_ is True else "TextPart"
809 if overwrite_:
810 logging.info(f'Overwrite SES set to True. Deleting existing template if exists.')
811 AWSHelpers.delete_ses_template(template_name_)
813 try:
814 response = ses_client.create_template(Template={
815 'TemplateName': template_name_,
816 'SubjectPart': subject_,
817 body_type: body
818 })
819 except ClientError as e:
820 msg = e.response["Error"]["Code"]
821 logging.error(msg)
822 raise ValueError(msg)
824 return response["ResponseMetadata"]["HTTPStatusCode"]
826 @staticmethod
827 def delete_ses_template(template_name_, access_key_=None, secret_key_=None, region_='us-east-2'):
828 """
829 Deletes an SES template.
830 :param template_name_: String. Name of SES template.
831 :param access_key_: String. AWS access key (optional). Default = None.
832 :param secret_key_: String. AWS secret key (optional). Default = None.
833 :param region_: String. AWS region. Default = 'us-east-2'.
834 :return: None.
835 """
837 ses_client = boto3.client(
838 'ses', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
839 try:
840 ses_client.delete_template(TemplateName=template_name_)
841 logging.info(f'Deleted template "{template_name_}".')
843 except ClientError as e:
844 logging.error(e)
846 @staticmethod
847 def send_ses_email(recipients_: list,
848 subject_,
849 content_,
850 html_=True,
851 sender_="sdcde@smiledirectclub.com",
852 access_key_=None,
853 secret_key_=None,
854 region_='us-east-2',
855 charset_="utf-8"):
856 """
857 Sends an SES e-mail.
858 :param recipients_: List. Recipients to receive the same copy of the email.
859 :param subject_: String. Subject of the email.
860 :param content_: String. Body of the email.
861 :param html_: Boolean. Indicates whether the email body is HTML or text. Default = True.
862 :param sender_: String. "FROM" address on the email to send. This email has to be verified by AWS first.
863 Default = "sdcde@smiledirectclub.com".
864 :param access_key_: String. AWS access key.
865 :param secret_key_: String. AWS secret key.
866 :param region_: String. AWS region. Default = 'us-east-2'.
867 :param charset_: String. Character set for the body of the SES e-mail. Default = "utf-8".
868 :return: Status Code of e-mail send from SES.
869 """
871 if not isinstance(recipients_, list):
872 raise ValueError(f'Parameter "recipients_" must be a list')
873 subject = {
874 'Charset': charset_,
875 'Data': subject_,
876 }
878 body_type = "Html" if html_ else "Text"
879 body = {body_type: {'Charset': charset_, 'Data': content_}}
881 ses_client = boto3.client(
882 'ses', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
884 try:
885 response = ses_client.send_email(
886 Destination={
887 'BccAddresses': recipients_,
888 },
889 Message={
890 'Body': body,
891 'Subject': subject,
892 },
893 Source=sender_,
894 )
895 except ClientError as e:
896 msg = e.response['Error']['Message']
897 logging.error(msg)
898 raise BaseException(msg)
899 else:
900 msg = response['ResponseMetadata']['HTTPStatusCode']
901 logging.info(msg)
902 return msg
904 @staticmethod
905 def send_templated_ses_email(recipients_,
906 template_name_,
907 template_data_,
908 sender_="sdcde@smiledirectclub.com",
909 access_key_=None,
910 secret_key_=None,
911 region_='us-east-2'):
912 """
913 Sends a templated SES e-mail.
914 :param recipients_: List. List of e-mails to send to.
915 :param template_name_: String. Template to apply (has to be previously uploaded).
916 :param template_data_: Various. Dynamic data to fit into the template.
917 :param sender_: String. "FROM" address on the email to send. This email has to be verified by AWS first.
918 Default = "sdcde@smiledirectclub.com".
919 :param access_key_: String. AWS access key.
920 :param secret_key_: String. AWS secret key.
921 :param region_: String. AWS region. Default = 'us-east-2'.
922 :return: String. Status code from SES.
923 """
925 if not isinstance(recipients_, list):
926 raise ValueError(f'Parameter "recipients_" must be a list')
927 if not isinstance(template_data_, str):
928 raise ValueError(f'Parameter "template_data_" must be a str')
930 ses_client = boto3.client(
931 'ses', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_)
932 try:
933 response = ses_client.send_templated_email(
934 Source=sender_,
935 Destination={
936 'BccAddresses': recipients_,
937 },
938 Template=template_name_,
939 TemplateData=template_data_)
940 except ClientError as e:
941 msg = e.response['Error']['Message']
942 logging.error(msg)
943 raise BaseException(msg)
944 except Exception as e:
945 raise BaseException(f"Error: {e}")
946 else:
947 return response['ResponseMetadata']['HTTPStatusCode']
949 @staticmethod
950 def update_lambda_function_code(function_name_,
951 s3_bucket_name_,
952 s3_prefix_="",
953 s3_obj_name_=None,
954 publish_=False,
955 publish_description_="",
956 dry_run_=False,
957 access_key_=None,
958 secret_key_=None,
959 region_='us-east-2'):
960 """
961 Updates a current Lambda's function code.
962 :param function_name_: String. Name or ARN of Lambda function in AWS.
963 :param s3_bucket_name_:. String. Name of S3 bucket where zipped up Lambda deployment package exists.
964 :param s3_prefix_:. String. Prefix in bucket.
965 :param s3_obj_name_: String. Name of file in bucket.
966 :param publish_: Boolean. If True, publishes a new version of the code. Default = False.
967 NOTE: A new version will not be published if the function's configuration and code haven't changed
968 since the last version
969 :param publish_description_: String. Description for version, if publish_=True. Default = "".
970 :param dry_run_: Boolean. If True, validates request parameters and access permissions without modifying
971 the function code. Default = False.
972 :param access_key_: String. AWS access key. Default = None.
973 :param secret_key_: String. AWS secret key. Default = None.
974 :param region_: String. AWS region. Default = 'us-east-2'.
975 :return: Dict. Response from Lambda service.
976 """
978 client = boto3.client(
979 'lambda', aws_access_key_id=access_key_, aws_secret_access_key=secret_key_, region_name=region_)
981 response = client.update_function_code(
982 FunctionName=function_name_,
983 S3Bucket=s3_bucket_name_,
984 S3Key=os.path.join(s3_prefix_, s3_obj_name_),
985 DryRun=dry_run_)
987 if dry_run_:
988 logging.info(f"Dry Run results:\n{response}")
989 return response
991 if response['ResponseMetadata']['HTTPStatusCode'] == 200:
992 if publish_:
993 response = client.publish_version(
994 FunctionName=function_name_, CodeSha256=response['CodeSha256'], Description=publish_description_)
995 if response['ResponseMetadata']['HTTPStatusCode'] == 201:
996 logging.info(f"{function_name_} published. Version is now {response['Version']}.")
997 else:
998 raise Exception(f"Version was not successfully published. See response:\n{response}")
999 else:
1000 raise Exception(f"Lambdas code was not updated as expected. See response:\n{response}")
1001 return response