Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import json 

2import logging 

3import os 

4import re 

5import sys 

6from datetime import datetime 

7 

8import boto3 

9import botocore 

10from boto3.dynamodb.conditions import Key 

11from botocore.exceptions import ClientError 

12from sdc_etl_libs.aws_helpers import aws_exceptions 

13from sdc_etl_libs.sdc_credentials.sdc_endpoint_credentials import SDCEndpointCredentials 

14from sdc_etl_libs.sdc_s3.SDCS3 import SDCS3 

15 

16 

17class AWSHelpers: 

18 

19 @staticmethod 

20 def get_account_id() -> str: 

21 """ 

22 Grab the AWS account id. Can be used to create resource ARN strings 

23 for use in other methods so that account id/ARN is not hard-coded. 

24 :return: Account ID as string 

25 """ 

26 

27 account_id = boto3.client("sts").get_caller_identity()["Account"] 

28 

29 return str(account_id) 

30 

31 @staticmethod 

32 def iterate_on_s3_response(response_: dict, bucket_name_: str, prefix_: str, files_: list, give_full_path_): 

33 """ 

34 Iterate over an S3 List Objects result and adds object file/object 

35 names to list. 

36 :param response_: Response from List Objects func. 

37 :param bucket_name_: Name of S3 bucket that was searched. 

38 :param prefix_: Prefix used to find files. 

39 :param files_: List append S3 URLs to. 

40 :return: None 

41 """ 

42 

43 for item in response_["Contents"]: 

44 if prefix_ in item["Key"]: 

45 if give_full_path_: 

46 files_.append("s3://" + bucket_name_ + "/" + item["Key"]) 

47 else: 

48 files_.append(os.path.basename(item["Key"])) 

49 

50 @staticmethod 

51 def get_file_list_s3(bucket_name_, 

52 prefix_, 

53 access_key_=None, 

54 secret_key_=None, 

55 region_='us-east-2', 

56 file_prefix_: str = None, 

57 file_suffix_: str = None, 

58 file_regex_: str = None, 

59 give_full_path_=False): 

60 """ 

61 Creates a list of items in an S3 bucket. 

62 :param bucket_name_: Name of S3 bucket to search 

63 :param prefix_: Prefix used to find files. 

64 :param access_key_: AWS Access Key for S3 bucket (if applicable) 

65 :param secret_key_: AWS Secret Key for S3 bucket (if applicable) 

66 :param region_: AWS region (Default = 'us-east-2') 

67 :param file_prefix_: If used, function will return files that start 

68 with this (case-sensitive). Can be used in tandem with file_suffix_ 

69 :param file_suffix_: If used, function will return files that end 

70 with this (case-sensitive). Can be used in tandem with file_prefix_ 

71 :param file_regex_: If used, will return all files that match this 

72 regex pattern. file_prefix_ & file_suffix_ will be ignored. 

73 :param give_full_path_: If False, only file name will be returned. If 

74 true, full path & file name will be returned. 

75 :return: List of S3 file/object names as strings 

76 """ 

77 

78 client = boto3.client( 

79 's3', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

80 

81 response = client.list_objects_v2(Bucket=bucket_name_, Prefix=prefix_) 

82 

83 all_files = [] 

84 

85 if "Contents" in response: 

86 AWSHelpers.iterate_on_s3_response(response, bucket_name_, prefix_, all_files, give_full_path_) 

87 while response["IsTruncated"]: 

88 print(response["NextContinuationToken"]) 

89 response = client.list_objects_v2( 

90 Bucket=bucket_name_, Prefix=prefix_, ContinuationToken=response["NextContinuationToken"]) 

91 AWSHelpers.iterate_on_s3_response(response, bucket_name_, prefix_, all_files, give_full_path_) 

92 

93 if file_regex_ or file_prefix_ or file_suffix_: 

94 pattern = file_regex_ if file_regex_ else \ 

95 f"{file_prefix_ if file_prefix_ else ''}.*{file_suffix_ if file_suffix_ else ''}" 

96 

97 files = [x for x in all_files if re.search(pattern, x)] 

98 

99 else: 

100 files = all_files 

101 

102 return files 

103 

104 @staticmethod 

105 def query_dynamodb_etl_table_for_files(table_name_, job_name_, region_='us-east-2'): 

106 """ 

107 Queries DynamoDB ETL table for list of file names for a job. 

108 :param table_name_: String. The name of the DynamoDB table. 

109 :param job_name_: String. THe name of the job. Used as partition key in table. 

110 :param region_: String. AWS Region. Default = 'us-east-2'. 

111 :return: Dict. Dictionary of file names ex {"s3://file":1}. 

112 """ 

113 

114 dynamodb = boto3.resource('dynamodb', region_name=region_) 

115 table = dynamodb.Table(table_name_) 

116 

117 response = table.query(KeyConditionExpression=Key('job_name').eq(job_name_)) 

118 data = response['Items'] 

119 out = {} 

120 

121 while 'LastEvaluatedKey' in response: 

122 try: 

123 response = table.query(ExclusiveStartKey=response['LastEvaluatedKey']) 

124 data.append(response['Items']) 

125 except Exception as e: 

126 logging.error(e) 

127 logging.error(f"Failed to scan table {table_name_}") 

128 

129 for file in data: 

130 out[file["file_name"]] = 1 

131 

132 return out 

133 

134 @staticmethod 

135 def insert_processed_file_list_into_dynamo(table_name_, job_name_, file_list_, region_='us-east-2'): 

136 """ 

137 Inserts file name sinto DynamoDB ETL table. 

138 :param table_name_: String. The name of the DynamoDB table. 

139 :param file_list_: List. List of file names. 

140 :param job_name_: String. THe name of the job. Used as partition key in table. 

141 :param region_: String. AWS Region. Default = 'us-east-2'. 

142 :return: None. 

143 """ 

144 

145 insert_data = [] 

146 

147 for filename in file_list_: 

148 insert_data.append({"job_name": job_name_, "file_name": filename}) 

149 

150 dynamodb = boto3.resource('dynamodb', region_name=region_) 

151 table = dynamodb.Table(table_name_) 

152 

153 try: 

154 with table.batch_writer() as batch: 

155 for item in insert_data: 

156 batch.put_item(Item=item) 

157 except Exception as e: 

158 logging.error(e) 

159 logging.error(f"Failed to put {item} to table {table_name_}") 

160 

161 @staticmethod 

162 def add_item_to_dynamodb_table(table_name_, item_, region_='us-east-2', access_key_=None, secret_key_=None): 

163 """ 

164 Adds a single item to a DynamoDB table. 

165 :param table_name_: Name of the DynamoDB Table. 

166 :param item_: String. Item to add to table. 

167 :param region_: String. AWS Region. Default = 'us-east-2'. 

168 :param access_key_: AWS access key. Default = None. 

169 :param secret_key_: AWS secret key. Default = NOne. 

170 :return: Dict. Response from DynamoDB. 

171 """ 

172 

173 dynamodb = boto3.resource( 

174 'dynamodb', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

175 table = dynamodb.Table(table_name_) 

176 

177 try: 

178 response = table.put_item(Item=item_) 

179 except ClientError as e: 

180 logging.error(f"It's dead, Jimmy! {e}") 

181 raise 

182 except Exception as e: 

183 logging.error(f"Something terrible happened! All we know is {e}") 

184 raise 

185 

186 return response["ResponseMetadata"]["HTTPStatusCode"] 

187 

188 @staticmethod 

189 def get_item_from_dynamodb_table(table_name_, 

190 key_, 

191 sort_key_="all", 

192 select_="all", 

193 consistent_read_=False, 

194 limit_=500, 

195 access_key_=None, 

196 secret_key_=None, 

197 region_='us-east-2'): 

198 """ 

199 Get an item from a DynamoDB table. 

200 :param table_name_: Name of the DynamoDB Table. 

201 :param key_: String. Value to query table for. 

202 :param sort_key_: String. Filtering methodology for the table query. Default = "all". 

203 Available options: 

204 ALL: Returns all items for a given partition key. 

205 {VALUE}: Returns items with a sort key matching passed value. 

206 FIRST: Returns item with first sort key (as sorted in the table) 

207 LAST: Returns item with last sort key (as sorted in the table) 

208 :param select_: String. Used to limit which item's attributes to return. 

209 Available options: 

210 ALL: returns all attributes 

211 {VALUE}: Returns one or more attributions specified in the parameter (e.g. "field1" or "field1,field2") 

212 COUNT: Returns number of matching items instead of matching items themselves. 

213 :param consistent_read_: Boolean. Specifies whether the read should be performed with strong or eventual consistency. 

214 Default = False. 

215 Available options: 

216 False: Eventually consistent. 

217 True: Strongly consistent. 

218 :param limit_: Int or 'ALL'. Specifies number of items to be evaluated(!) before returning the results. 

219 Default = 500. 

220 Available options: 

221 ALL: Evaluates all records. 

222 {VALUE}: Stops evaluating records at value. 

223 :param region_: String. AWS Region. Default = 'us-east-2'. 

224 :param access_key_: AWS access key. Default = None. 

225 :param secret_key_: AWS secret key. Default = None. 

226 :return: Dict. DynamoDB response with results of query. 

227 """ 

228 

229 dynamodb = boto3.resource( 

230 'dynamodb', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

231 

232 table = dynamodb.Table(table_name_) 

233 key_schema = table.key_schema 

234 for k in key_schema: 

235 if k["KeyType"] == "HASH": 

236 hash_value = k["AttributeName"] 

237 elif k["KeyType"] == "RANGE": 

238 range_value = k["AttributeName"] 

239 

240 condition = Key(hash_value).eq(key_) 

241 scan_index_forward = True 

242 

243 if sort_key_.lower() == "all": 

244 pass 

245 elif sort_key_.lower() == "first": 

246 limit_ = 1 

247 elif sort_key_.lower() == "last": 

248 limit_ = 1 

249 scan_index_forward = False 

250 elif (sort_key_.lower()).isnumeric(): 

251 condition = Key(hash_value).eq(key_) & Key(range_value).eq(int(sort_key_)) 

252 else: 

253 condition = Key(hash_value).eq(key_) & Key(range_value).eq(sort_key_) 

254 

255 attributes_to_get = [] 

256 if select_.lower() == "all": 

257 select = "ALL_ATTRIBUTES" 

258 elif select_.lower() == "count": 

259 select = "COUNT" 

260 else: 

261 select = "SPECIFIC_ATTRIBUTES" 

262 attributes_to_get = select_.split(",") 

263 

264 try: 

265 if select != "SPECIFIC_ATTRIBUTES": 

266 response = table.query( 

267 KeyConditionExpression=condition, 

268 Limit=limit_, 

269 ScanIndexForward=scan_index_forward, 

270 Select=select, 

271 ConsistentRead=consistent_read_) 

272 else: 

273 response = table.query( 

274 KeyConditionExpression=condition, 

275 Limit=limit_, 

276 ScanIndexForward=scan_index_forward, 

277 Select=select, 

278 ProjectionExpression=select_, 

279 ConsistentRead=consistent_read_) 

280 

281 except ClientError as e: 

282 logging.error(e.response['Error']['Message']) 

283 resp = {"statusCode": 400, "Description": e.response['Error']['Message']} 

284 return resp 

285 else: 

286 resp = {} 

287 resp["statusCode"] = response["ResponseMetadata"]["HTTPStatusCode"] 

288 if select == "COUNT": 

289 resp["Payload"] = response["Count"] 

290 else: 

291 resp["Payload"] = response['Items'] 

292 return resp 

293 

294 @staticmethod 

295 def delete_item_from_dynamodb_table(table_name_, 

296 key_, 

297 sort_key_="all", 

298 region_='us-east-2', 

299 access_key_=None, 

300 secret_key_=None): 

301 """ 

302 Deletes item(s) from a DynamoDB table. 

303 :param table_name_: Name of the DynamoDB Table. 

304 :param key_: String. Value to delete from table. 

305 :param sort_key_: String. Filtering methodology for the table delete. Default = "all". 

306 Available options: 

307 ALL: Returns all items for a given partition key. 

308 {VALUE}: Returns items with a sort key matching passed value. 

309 :param region_: String. AWS Region. Default = 'us-east-2'. 

310 :param access_key_: AWS access key. Default = None. 

311 :param secret_key_: AWS secret key. Default = None. 

312 :return: Dict. DynamoDB response with results of operation. 

313 """ 

314 

315 dynamodb = boto3.resource( 

316 'dynamodb', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

317 

318 table = dynamodb.Table(table_name_) 

319 key_schema = table.key_schema 

320 for k in key_schema: 

321 if k["KeyType"] == "HASH": 

322 hash_value = k["AttributeName"] 

323 elif k["KeyType"] == "RANGE": 

324 range_value = k["AttributeName"] 

325 

326 key = {hash_value: key_} 

327 

328 if sort_key_.lower() == "all": 

329 pass 

330 elif (sort_key_.lower()).isnumeric(): 

331 key[range_value] = int(sort_key_) 

332 else: 

333 key[range_value] = sort_key_ 

334 

335 try: 

336 response = table.delete_item(Key=key) 

337 

338 except ClientError as e: 

339 logging.error(e.response['Error']['Message']) 

340 raise 

341 else: 

342 return response["ResponseMetadata"]["HTTPStatusCode"] 

343 

344 @staticmethod 

345 def get_secrets(secret_id_, access_key_=None, secret_key_=None, region_='us-east-2', decode_=True): 

346 """ 

347 Retrieves AWS Secrets data. 

348 :param secret_id_: ID of secret 

349 :param access_key_: AWS Access Key (Optional) 

350 :param secret_key_: AWS Secret Key (Optional) 

351 :param region_: AWS Region 

352 :param decode_: If True, decode Base64 private keys in secrets_dict 

353 :return: Dictionary containing results of secrets 

354 """ 

355 

356 try: 

357 secrets = boto3.client( 

358 'secretsmanager', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

359 

360 response = secrets.get_secret_value(SecretId=secret_id_) 

361 secret_dict = json.loads(response["SecretString"]) 

362 

363 if decode_: 

364 # Decode Base64 private keys in secrets_dict 

365 secret_dict = SDCEndpointCredentials.get_decoded_credentials(secret_dict) 

366 

367 return secret_dict 

368 

369 except botocore.exceptions.ClientError: 

370 ex_type, ex_value, ex_traceback = sys.exc_info() 

371 if "ExpiredTokenException" in str(ex_value): 

372 raise aws_exceptions.ExpiredTokenException(ex_value) 

373 # can add more here if needed 

374 else: 

375 raise 

376 

377 except Exception as e: 

378 raise 

379 

380 @staticmethod 

381 def create_secrets(secret_name_, 

382 secret_desc_=None, 

383 secret_string_=None, 

384 access_key_=None, 

385 secret_key_=None, 

386 region_='us-east-2'): 

387 """ 

388 Creates an AWS Secret. 

389 

390 :param secret_name_: Name for the secret 

391 :param secret_desc_: Description for the secret (Optional) 

392 :param secret_string_: String in dictionary format containing the 

393 secret key/value pairs 

394 :param access_key_: AWS access key (Optional) 

395 :param secret_key_: AWS secret key (Optional) 

396 :param region_: AWS region (Default = 'us-east-2' 

397 :return: Dictionary containing results of secrets creation 

398 

399 Example of secret_string_: 

400 { 

401 "username":"atticus", 

402 "password":"abc123" # pragma: allowlist secret 

403 } 

404 """ 

405 

406 try: 

407 secrets = boto3.client( 

408 'secretsmanager', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

409 

410 response = secrets.create_secret(Name=secret_name_, Description=secret_desc_, SecretString=secret_string_) 

411 

412 return response 

413 

414 except Exception as e: 

415 logging.error("Failed to create secret. {}".format(e)) 

416 

417 @staticmethod 

418 def sns_create_topic(topic_name_, access_key_=None, secret_key_=None, region_='us-east-2'): 

419 """ 

420 Create an SNS topic where messages can be sent. 

421 

422 :param topic_name_: Desired name of SNS topic 

423 :param access_key_: AWS access key (Optional) 

424 :param secret_key_: AWS secret key (Optional) 

425 :param region_: AWS region 

426 :return: Dictionary of created SNS topic details 

427 """ 

428 

429 sns = boto3.client('sns', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

430 response = sns.create_topic(Name=topic_name_) 

431 return response 

432 

433 @staticmethod 

434 def sns_subscribe(topic_name_, 

435 protocol_='email', 

436 endpoint_='', 

437 topic_arn_=None, 

438 access_key_=None, 

439 secret_key_=None, 

440 region_='us-east-2'): 

441 """ 

442 Subscribe to an SNS topic. 

443 

444 :param topic_name_: Topic name (ex. 'prod-glue-tnt-events') 

445 :param protocol_: Method of delivery (Default = 'email') 

446 :param endpoint_: Where to send to (Delivery default is 'email', 

447 so email address here) 

448 :param access_key_: AWS access key (Optional) 

449 :param secret_key_: AWS secret key (Optional) 

450 :param region_: AWS region (Default us-east-2) 

451 :param topic_arn_: Full Topic ARN, if not supplying topic_name_ 

452 (ex. arn:aws:sns:us-east-2:{account_id}:prod-glue-tnt-alerts) 

453 :return: Dictionary of subscribe results 

454 """ 

455 sns = boto3.client('sns', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

456 

457 if topic_arn_: 

458 topic_arn = topic_arn_ 

459 elif topic_name_: 

460 account_id = AWSHelpers.get_account_id() 

461 topic_arn = 'arn:aws:sns:{}:{}:{}'.format(region_, account_id, topic_name_) 

462 else: 

463 raise Exception("Must provide a topic name or topic ARN.") 

464 

465 response = sns.subscribe(TopicArn=topic_arn, Protocol=protocol_, Endpoint=endpoint_) 

466 

467 return response 

468 

469 @staticmethod 

470 def sns_publish(topic_name_, 

471 message_=None, 

472 subject_=None, 

473 topic_arn_=None, 

474 access_key_=None, 

475 secret_key_=None, 

476 region_='us-east-2'): 

477 """ 

478 Simple method for sending a message via SNS. 

479 Note: a topic needs to have already been set up to utilize 

480 this function. 

481 

482 :param topic_name_: Topic name (ex. 'prod-glue-tnt-events') 

483 :param subject_: Subject for e-mail. If None, will default 

484 to 'AWS Notification for Data Engineering'. 

485 :param message_: Message to send to topic. Will be converted to string. 

486 :param access_key_: AWS access key (Optional) 

487 :param secret_key_: AWS secret key (Optional) 

488 :param region_: AWS region. 'us-east-2' default if None. 

489 :param topic_arn_: Full Topic ARN, if not supplying topic_name_ 

490 (ex. arn:aws:sns:us-east-2:{account_id}:prod-glue-tnt-alerts) 

491 :return: Logging to console of message sent. 

492 """ 

493 

494 sns = boto3.client('sns', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

495 

496 if topic_arn_: 

497 topic_arn = topic_arn_ 

498 elif topic_name_: 

499 account_id = AWSHelpers.get_account_id() 

500 topic_arn = 'arn:aws:sns:{}:{}:{}'.format(region_, account_id, topic_name_) 

501 else: 

502 raise Exception("Must provide a topic name or topic ARN.") 

503 

504 subject = subject_ if subject_ else "AWS Notification for " \ 

505 "Data Engineering" 

506 

507 response = sns.publish(TopicArn=topic_arn, Subject=str(subject), Message=str(message_)) 

508 

509 print(response) 

510 logging.info("Sent the following message to SNS topic:\n{}".format(message_)) 

511 

512 @staticmethod 

513 def cloudwatch_put_rule(event_name_, 

514 event_pattern_, 

515 event_desc_: str = '', 

516 state_="ENABLED", 

517 access_key_=None, 

518 secret_key_=None, 

519 region_='us-east-2'): 

520 """ 

521 Creates a Cloudwatch Event rule. 

522 :param event_name_: String. Name for Cloudwatch Event. 

523 :param event_desc_:String. Description for Cloudwatch Event. 

524 :param event_pattern_: Dict. Pattern for event (Represented as JSON). More info at: 

525 https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/CloudWatchEventsandEventPatterns.html 

526 :param state_: String. State of Event. Default = 'ENABLED' 

527 Available options: 

528 'ENABLED': Event is enabled. 

529 'DISABLED': Event is disabled. 

530 :param access_key_: String. AWS access key (Optional) 

531 :param secret_key_: String. AWS secret key (Optional) 

532 :param region_: String. AWS region. Default = 'us-east-2'. 

533 :return: Dict. Response from CloudWatch. 

534 """ 

535 

536 events = boto3.client( 

537 'events', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

538 

539 response = events.put_rule(Name=event_name_, Description=event_desc_, EventPattern=event_pattern_, State=state_) 

540 

541 return response 

542 

543 @staticmethod 

544 def cloudwatch_put_target(event_name_, 

545 target_name_, 

546 target_id_=None, 

547 input_map_=None, 

548 input_template_=None, 

549 access_key_=None, 

550 secret_key_=None, 

551 region_='us-east-2'): 

552 """ 

553 Assigns a target for a Cloudwatch Event rule. 

554 Note: We currently use SNS as a target with e-mail subscription. This 

555 function is setup for that purpose, but, can be modified later on for 

556 other targets/subs. 

557 

558 :param event_name_: String. Name of Cloudwatch Event we want target assigned to 

559 :param target_name_: String. Name of target (For SNS, topic name. Will be 

560 converted to ARN). 

561 :param target_id_: String. Name of ID for target (For SNS, leave as None. 

562 Will be converted to topic name from target_name_ value). 

563 :param input_map_: Dict. JSON string of key/value pairs 

564 from event details (Optional) 

565 :param input_template_: String. Message layout (Optional). 

566 :param access_key_: String. AWS access key (Optional) 

567 :param secret_key_: String. AWS secret key (Optional) 

568 :param region_: String. AWS region. Default = 'us-east-2'. 

569 :return: Dict. Response from CloudWatch. 

570 """ 

571 

572 events = boto3.client( 

573 'events', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

574 

575 account_id = AWSHelpers.get_account_id() 

576 topic_arn = 'arn:aws:sns:{}:{}:{}'.format(region_, account_id, target_name_) 

577 

578 response = events.put_targets( 

579 Rule=event_name_, 

580 Targets=[{ 

581 'Arn': topic_arn, 

582 'Id': target_name_, 

583 'InputTransformer': { 

584 'InputPathsMap': input_map_, 

585 'InputTemplate': input_template_ 

586 } 

587 }]) 

588 

589 return response 

590 

591 @staticmethod 

592 def cloud_watch_generate_query(id_, **params): 

593 """ 

594 Generates a query to run against CLoudWatch. 

595 :param id_: String. A short name used to tie this object to the results in the response. 

596 :param **params: Dict. Various pieces of data used to generate the query. 

597 'Service': String. Service name. 

598 'MetricName': String. Name of the metric. 

599 'Dimensions': Dict. The dimensions for the metric associated with the alarm. 

600 'Period': Int. The granularity, in seconds, of the returned data points. 

601 'MetricStat': Dict. The metric to be returned, along with statistics, period, and units. 

602 'Label': String. A human-readable label for this metric or expression. 

603 :return: String. Query. 

604 """ 

605 

606 service_ = params.get('Service') 

607 

608 if not service_: 

609 logging.error(f'"Service" keyword was not provided') 

610 raise ValueError(f'"Service" keyword was not provided') 

611 

612 metric_name_ = params.get('MetricName') 

613 if not metric_name_: 

614 logging.error(f'"MetricName" keyword was not provided') 

615 raise ValueError(f'"MetricName" keyword was not provided') 

616 

617 dims_ = params.get('Dimensions') 

618 dimensions_ = [] 

619 if isinstance(dims_, dict): 

620 for dim in dims_: 

621 dim_val = dict() 

622 dim_val['Name'] = dim 

623 dim_val['Value'] = dims_[dim] 

624 dimensions_.append(dim_val) 

625 else: 

626 logging.error(f'"Dimensions" keyword must be a dictionary') 

627 raise ValueError(f'"Dimensions" keyword must be a dictionary') 

628 

629 query_ = { 

630 'Id': id_, 

631 'MetricStat': { 

632 'Metric': { 

633 'Namespace': f'AWS/{service_}', 

634 'MetricName': metric_name_, 

635 'Dimensions': dimensions_ 

636 } 

637 }, 

638 'ReturnData': True 

639 } 

640 

641 period_ = params.get('Period') 

642 if not period_: 

643 logging.error(f'"Period" keyword was not provided') 

644 raise ValueError(f'"Period" keyword was not provided') 

645 elif period_: 

646 query_['MetricStat']['Period'] = period_ 

647 

648 stat_ = params.get('Stat') 

649 if not period_: 

650 logging.error(f'"Stat" keyword was not provided') 

651 raise ValueError(f'"Stat" keyword was not provided') 

652 elif stat_: 

653 query_['MetricStat']['Stat'] = stat_ 

654 

655 unit_ = params.get('Unit') 

656 if unit_: 

657 query_['MetricStat']['Unit'] = unit_ 

658 

659 label_ = params.get('Label') 

660 if label_: 

661 query_['Label'] = label_ 

662 

663 return query_ 

664 

665 @staticmethod 

666 def cloudwatch_get_metric_data(start_time_, 

667 end_time_, 

668 access_key_=None, 

669 secret_key_=None, 

670 region_='us-east-2', 

671 **params): 

672 """ 

673 Retrieves metric data from CloudWatch. 

674 :param start_time_: Datetime. The time stamp indicating the earliest data to be returned. 

675 :param end_time_: Datetime. The time stamp indicating the latest data to be returned. 

676 :param access_key_: AWS access key (Optional). 

677 :param secret_key_: AWS secret key (Optional). 

678 :param region_: AWS region (Default = 'us-east-2'. 

679 :param **params: Dict. Various pieces of information used to generate query for CloudWatch. 

680 See cloud_watch_generate_query() for more information. 

681 :return: Dict. Response from CloudWatch. 

682 """ 

683 

684 cw_client = boto3.client( 

685 'cloudwatch', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

686 

687 metric_data_queries_ = [] 

688 for key in params: 

689 query_id = key 

690 query_params = params[key] 

691 query_ = AWSHelpers.cloud_watch_generate_query(query_id, **query_params) 

692 metric_data_queries_.append(query_) 

693 

694 if isinstance(start_time_, str): 

695 start_time_ = datetime.strptime(start_time_, "%Y-%m-%d %H:%M:%S") 

696 elif isinstance(start_time_, datetime): 

697 pass 

698 else: 

699 raise ValueError(f'Something not right with start_time_ data type') 

700 

701 if isinstance(end_time_, str): 

702 end_time_ = datetime.strptime(end_time_, "%Y-%m-%d %H:%M:%S") 

703 elif isinstance(end_time_, datetime): 

704 pass 

705 else: 

706 raise ValueError(f'Something not right with end_time_ data type') 

707 

708 next_token_ = None 

709 while True: 

710 kwargs = {'MetricDataQueries': metric_data_queries_, 'StartTime': start_time_, 'EndTime': end_time_} 

711 

712 if next_token_: 

713 kwargs['NextToken'] = next_token_ 

714 resp = cw_client.get_metric_data(**kwargs) 

715 next_token_ = resp.get('NextToken') 

716 if not next_token_: 

717 break 

718 

719 return resp 

720 

721 @staticmethod 

722 def create_ses_template(template_name_, 

723 subject_, 

724 body_=None, 

725 overwrite_=False, 

726 src_="string", 

727 html_=True, 

728 s3_params_=None, 

729 filepath_=None, 

730 access_key_=None, 

731 secret_key_=None, 

732 region_='us-east-2'): 

733 """ 

734 Creates an SES template. 

735 :param template_name_: String. Name of SES template (mandatory). 

736 :param subject_: String. email subject portion of the template (mandatory). 

737 :param body_: String. E-mail body portion of the template. Default = None. 

738 Note. Required parameter when src_="string" 

739 :param overwrite_: Boolean. Flag which indicates whether existing template with the same name should be 

740 overwritten. Default = False. 

741 :param src_: String. Indicates the source of template data. 

742 Available options: 

743 's3': Template data is provided in s3, S3_params_ must be provided. 

744 'file': Template data is provided in local file, filepath_ must be provided. 

745 'string': Template data is passed in string, body_ must be provided 

746 :param html_: Boolean. Specifies whether the email body is html or text. Default = True. 

747 :param s3_params_: Dict. Dictionary of s3 information/options. NOTE: Required parameter when src_="s3". 

748 Default = None. 

749 :param filepath_: String. File path to template file. Required parameter when src_="file". Default = None. 

750 :param access_key_: String. AWS access key (optional). Default = None. 

751 :param secret_key_: String. AWS secret key (optional). Default = None. 

752 :param region_: String. AWS region. Default = 'us-east-2'. 

753 :return: String. Status Code of template generation from SES. 

754 """ 

755 

756 ses_client = boto3.client( 

757 'ses', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

758 

759 if src_.lower() not in ["s3", "file", "string"]: 

760 logging.error('Parameter "src_" must be one if these options: "s3", "file", "string"') 

761 raise 

762 

763 if subject_ is None: 

764 logging.error('Parameter subject_ must be provided.') 

765 raise 

766 

767 if src_.lower() == "s3": 

768 if s3_params_ is None: 

769 raise Exception('Must supply S3 parameters as dict in s3_params_') 

770 if not isinstance(s3_params_, dict): 

771 raise Exception('Parameter "s3_params_" must be of type Dict') 

772 if "bucket_name" not in s3_params_ or "filename" not in s3_params_: 

773 raise Exception('Both "bucket_name" and "filename" must be provided when S3 selected for src_') 

774 

775 filename = s3_params_["filename"] 

776 bucket_name = s3_params_["bucket_name"] 

777 file = filename.split(".")[0] 

778 prefix = "" if "prefix" not in s3_params_ else s3_params_["prefix"] 

779 compression_type = None if "compression_type" not in s3_params_ else s3_params_["compression_type"] 

780 decode = 'utf-8' if "decode" not in s3_params_ else s3_params_["decode"] 

781 

782 bucket = SDCS3() 

783 bucket.connect() 

784 body = bucket.get_file_as_file_object( 

785 bucket_name_=bucket_name, 

786 prefix_=prefix, 

787 file_name_=file, 

788 decode_=decode, 

789 compression_type_=compression_type) 

790 

791 elif src_.lower() == "file": 

792 if filepath_ is None: 

793 raise Exception('Must supply filepath to local file with template in "filepath_" when ' 

794 '"file" selected for src_') 

795 else: 

796 with open(filepath_) as f: 

797 body = f.read() 

798 

799 elif src_.lower() == "string": 

800 if body_ is None: 

801 raise Exception('Parameter body_ must be provided when "string" selected for src_') 

802 body = body_ 

803 

804 else: 

805 raise Exception(f'{src_} is not a valid source for SES template.') 

806 

807 body_type = "HtmlPart" if html_ is True else "TextPart" 

808 

809 if overwrite_: 

810 logging.info(f'Overwrite SES set to True. Deleting existing template if exists.') 

811 AWSHelpers.delete_ses_template(template_name_) 

812 

813 try: 

814 response = ses_client.create_template(Template={ 

815 'TemplateName': template_name_, 

816 'SubjectPart': subject_, 

817 body_type: body 

818 }) 

819 except ClientError as e: 

820 msg = e.response["Error"]["Code"] 

821 logging.error(msg) 

822 raise ValueError(msg) 

823 

824 return response["ResponseMetadata"]["HTTPStatusCode"] 

825 

826 @staticmethod 

827 def delete_ses_template(template_name_, access_key_=None, secret_key_=None, region_='us-east-2'): 

828 """ 

829 Deletes an SES template. 

830 :param template_name_: String. Name of SES template. 

831 :param access_key_: String. AWS access key (optional). Default = None. 

832 :param secret_key_: String. AWS secret key (optional). Default = None. 

833 :param region_: String. AWS region. Default = 'us-east-2'. 

834 :return: None. 

835 """ 

836 

837 ses_client = boto3.client( 

838 'ses', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

839 try: 

840 ses_client.delete_template(TemplateName=template_name_) 

841 logging.info(f'Deleted template "{template_name_}".') 

842 

843 except ClientError as e: 

844 logging.error(e) 

845 

846 @staticmethod 

847 def send_ses_email(recipients_: list, 

848 subject_, 

849 content_, 

850 html_=True, 

851 sender_="sdcde@smiledirectclub.com", 

852 access_key_=None, 

853 secret_key_=None, 

854 region_='us-east-2', 

855 charset_="utf-8"): 

856 """ 

857 Sends an SES e-mail. 

858 :param recipients_: List. Recipients to receive the same copy of the email. 

859 :param subject_: String. Subject of the email. 

860 :param content_: String. Body of the email. 

861 :param html_: Boolean. Indicates whether the email body is HTML or text. Default = True. 

862 :param sender_: String. "FROM" address on the email to send. This email has to be verified by AWS first. 

863 Default = "sdcde@smiledirectclub.com". 

864 :param access_key_: String. AWS access key. 

865 :param secret_key_: String. AWS secret key. 

866 :param region_: String. AWS region. Default = 'us-east-2'. 

867 :param charset_: String. Character set for the body of the SES e-mail. Default = "utf-8". 

868 :return: Status Code of e-mail send from SES. 

869 """ 

870 

871 if not isinstance(recipients_, list): 

872 raise ValueError(f'Parameter "recipients_" must be a list') 

873 subject = { 

874 'Charset': charset_, 

875 'Data': subject_, 

876 } 

877 

878 body_type = "Html" if html_ else "Text" 

879 body = {body_type: {'Charset': charset_, 'Data': content_}} 

880 

881 ses_client = boto3.client( 

882 'ses', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

883 

884 try: 

885 response = ses_client.send_email( 

886 Destination={ 

887 'BccAddresses': recipients_, 

888 }, 

889 Message={ 

890 'Body': body, 

891 'Subject': subject, 

892 }, 

893 Source=sender_, 

894 ) 

895 except ClientError as e: 

896 msg = e.response['Error']['Message'] 

897 logging.error(msg) 

898 raise BaseException(msg) 

899 else: 

900 msg = response['ResponseMetadata']['HTTPStatusCode'] 

901 logging.info(msg) 

902 return msg 

903 

904 @staticmethod 

905 def send_templated_ses_email(recipients_, 

906 template_name_, 

907 template_data_, 

908 sender_="sdcde@smiledirectclub.com", 

909 access_key_=None, 

910 secret_key_=None, 

911 region_='us-east-2'): 

912 """ 

913 Sends a templated SES e-mail. 

914 :param recipients_: List. List of e-mails to send to. 

915 :param template_name_: String. Template to apply (has to be previously uploaded). 

916 :param template_data_: Various. Dynamic data to fit into the template. 

917 :param sender_: String. "FROM" address on the email to send. This email has to be verified by AWS first. 

918 Default = "sdcde@smiledirectclub.com". 

919 :param access_key_: String. AWS access key. 

920 :param secret_key_: String. AWS secret key. 

921 :param region_: String. AWS region. Default = 'us-east-2'. 

922 :return: String. Status code from SES. 

923 """ 

924 

925 if not isinstance(recipients_, list): 

926 raise ValueError(f'Parameter "recipients_" must be a list') 

927 if not isinstance(template_data_, str): 

928 raise ValueError(f'Parameter "template_data_" must be a str') 

929 

930 ses_client = boto3.client( 

931 'ses', region_name=region_, aws_access_key_id=access_key_, aws_secret_access_key=secret_key_) 

932 try: 

933 response = ses_client.send_templated_email( 

934 Source=sender_, 

935 Destination={ 

936 'BccAddresses': recipients_, 

937 }, 

938 Template=template_name_, 

939 TemplateData=template_data_) 

940 except ClientError as e: 

941 msg = e.response['Error']['Message'] 

942 logging.error(msg) 

943 raise BaseException(msg) 

944 except Exception as e: 

945 raise BaseException(f"Error: {e}") 

946 else: 

947 return response['ResponseMetadata']['HTTPStatusCode'] 

948 

949 @staticmethod 

950 def update_lambda_function_code(function_name_, 

951 s3_bucket_name_, 

952 s3_prefix_="", 

953 s3_obj_name_=None, 

954 publish_=False, 

955 publish_description_="", 

956 dry_run_=False, 

957 access_key_=None, 

958 secret_key_=None, 

959 region_='us-east-2'): 

960 """ 

961 Updates a current Lambda's function code. 

962 :param function_name_: String. Name or ARN of Lambda function in AWS. 

963 :param s3_bucket_name_:. String. Name of S3 bucket where zipped up Lambda deployment package exists. 

964 :param s3_prefix_:. String. Prefix in bucket. 

965 :param s3_obj_name_: String. Name of file in bucket. 

966 :param publish_: Boolean. If True, publishes a new version of the code. Default = False. 

967 NOTE: A new version will not be published if the function's configuration and code haven't changed 

968 since the last version 

969 :param publish_description_: String. Description for version, if publish_=True. Default = "". 

970 :param dry_run_: Boolean. If True, validates request parameters and access permissions without modifying 

971 the function code. Default = False. 

972 :param access_key_: String. AWS access key. Default = None. 

973 :param secret_key_: String. AWS secret key. Default = None. 

974 :param region_: String. AWS region. Default = 'us-east-2'. 

975 :return: Dict. Response from Lambda service. 

976 """ 

977 

978 client = boto3.client( 

979 'lambda', aws_access_key_id=access_key_, aws_secret_access_key=secret_key_, region_name=region_) 

980 

981 response = client.update_function_code( 

982 FunctionName=function_name_, 

983 S3Bucket=s3_bucket_name_, 

984 S3Key=os.path.join(s3_prefix_, s3_obj_name_), 

985 DryRun=dry_run_) 

986 

987 if dry_run_: 

988 logging.info(f"Dry Run results:\n{response}") 

989 return response 

990 

991 if response['ResponseMetadata']['HTTPStatusCode'] == 200: 

992 if publish_: 

993 response = client.publish_version( 

994 FunctionName=function_name_, CodeSha256=response['CodeSha256'], Description=publish_description_) 

995 if response['ResponseMetadata']['HTTPStatusCode'] == 201: 

996 logging.info(f"{function_name_} published. Version is now {response['Version']}.") 

997 else: 

998 raise Exception(f"Version was not successfully published. See response:\n{response}") 

999 else: 

1000 raise Exception(f"Lambdas code was not updated as expected. See response:\n{response}") 

1001 return response