Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" Class to fetch data from the Impact API """ 

2# pylint: disable=E0401, R0902, R0913, R0903 

3import json 

4import logging 

5 

6import requests 

7from jinja2 import Environment, FileSystemLoader, select_autoescape 

8from sdc_etl_libs.api_helpers.API import API 

9from sdc_etl_libs.sdc_file_helpers.SDCFileHelpers import SDCFileHelpers 

10 

11logging.info("EXECUTING: %s", __name__) 

12 

13 

14class Impact(API): 

15 """Connector/Wrapper class for fetching Impact reports. 

16 Ref.: https://developer.impact.com/default/documentation/Adv-v8 

17 """ 

18 

19 def __init__(self, schema_: dict = None, endpoint_schema_: dict = None, **kwargs): 

20 """Initiate API wrapper instance 

21 :param schema_: The full dictionary of the schema used 

22 :type schema_: dict 

23 :param endpoint_schema_: data_source_ part of the schema 

24 :type endpoint_schema_: dict 

25 :param kwargs: collection of named args required for this particular class 

26 :type kwargs: dict 

27 """ 

28 

29 meta_name = schema_['name'] 

30 meta_path = SDCFileHelpers.get_file_path("metadata", 'Vendors/Impact/{}.json'.format(meta_name)) 

31 meta = json.loads(open(meta_path).read()) 

32 

33 self.base_url = endpoint_schema_["info"]["access"]['base_url'] 

34 self.endpoint_name = endpoint_schema_["info"]["access"]['endpoint_name'] 

35 self.day = kwargs['day_'] 

36 self.query_params = meta['query_params_'] 

37 self.data_map = meta['data_map_'] 

38 self.add_date = meta.get('add_date_') 

39 

40 self.credentials = self.get_credentials(source_="aws_secrets", aws_secret_id_="Impact/API") # nosec 

41 

42 self.account_id = self.credentials['account_id'] \ 

43 if not kwargs.get('account_id_') else kwargs.get('account_id_') 

44 self.auth_token = self.credentials['auth_token'] \ 

45 if not kwargs.get('auth_token_') else kwargs.get('auth_token_') 

46 

47 def _fetch_report(self, url): 

48 """Fetches data from the impact API 

49 :param url: Well formed query containing all data to query the API 

50 :type url: str 

51 :return: List of events fetched by all the instances of the recursion, until no more links are returned. 

52 :rtype: list 

53 """ 

54 

55 response = requests.request("GET", url, auth=(self.account_id, self.auth_token)) 

56 return response.json()["Records"] 

57 

58 def _request_report(self): 

59 """ Fetches the raw data from the API and reformats it 

60 :return: List of elements fetched from API reformatted to comply with the required schema. 

61 :rtype: list(dict) 

62 """ 

63 template_loader = FileSystemLoader(searchpath=SDCFileHelpers.get_file_path(type_='template', path_="Impact")) 

64 template_env = Environment(loader=template_loader, autoescape=select_autoescape(disabled_extensions=['j2'])) 

65 template = template_env.get_template('impact_report.j2') 

66 

67 url = template.render( 

68 base_url=self.base_url, 

69 account_id=self.account_id, 

70 report=self.endpoint_name, 

71 day=self.day, 

72 other_properties=self.query_params) 

73 

74 records = self._fetch_report(url) 

75 

76 return self._transform_records(records, self.data_map, self.add_date) 

77 

78 def _transform_records(self, records, mapping, add_date=None): 

79 """ Receives a list of dicts with raw data and applies a mapping to reformat it to comply with the schema 

80 :param records: List of records fetched from the API 

81 :type records: list(dict) 

82 :param mapping: Dictionary with the location of the data in the raw data as key and the target location as value 

83 :type mapping: dict 

84 :param add_date: Flag indicating if the report date must be added as a column or not. 

85 :type add_date: bool 

86 :return: New list with the number of records as input, reformatted according to the mapping 

87 :rtype: list(dict) 

88 """ 

89 out = [] 

90 for record in records: 

91 row = {} 

92 for source, target in mapping.items(): 

93 row[target] = record[source] 

94 if add_date: 

95 row['DATE'] = self.day 

96 out.append(row) 

97 return out 

98 

99 def get_response_data(self): 

100 """ Main public function""" 

101 logging.info('Requesting Impact reports') 

102 return self._request_report()