Coverage for libs/sdc_etl_libs/api_helpers/apis/Impact/ImpactAPI.py : 100%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1""" Class to fetch data from the Impact API """
2# pylint: disable=E0401, R0902, R0913, R0903
3import json
4import logging
6import requests
7from jinja2 import Environment, FileSystemLoader, select_autoescape
8from sdc_etl_libs.api_helpers.API import API
9from sdc_etl_libs.sdc_file_helpers.SDCFileHelpers import SDCFileHelpers
11logging.info("EXECUTING: %s", __name__)
14class Impact(API):
15 """Connector/Wrapper class for fetching Impact reports.
16 Ref.: https://developer.impact.com/default/documentation/Adv-v8
17 """
19 def __init__(self, schema_: dict = None, endpoint_schema_: dict = None, **kwargs):
20 """Initiate API wrapper instance
21 :param schema_: The full dictionary of the schema used
22 :type schema_: dict
23 :param endpoint_schema_: data_source_ part of the schema
24 :type endpoint_schema_: dict
25 :param kwargs: collection of named args required for this particular class
26 :type kwargs: dict
27 """
29 meta_name = schema_['name']
30 meta_path = SDCFileHelpers.get_file_path("metadata", 'Vendors/Impact/{}.json'.format(meta_name))
31 meta = json.loads(open(meta_path).read())
33 self.base_url = endpoint_schema_["info"]["access"]['base_url']
34 self.endpoint_name = endpoint_schema_["info"]["access"]['endpoint_name']
35 self.day = kwargs['day_']
36 self.query_params = meta['query_params_']
37 self.data_map = meta['data_map_']
38 self.add_date = meta.get('add_date_')
40 self.credentials = self.get_credentials(source_="aws_secrets", aws_secret_id_="Impact/API") # nosec
42 self.account_id = self.credentials['account_id'] \
43 if not kwargs.get('account_id_') else kwargs.get('account_id_')
44 self.auth_token = self.credentials['auth_token'] \
45 if not kwargs.get('auth_token_') else kwargs.get('auth_token_')
47 def _fetch_report(self, url):
48 """Fetches data from the impact API
49 :param url: Well formed query containing all data to query the API
50 :type url: str
51 :return: List of events fetched by all the instances of the recursion, until no more links are returned.
52 :rtype: list
53 """
55 response = requests.request("GET", url, auth=(self.account_id, self.auth_token))
56 return response.json()["Records"]
58 def _request_report(self):
59 """ Fetches the raw data from the API and reformats it
60 :return: List of elements fetched from API reformatted to comply with the required schema.
61 :rtype: list(dict)
62 """
63 template_loader = FileSystemLoader(searchpath=SDCFileHelpers.get_file_path(type_='template', path_="Impact"))
64 template_env = Environment(loader=template_loader, autoescape=select_autoescape(disabled_extensions=['j2']))
65 template = template_env.get_template('impact_report.j2')
67 url = template.render(
68 base_url=self.base_url,
69 account_id=self.account_id,
70 report=self.endpoint_name,
71 day=self.day,
72 other_properties=self.query_params)
74 records = self._fetch_report(url)
76 return self._transform_records(records, self.data_map, self.add_date)
78 def _transform_records(self, records, mapping, add_date=None):
79 """ Receives a list of dicts with raw data and applies a mapping to reformat it to comply with the schema
80 :param records: List of records fetched from the API
81 :type records: list(dict)
82 :param mapping: Dictionary with the location of the data in the raw data as key and the target location as value
83 :type mapping: dict
84 :param add_date: Flag indicating if the report date must be added as a column or not.
85 :type add_date: bool
86 :return: New list with the number of records as input, reformatted according to the mapping
87 :rtype: list(dict)
88 """
89 out = []
90 for record in records:
91 row = {}
92 for source, target in mapping.items():
93 row[target] = record[source]
94 if add_date:
95 row['DATE'] = self.day
96 out.append(row)
97 return out
99 def get_response_data(self):
100 """ Main public function"""
101 logging.info('Requesting Impact reports')
102 return self._request_report()