1""" Class to fetch data from the TikTok API """ 

2# pylint: disable=E0401, R0902, R0913, R0903, W0613 

3import json 

4import logging 

5import urllib.parse 

6 

7import requests 

8from jinja2 import Environment, FileSystemLoader, select_autoescape 

9 

10from sdc_etl_libs.api_helpers.API import API 

11from sdc_etl_libs.api_helpers.SDCAPIExceptions import AccessException 

12from sdc_etl_libs.sdc_file_helpers.SDCFileHelpers import SDCFileHelpers 

13 

14logging.info("EXECUTING: %s", __name__) 

15 

16 

17class TikTok(API): 

18 """ 

19 Connector/Wrapper class for access Tiktok campaign information/metrics through. 

20 Ref.: 

21 https://ads.tiktok.com/marketing_api/docs?id=8 

22 """ 

23 

24 PAGE_SIZE = 1000 

25 

26 def __init__(self, schema_: dict, endpoint_schema_: dict, **kwargs): 

27 """Initiate API wrapper instance 

28 :param schema_: The full dictionary of the schema used 

29 :type schema_: dict 

30 :param endpoint_schema_: data_source_ part of the schema 

31 :type endpoint_schema_: dict 

32 :param kwargs: collection of named args required for this particular class 

33 :type kwargs: dict 

34 """ 

35 credentials_source = endpoint_schema_["info"]["access"]["credentials"] 

36 self.credentials = self.get_credentials( 

37 source_=credentials_source['type'], aws_secret_id_=credentials_source['opts']['name']) 

38 

39 self.app_id = self.credentials['app_id'] \ 

40 if not kwargs.get('app_id') else kwargs.get('app_id') 

41 self.secret = self.credentials['secret'] \ 

42 if not kwargs.get('secret') else kwargs.get('secret') 

43 self.advertiser_id = self.credentials['advertiser_id'] \ 

44 if not kwargs.get('advertiser_id') else kwargs.get('advertiser_id') 

45 self.access_token = self.credentials['access_token'] \ 

46 if not kwargs.get('access_token') else kwargs.get('access_token') 

47 

48 self.day = kwargs['day_'] 

49 self.base_url = endpoint_schema_["info"]["access"]['base_url'] 

50 self.endpoint = endpoint_schema_["info"]["access"]['endpoint_name'] 

51 

52 logging.info("ad_status: %s", kwargs.get('ad_status')) 

53 logging.debug("ad_status type ====> %s", type(kwargs.get('ad_status'))) 

54 

55 ad_status = "AD_STATUS_ALL" if not kwargs.get('ad_status') else kwargs.get('ad_status') 

56 

57 def validate(ad_status): 

58 """ 

59 TikTok variables documented in https://ads.tiktok.com/marketing_api/docs?id=100641 

60 Validate ad_status variable in the range of values 

61 To filter deleted ads the variable ad_status must set to 

62 AD_STATUS_CAMPAIGN_DELETE, AD_STATUS_ADGROUP_DELETE, AD_STATUS_DELETE 

63 :param ad_status: list of status 

64 :type list:str 

65 :return: boolean 

66 :rtype: boolean 

67 """ 

68 self_ad_status = ad_status if (ad_status in [ 

69 "AD_STATUS_CAMPAIGN_DELETE", "AD_STATUS_ADGROUP_DELETE", "AD_STATUS_DELETE", 

70 "AD_STATUS_ADVERTISER_AUDIT_DENY", "AD_STATUS_ADVERTISER_AUDIT", "AD_STATUS_BALANCE_EXCEED", 

71 "AD_STATUS_CAMPAIGN_EXCEED", "AD_STATUS_BUDGET_EXCEED", "AD_STATUS_AUDIT", "AD_STATUS_REAUDIT", 

72 "AD_STATUS_AUDIT_DENY", "AD_STATUS_ADGROUP_AUDIT_DENY", "AD_STATUS_NOT_START", "AD_STATUS_DONE", 

73 "AD_STATUS_CAMPAIGN_DISABLE", "AD_STATUS_ADGROUP_DISABLE", "AD_STATUS_DISABLE", "AD_STATUS_DELIVERY_OK", 

74 "AD_STATUS_ALL", "AD_STATUS_NOT_DELETE" 

75 ]) else None 

76 return self_ad_status is not None 

77 

78 self.list_status = "".join(ad_status.split()).upper().split(",") 

79 is_valid_status = all(list(map(validate, self.list_status))) 

80 

81 if not is_valid_status: 

82 """Variable AD_STATUS does not match one of the valid range of values""" 

83 raise AccessException(f"Variable AD_STATUS is {ad_status}. Please enter correct AD_STATUS") 

84 

85 def _consume_api(self, method, query_params, path): 

86 """Fetches data from the API 

87 :param method: HTTP Method to use while quering the API 

88 :type method: str 

89 :param query_params: Key, value pairs of parameters to add to the query 

90 :type query_params: dict 

91 :param path: endpoint to consume of the API 

92 :type path: str 

93 :return: Return code, result pagination info and list of events fetched. 

94 :rtype: tuple 

95 """ 

96 headers = {'Content-Type': 'application/json', 'Access-Token': self.access_token} 

97 

98 template_loader = FileSystemLoader(searchpath=SDCFileHelpers.get_file_path(type_='template', path_="TikTok")) 

99 template_env = Environment(loader=template_loader, autoescape=select_autoescape(disabled_extensions=['j2'])) 

100 template = template_env.get_template('get_query.j2') 

101 url = template.render(base_url=self.base_url, path=path, other_properties=query_params) 

102 ret = requests.request(method=method, url=url, headers=headers) 

103 

104 if ret.status_code == 200: 

105 return ret.status_code, ret.json()['data'].get('page_info'), ret.json()['data']['list'] 

106 return ret.status_code, None, None 

107 

108 def _fetch_data(self, method, query_params, path): 

109 """Fetches data from an API endpoint managing pagination 

110 :param method: type of HTTP query to perform ('GET', 'POST', 'DELETE', etc...) 

111 :type method: str 

112 :param query_params: Key Value pairs of params to send to the enpoint in the query 

113 :type query_params: dict 

114 :param path: endpoint to consume 

115 :type path: str 

116 :return: list of dicts fetched from API 

117 :rtype: list 

118 """ 

119 query_params['page'] = 1 

120 query_params['page_size'] = self.PAGE_SIZE 

121 

122 ret = [] 

123 

124 while True: 

125 status, page_info, fetched_data = self._consume_api(method, query_params, path) 

126 if status != 200: 

127 logging.error("Query to the API Failed") 

128 break 

129 ret = ret + fetched_data 

130 if page_info and page_info['total_page'] > query_params['page']: 

131 query_params['page'] = query_params['page'] + 1 

132 else: 

133 break 

134 return ret 

135 

136 @staticmethod 

137 def _build_fields(fields_list): 

138 """Using a list of values, create a URL safe string with an array of values 

139 :param fields_list: list of values to be sent inside a query 

140 :type fields_list: list 

141 :return: URI safe string with the list of values to be sent 

142 :rtype: str 

143 """ 

144 return urllib.parse.quote(json.dumps(fields_list)) 

145 

146 def _fetch_ad_ids(self): 

147 """Prepare params and formats output of ads report API query 

148 :return: All available ads indexed by ad_id 

149 :rtype: dict 

150 """ 

151 

152 def _aux__fetch_ad_ids(ad_status): 

153 """ 

154 moved _fetch_ad_ids code to an auxiliary function 

155 to abstract the API call and put conditions 

156 before calling the original function. 

157 """ 

158 fields = ['ad_id', 'ad_name', 'adgroup_id', 'adgroup_name', 'campaign_id', 'campaign_name'] 

159 ad_fields = self._build_fields(fields) 

160 filtering = {"status": ad_status} 

161 ad_filtering = self._build_fields(filtering) 

162 ad_params = { 

163 'advertiser_id': self.advertiser_id, 

164 'fields': ad_fields, 

165 'filtering': ad_filtering, 

166 'page_size': self.PAGE_SIZE 

167 } 

168 logging.info("'FILTERING STATUS: %s", ad_status) 

169 ads = self._fetch_data('GET', ad_params, 'ad/get/') 

170 return {ad['ad_id']: {field.upper(): ad[field] for field in fields} for ad in ads} 

171 

172 dict_ad_aids = {} 

173 for ad_status in self.list_status: 

174 dict_ad_aids.update(_aux__fetch_ad_ids(ad_status)) 

175 return dict_ad_aids 

176 

177 def _fetch_ads_report(self, fields, dimensions, ad_ids): 

178 """Prepare params to fetch ads report from API 

179 :param fields: URL escaped string with a list of fields to fetch 

180 :type fields: str 

181 :param dimensions: URL escaped string with a list of dimensions to fetch 

182 :type dimensions: str 

183 :param ad_ids: URL escaped string with a list of ad_ids to fetch 

184 :type ad_ids: str 

185 :return: fetched data from ads report API 

186 :rtype: list 

187 """ 

188 report_params = { 

189 'advertiser_id': self.advertiser_id, 

190 'fields': fields, 

191 'ad_ids': ad_ids, 

192 'dimensions': dimensions, 

193 'start_date': self.day, 

194 'end_date': self.day 

195 } 

196 return self._fetch_data('GET', report_params, 'audience/ad/get/') 

197 

198 def _prepare_ads_report(self, report_data, ads_dict): 

199 """Gets fetched ads data and formats it for SDCDataExchange insertion 

200 :param report_data: Metrics data fetched from the ads report API 

201 :type report_data: list 

202 :param ads_dict: Ad data fetched from the ads API 

203 :type ads_dict: list 

204 :return: Formatted Ads report for all the ads available 

205 :rtype: list 

206 """ 

207 metrics = ['STAT_COST', 'CLICK_CNT', 'SHOW_CNT', 'TIME_ATTR_CONVERT_CNT'] 

208 report = [] 

209 for row in report_data: 

210 ad_row = ads_dict[row['dimensions']['ad_id']] 

211 ad_row['DATE'] = self.day 

212 for metric in metrics: 

213 ad_row[metric] = row['metrics'][metric.lower()] 

214 report.append(ad_row) 

215 return report 

216 

217 def _get_ads_report(self): 

218 """Procedure to fetch the Ads report from the specific endpoints of the API 

219 :return: Ads report for all the ads available for the day 

220 :rtype: dict 

221 """ 

222 ads_dict = self._fetch_ad_ids() 

223 ad_ids = list(ads_dict.keys()) 

224 ad_ids_chunks = [ad_ids[i:i + 100] for i in range(0, len(ad_ids), 100)] 

225 

226 report_fields = self._build_fields(['time_attr_convert_cnt', 'click_cnt', 'show_cnt', 'stat_cost']) 

227 report_dimensions = self._build_fields(['PLACEMENT']) 

228 

229 report_data = [] 

230 for chunk in ad_ids_chunks: 

231 report_data = report_data + self._fetch_ads_report(report_fields, report_dimensions, 

232 self._build_fields(chunk)) 

233 return self._prepare_ads_report(report_data, ads_dict) 

234 

235 def get_response_data(self): 

236 """Public method to fetch the requested report on instantiation 

237 :return: All data required to write the report into a SDCExchange sink 

238 :rtype: dict 

239 """ 

240 return getattr(self, '_get_' + self.endpoint)()