Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import datetime 

2import json 

3import logging 

4import os 

5 

6import FuelSDK 

7import pandas as pd 

8from dateutil import parser 

9from sdc_etl_libs.api_helpers.API import API 

10from sdc_etl_libs.api_helpers.apis.ExactTarget.ExactTargetHelpers import \ 

11 ExactTargetHelpers 

12from sdc_etl_libs.sdc_dataframe.Dataframe import Dataframe 

13from sdc_etl_libs.sdc_dataframe.SDCDataframeEnums import SDCDFTypes 

14from sdc_etl_libs.sdc_file_helpers.SDCFileHelpers import SDCFileHelpers 

15 

16logging.basicConfig(level=logging.INFO) 

17 

18 

19 

20class ExactTarget(API): 

21 

22 def __init__(self, credential_type_="aws_secrets"): 

23 # secrets in the from of {"username":,"password":, "api_key": } 

24 # self.base_url = base_url_ 

25 

26 self.credentials = self.get_credentials(credential_type_, "exact-target/api") 

27 

28 self.client = FuelSDK.ET_Client(params=self.credentials) 

29 

30 def normalize_data(self, data_): 

31 for index in range(len(data_)): 

32 data_[index] = pd.io.json.json_normalize( 

33 ExactTargetHelpers.suds_to_dict(data_[index]), sep='_').to_dict(orient='records')[0] 

34 return data_ 

35 

36 def or_two_filters(self, filter1_, filter2_): 

37 """ 

38 :param filter1_: A filter returned by the 

39 get_filter_for_last_n_minutes function 

40 :param filter2_: A filter returned by the 

41 get_filter_for_last_n_minutes function 

42 :return: a complex filter dict that is logical "or" together.. 

43 """ 

44 out_filter = {'LeftOperand': filter1_, 'LogicalOperator': 'OR', 'RightOperand': filter2_} 

45 return out_filter 

46 

47 def get_filter_for_last_n_minutes(self, date_property_, minutes_, datetime_): 

48 """ 

49 This function creates a date filter of the previous n minutes. 

50 https://developer.salesforce.com/docs/atlas.en-us.noversion.mc-apis 

51 .meta/mc-apis/creating_a_filterdefinition_object.htm 

52 :param date_propery_: property name that corresponds to date we want 

53 to filter by. 

54 :param minutes_: an integer that is the number of minutes in the past 

55 you want to cover. 

56 :param datetime_: dateime object or a string that represents a datetime. 

57 :return: a filter dict that is the data from the previous day. 

58 """ 

59 if not isinstance(minutes_, int): 

60 raise Exception("Minutes must be an int for the filter.") 

61 if type(datetime_) == str: 

62 datetime_ = parser.parse(datetime_) 

63 

64 from_dt = (datetime_ - datetime.timedelta(minutes=minutes_)).strftime("%Y-%m-%dT%H:%M:%S") 

65 to_dt = datetime_.strftime("%Y-%m-%dT%H:%M:%S") 

66 

67 logging.info(f"from {from_dt} to {to_dt}") 

68 

69 from_filter = {'Property': date_property_, 'SimpleOperator': 'greaterThan', 'DateValue': from_dt} 

70 to_filter = {'Property': date_property_, 'SimpleOperator': 'lessThanOrEqual', 'DateValue': to_dt} 

71 date_filter = {'LeftOperand': from_filter, 'LogicalOperator': 'AND', 'RightOperand': to_filter} 

72 

73 return date_filter 

74 

75 def process_endpoint(self, endpoint_name_, endpoint_, filter_): 

76 out = [] 

77 endpoint_ = endpoint_() 

78 endpoint_.auth_stub = self.client 

79 if not isinstance(filter_, type(None)): 

80 endpoint_.search_filter = filter_ 

81 

82 request = endpoint_.get() 

83 

84 if not request.status: 

85 raise Exception(f"Request failed on endpoint {endpoint_name_}, message: {request.message}") 

86 

87 out = out + request.results 

88 while request.more_results: 

89 request = endpoint_.getMoreResults() 

90 out = out + request.results 

91 

92 return out 

93 

94 def get_events(self, filter_=None): 

95 file_name = SDCFileHelpers.get_file_path("schema", "ExactTarget/events.json") 

96 json_data = json.loads(open(file_name).read()) 

97 

98 df = Dataframe(SDCDFTypes.PANDAS, json_data) 

99 data = [] 

100 

101 endpoints = { 

102 'SentEvent': FuelSDK.ET_SentEvent, 

103 'ClickEvent': FuelSDK.ET_ClickEvent, 

104 'OpenEvent': FuelSDK.ET_OpenEvent, 

105 'BounceEvent': FuelSDK.ET_BounceEvent, 

106 'UnsubEvent': FuelSDK.ET_UnsubEvent 

107 } 

108 

109 for name, endpoint in endpoints.items(): 

110 data = data + self.process_endpoint(name, endpoint, filter_) 

111 

112 data = self.normalize_data(data) 

113 

114 if len(data) >= 1: 

115 df.load_data(data) 

116 return df 

117 else: 

118 logging.warning("Received no data") 

119 return None 

120 

121 def get_subscribers(self, filter_=None): 

122 file_name = SDCFileHelpers.get_file_path("schema", "ExactTarget/subscribers.json") 

123 

124 json_data = json.loads(open(file_name).read()) 

125 

126 df = Dataframe(SDCDFTypes.PANDAS, json_data) 

127 data = [] 

128 

129 data = self.process_endpoint("Subscriber", FuelSDK.ET_Subscriber, filter_) 

130 

131 data = self.normalize_data(data) 

132 

133 if len(data) >= 1: 

134 df.load_data(data) 

135 return df 

136 

137 else: 

138 logging.warning("Received no data.") 

139 return None 

140 

141 def get_list_subscribers(self, filter_=None): 

142 """ 

143 This data has a Status property that can change over time. Will need to 

144 use this func twice to (1) grab new records for a day by CreatedDate 

145 and insert and (2) grab updated records for a day by ModifiedDate 

146 and upsert. 

147 

148 :param filter_: 

149 :return: 

150 """ 

151 file_name = SDCFileHelpers.get_file_path("schema", "ExactTarget/list_subscribers.json") 

152 

153 json_data = json.loads(open(file_name).read()) 

154 

155 df = Dataframe(SDCDFTypes.PANDAS, json_data) 

156 data = [] 

157 

158 data = self.process_endpoint("List_Subscriber", FuelSDK.ET_List_Subscriber, filter_) 

159 

160 data = self.normalize_data(data) 

161 

162 if len(data) >= 1: 

163 df.load_data(data) 

164 return df 

165 

166 else: 

167 logging.warning("Received no data.") 

168 return None 

169 

170 def get_sends(self, filter_=None): 

171 

172 file_name = SDCFileHelpers.get_file_path("schema", "ExactTarget/sends.json") 

173 

174 json_data = json.loads(open(file_name).read()) 

175 

176 df = Dataframe(SDCDFTypes.PANDAS, json_data) 

177 data = [] 

178 

179 data = self.process_endpoint("Send", FuelSDK.ET_Send, filter_) 

180 

181 data = self.normalize_data(data) 

182 

183 if len(data) >= 1: 

184 df.load_data(data) 

185 return df 

186 

187 else: 

188 logging.warning("Received no data.") 

189 return None