Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1 

2import collections 

3import json 

4import logging 

5import requests 

6import datetime 

7from dateutil import parser 

8from sdc_etl_libs.api_helpers.API import API 

9from sdc_etl_libs.sdc_dataframe.Dataframe import Dataframe 

10from sdc_etl_libs.sdc_dataframe.SDCDataframeEnums import SDCDFTypes 

11from sdc_etl_libs.sdc_file_helpers.SDCFileHelpers import SDCFileHelpers 

12from sdc_etl_libs.sdc_data_schema.schema_validation import SchemaValidation 

13from sdc_etl_libs.sdc_data_schema.schema_toolbox import SchemaToolbox 

14 

15logging.basicConfig(level=logging.INFO) 

16 

17 

18class NewRelic(API): 

19 

20 def __init__(self): 

21 

22 self.credentials = self.get_credentials("aws_secrets", "new_relic/api") 

23 self.base_url = None 

24 self.headers = {'X-API-Key': self.credentials["api_key"]} 

25 

26 def get_date_filter(self, datetime_, days_, periods_=3600): 

27 """ 

28 Generates the dates portion of the New Relic API call URL. 

29 :param datetime_: Datetime object to serve as end date. 

30 :param days_: Number of days to go back from datetime_ to set as start 

31 date. 

32 :param periods_: Int. Method for breaking up metrics data into minutes, 

33 hours, etc. The ability to 

34 set this is governed by how many days are being pulled at one time. 

35 More info can be found here: 

36 https://docs.newrelic.com/docs/apis/rest-api-v2/requirements/extract-metric-timeslice-data#period 

37 Default is set to 3600, which returns 1-hour data slices when date 

38 range is less than 8 days. 

39 :return: URL date portion as string. 

40 """ 

41 

42 if type(datetime_) == str: 

43 datetime_ = parser.parse(datetime_) 

44 

45 startdate = (datetime_ - datetime.timedelta(days_)).strftime('%Y-%m-%dT01:00:00+00:00') 

46 enddate = datetime_.strftime('%Y-%m-%dT01:00:00+00:00') 

47 url_date_string = f"from={startdate}&to={enddate}&period={periods_}" 

48 

49 return url_date_string 

50 

51 def get_fields_filter(self, names_, values_): 

52 """ 

53 Generates the names and values portion of the New Relic API call URL. 

54 :param names_: List of metric names requested. 

55 :param values_: List of values requested. 

56 :return: URL string with names and values properly formatted, 

57 """ 

58 

59 if not isinstance(names_, list) and not isinstance(values_, list): 

60 raise Exception("Args names_ and values_ must be a Lists") 

61 

62 names = ["names[]={}".format(i) for i in names_] 

63 values = ["values[]={}".format(i) for i in values_] 

64 names.extend(values) 

65 url_fields_string = f"&".join(names) 

66 

67 return url_fields_string 

68 

69 def flatten_metrics_data(self, json_data_): 

70 """ 

71 Flattens the New Relic metrics timeslice data so it can be loaded into 

72 the SDCDataframe object. 

73 

74 New Relics metric data is returned in the below format. The "metrics" 

75 portion of the data is passed to this function for flattening, which 

76 will convert: 

77 

78 { 

79 "metrics": [ 

80 { 

81 "name": "EndUser/UserAgent/Mobile/Browser", 

82 "timeslices": [ 

83 { 

84 "from": "2015-10-01T01:00:00+00:00", 

85 "to": "2015-10-04T01:00:00+00:00", 

86 "values": { 

87 "average_fe_response_time": 0, 

88 "average_be_response_time": 0 

89 } 

90 } 

91 ] 

92 } 

93 ] 

94 } 

95 

96 

97 to: 

98 

99 [ 

100 { 

101 "name": "EndUser/UserAgent/Mobile/Browser", 

102 "from": "2015-10-01T01:00:00+00:00", 

103 "to": "2015-10-04T01:00:00+00:00", 

104 "average_fe_response_time": 0, 

105 "average_be_response_time": 0 

106 } 

107 ] 

108 

109 :param json_data_: List of JSON records to be flattened. 

110 :return: List of flattened dictionary records. 

111 """ 

112 

113 def flatten_dictionary(json_data_): 

114 """ 

115 Flattens JSON data into list of tuples. 

116 :param json_data_: JSON data to be flattened. 

117 :return: List of tuples. 

118 """ 

119 items = [] 

120 for key, value in json_data_.items(): 

121 if isinstance(value, collections.MutableMapping): 

122 items.extend(flatten_dictionary(value).items()) 

123 else: 

124 items.append((key, value)) 

125 return dict(items) 

126 

127 results = [] 

128 for metric in json_data_: 

129 for item in metric["timeslices"]: 

130 temp = {} 

131 temp["name"] = metric["name"] 

132 temp.update(flatten_dictionary(item)) 

133 results.append(temp) 

134 

135 return results 

136 

137 def process_metrics_endpoint(self, base_endpoint_url_, date_filter_=None, 

138 fields_filter_=None): 

139 """ 

140 Processes New Relic metric data request. A New Relic API metrics response 

141 looks like the below. The data from "metric_data"."metrics" is returned by 

142 this function in JSON format. 

143 

144 { 

145 "metric_data": { 

146 "from": "2015-10-01T01:00:00+00:00", 

147 "to": "2019-10-31T01:00:00+00:00", 

148 "metrics_not_found": [], 

149 "metrics_found": [ 

150 "EndUser/UserAgent/Mobile/Browser", 

151 "EndUser/UserAgent/Tablet/Browser", 

152 "EndUser/UserAgent/Desktop/Browser" 

153 ], 

154 "metrics": [ 

155 { 

156 "name": "EndUser/UserAgent/Mobile/Browser", 

157 "timeslices": [ 

158 { 

159 "from": "2015-10-01T01:00:00+00:00", 

160 "to": "2015-10-04T01:00:00+00:00", 

161 "values": { 

162 "average_fe_response_time": 0, 

163 "average_be_response_time": 0 

164 } 

165 } 

166 ] 

167 } 

168 ] 

169 } 

170 } 

171 

172 :param base_endpoint_url_: New Relic API base URL. 

173 :param date_filter_: Date filter as string. 

174 :param fields_filter_: Fields (names and values requested) as string. 

175 :return: List of JSON records. 

176 """ 

177 

178 data = [] 

179 requests_url = f"{base_endpoint_url_}" \ 

180 f"{fields_filter_ if fields_filter_ else ''}" \ 

181 f"{'&'+date_filter_ if date_filter_ else ''}" 

182 

183 logging.info(requests_url) 

184 

185 r = requests.get(requests_url, headers=self.headers) 

186 

187 if r.status_code == 200: 

188 try: 

189 data_json = json.loads(r.content) 

190 if not data_json: 

191 logging.info(f"No results.") 

192 except Exception as e: 

193 logging.error(e) 

194 raise Exception(f"Unable to process data: {r.content}") 

195 

196 for item in data_json["metric_data"]["metrics"]: 

197 data.append(item) 

198 

199 logging.info(f"Grabbed {len(data_json):,} record(s).") 

200 else: 

201 raise Exception( 

202 f"Failed to get access group data from api. " 

203 f"Status: {r.status_code}") 

204 

205 return data 

206 

207 def get_metrics_data(self, data_schema_name_, date_filter_=None, 

208 fields_filter_=None): 

209 """ 

210 Initiatives a New Relic API request - processing the data, flattening and 

211 loading into an SDCDataframe object. 

212 :param data_schema_name_: JSON schema file name for data. 

213 :param date_filter_: Date filter as string. 

214 :param fields_filter_: Fields (names and values requested) as string. 

215 :return: SDCDataframe object. 

216 """ 

217 

218 data_schema = json.loads(open(SDCFileHelpers.get_file_path( 

219 'schema', f"NewRelic/{data_schema_name_}.json")).read()) 

220 validation = SchemaValidation() 

221 validated_schema = validation.validate_schema(data_schema) 

222 validated_source_endpoint_schema = SchemaToolbox.get_endpoint_data_from_schema(validated_schema, "main_source") 

223 self.base_url = validated_source_endpoint_schema["info"]["access"]["base_url"] 

224 

225 df = Dataframe(SDCDFTypes.PANDAS, validated_schema) 

226 

227 requests_url = self.base_url + f'/metrics/data.json?' 

228 

229 data = self.process_metrics_endpoint(requests_url, date_filter_, 

230 fields_filter_) 

231 

232 flattened_data = self.flatten_metrics_data(data) 

233 

234 if len(data) >= 1: 

235 df.load_data(flattened_data) 

236 return df 

237 

238 else: 

239 logging.warning("Received no data.") 

240 return None