1 

2import datetime 

3from dateutil import parser 

4import json 

5import logging 

6import requests 

7from sdc_etl_libs.api_helpers.apis.Ultipro.Ultipro import Ultipro 

8from sdc_etl_libs.sdc_dataframe.Dataframe import Dataframe 

9from sdc_etl_libs.sdc_dataframe.SDCDataframeEnums import SDCDFTypes 

10from sdc_etl_libs.sdc_file_helpers.SDCFileHelpers import SDCFileHelpers 

11from sdc_etl_libs.sdc_data_schema.schema_validation import SchemaValidation 

12from sdc_etl_libs.sdc_data_schema.schema_toolbox import SchemaToolbox 

13 

14logging.basicConfig(level=logging.INFO) 

15 

16class UltiproRESTAPIs(Ultipro): 

17 

18 def __init__(self): 

19 super().__init__() 

20 self.rest_authenticate("username", "password") 

21 

22 def get_daily_filter(self, datetime_, days_=1, type_=1, field_=None): 

23 """ 

24 This function creates a date filter of the previous day. 

25 :param datetime_: Datetime object tp serve as end date. 

26 :param days_: Number of days to go back from datetime_ to set as start date. 

27 :param type_: Type of filter to generate. Date range is between the 

28 start date (date_time less the days_) to the end date (datetime_) 

29 

30 Options include: 

31 1 = Generates URL filter string with the field_ name provided. 

32 2 = Generates URL filter string with literal "startDate" 

33 and "endDate" fields used. 

34 

35 :param field_: List of fields to create complex filter with (with type 1 filter only) 

36 :return: Filter string of a date range to use in request URL. 

37 """ 

38 

39 if type(datetime_) == str: 

40 datetime_ = parser.parse(datetime_) 

41 

42 if type_ == 1: 

43 

44 if not field_: 

45 raise Exception("Must provide field for Type 1 filtering on REST API") 

46 

47 startdate = (datetime_ - datetime.timedelta(days_)).strftime("%Y-%m-%d") 

48 enddate = datetime_.strftime("%Y-%m-%d") 

49 url_filter = f'{field_}={{{startdate}T00:00:00,{enddate}T23:59:59}}' 

50 return url_filter 

51 

52 elif type_ == 2: 

53 

54 startdate = (datetime_ - datetime.timedelta(days_)).strftime("%Y/%m/%d") 

55 enddate = datetime_.strftime("%Y/%m/%d") 

56 url_filter = f'startDate={startdate}T00:00:00&endDate={enddate}T23:59:59' 

57 return url_filter 

58 

59 else: 

60 raise Exception(f"Filter type {type_} not supported.") 

61 

62 def process_endpoint(self, base_endpoint_url_, filter_, limit_=None): 

63 """ 

64 This function handles the pagination of the ultipro api calls. 

65 :param base_endpoint_url_: base url for the api : string 

66 :param filter_: filter for api endpoint: string 

67 :param limit_: How many records to return per page : int. 

68 NOTE: Different endpoints have different limits. If set too high, 

69 the API will either automatically set it to the max, or, error out. 

70 :return: List of dictionary results. 

71 """ 

72 

73 data = [] 

74 page = 1 

75 while True: 

76 if page > 1000: 

77 break 

78 if filter_ is not None: 

79 requests_url = f"{base_endpoint_url_}?{filter_}&page={page}" 

80 else: 

81 requests_url = f"{base_endpoint_url_}?page={page}" 

82 

83 if limit_ is not None: 

84 if type(limit_) == int: 

85 requests_url = f"{requests_url}&per_page={limit_}" 

86 else: 

87 raise Exception("limit_ must be of type integer") 

88 

89 r = requests.get(requests_url, auth=self.auth, 

90 headers={"US-Customer-Api-Key":self.credentials["api_key"]}) 

91 

92 if r.status_code == 200: 

93 try: 

94 data_json = json.loads(r.content) 

95 except Exception as e: 

96 logging.error(e) 

97 raise Exception(f"Unable to process data: {r.content}") 

98 

99 if len(data_json) < 1: 

100 break 

101 

102 for item in data_json: 

103 data.append(item) 

104 

105 logging.info(f"Grabbed {len(data_json):,} record(s) from page " 

106 f"{page}. {len(data):,} total records so far.") 

107 

108 else: 

109 raise Exception( 

110 f"Failed to get data from api. status: {r.status_code}") 

111 

112 page += 1 

113 

114 return data 

115 

116 def get_employment_details(self, filter_=None, limit_=100): 

117 """ 

118 This function grabs data from the employment-details api and returns a dataframe. 

119 :param filter_: Specific filters to narrow down queried data. 

120 :param limit_: How many records to return per page. Default (and endpoint max) is 100. 

121 https://connect.ultipro.com/documentation#/api/817/EmploymentDetails/get__personnel_v1_employment-details 

122 :return: Dataframe 

123 """ 

124 

125 data_schema = json.loads(open(SDCFileHelpers.get_file_path( 

126 'schema', "Ultipro/rest_apis/employment-details.json")).read()) 

127 validation = SchemaValidation() 

128 validated_schema = validation.validate_schema(data_schema) 

129 validated_source_endpoint_schema = SchemaToolbox.get_endpoint_data_from_schema(validated_schema, "main_source") 

130 self.base_url = validated_source_endpoint_schema["info"]["access"]["base_url"] 

131 df = Dataframe(SDCDFTypes.PANDAS, validated_schema) 

132 

133 base_endpoint_url = self.base_url + '/employment-details' 

134 

135 data = self.process_endpoint(base_endpoint_url, filter_, limit_) 

136 

137 if len(data) >= 1: 

138 df.load_data(data) 

139 return df 

140 else: 

141 logging.warning("Received no data") 

142 return None 

143 

144 def get_employee_changes(self, filter_=None, limit_=200): 

145 """ 

146 This function will get data from the employment changes api and return a dataframe 

147 :param filter_: Specific filters to narrow down queried data. 

148 :param limit_: How many records to return per page. Default (and endpoint max) is 200. 

149 https://connect.ultipro.com/documentation#/api/199/Changes%20By%20Date/get__personnel_v1_employee-changes 

150 :return: Dataframe 

151 """ 

152 

153 data_schema = json.loads(open(SDCFileHelpers.get_file_path( 

154 'schema', "Ultipro/rest_apis/employee-changes.json")).read()) 

155 validation = SchemaValidation() 

156 validated_schema = validation.validate_schema(data_schema) 

157 validated_source_endpoint_schema = SchemaToolbox.get_endpoint_data_from_schema(validated_schema, "main_source") 

158 self.base_url = validated_source_endpoint_schema["info"]["access"]["base_url"] 

159 df = Dataframe(SDCDFTypes.PANDAS, validated_schema) 

160 

161 base_endpoint_url = self.base_url + '/employee-changes' 

162 

163 data = self.process_endpoint(base_endpoint_url, filter_, limit_) 

164 

165 if len(data) >= 1: 

166 df.load_data(data) 

167 return df 

168 else: 

169 logging.warning("Received no data") 

170 return None 

171 

172 def get_compensation_details(self, filter_=None, limit_=100): 

173 """ 

174 This functions grabs the compensation-details api and returns a dataframe. 

175 :param filter_: Specific filters to narrow down queried data. 

176 :param limit_: How many records to return per page. Default (and endpoint max) is 200. 

177 https://connect.ultipro.com/documentation#/api/823/CompensationDetails/get__personnel_v1_compensation-details 

178 :return: Dataframe 

179 """ 

180 

181 data_schema = json.loads(open(SDCFileHelpers.get_file_path( 

182 'schema', "Ultipro/rest_apis/compensation-details.json")).read()) 

183 validation = SchemaValidation() 

184 validated_schema = validation.validate_schema(data_schema) 

185 validated_source_endpoint_schema = SchemaToolbox.get_endpoint_data_from_schema(validated_schema, "main_source") 

186 self.base_url = validated_source_endpoint_schema["info"]["access"]["base_url"] 

187 df = Dataframe(SDCDFTypes.PANDAS, validated_schema) 

188 

189 base_endpoint_url = self.base_url + '/compensation-details' 

190 

191 data = self.process_endpoint(base_endpoint_url, filter_, limit_) 

192 

193 if len(data) >= 1: 

194 df.load_data(data) 

195 return df 

196 else: 

197 logging.warning("Received no data") 

198 return None 

199 

200 def get_pto_plans(self, filter_=None, limit_=1000): 

201 """ 

202 This functions grabs the pto-plans api and returns a SDCDataframe object 

203 with data loaded into the dataframe if data is available. 

204 :param filter_: Specific filters to narrow down queried data. 

205 :param limit_: How many records to return per page. Default (and endpoint max) is 1,000. 

206 

207 https://connect.ultipro.com/documentation#/api/721/ 

208 :return: Dataframe 

209 """ 

210 

211 data_schema = json.loads(open(SDCFileHelpers.get_file_path( 

212 'schema', "Ultipro/rest_apis/pto-plans.json")).read()) 

213 validation = SchemaValidation() 

214 validated_schema = validation.validate_schema(data_schema) 

215 validated_source_endpoint_schema = SchemaToolbox.get_endpoint_data_from_schema(validated_schema, "main_source") 

216 self.base_url = validated_source_endpoint_schema["info"]["access"]["base_url"] 

217 df = Dataframe(SDCDFTypes.PANDAS, validated_schema) 

218 

219 base_endpoint_url = self.base_url + '/pto-plans' 

220 

221 data = self.process_endpoint(base_endpoint_url, filter_, limit_) 

222 

223 if len(data) >= 1: 

224 df.load_data(data) 

225 return df 

226 else: 

227 logging.warning("Received no data") 

228 return None 

229 

230 def get_person_details(self, filter_=None, limit_=100): 

231 """ 

232 This functions grabs the person-details api and returns a 

233 SDCDataframe object with data loaded into the dataframe if data is available. 

234 :param filter_: Specific filters to narrow down queried data. 

235 :param limit_: How many records to return per page. Default (and endpoint max) is 100. 

236 

237 https://connect.ultipro.com/documentation#/api/811/PersonDetails/get__personnel_v1_person-details 

238 :return: Dataframe 

239 """ 

240 

241 data_schema = json.loads(open(SDCFileHelpers.get_file_path( 

242 'schema', "Ultipro/rest_apis/person-details.json")).read()) 

243 validation = SchemaValidation() 

244 validated_schema = validation.validate_schema(data_schema) 

245 validated_source_endpoint_schema = SchemaToolbox.get_endpoint_data_from_schema(validated_schema, "main_source") 

246 self.base_url = validated_source_endpoint_schema["info"]["access"]["base_url"] 

247 df = Dataframe(SDCDFTypes.PANDAS, validated_schema) 

248 

249 base_endpoint_url = self.base_url + '/person-details' 

250 

251 data = self.process_endpoint(base_endpoint_url, filter_, limit_) 

252 

253 if len(data) >= 1: 

254 df.load_data(data) 

255 return df 

256 else: 

257 logging.warning("Received no data") 

258 return None 

259 

260 def get_employee_job_history_details(self, filter_=None, limit_=200): 

261 """ 

262 This functions grabs the employee-job-history-details api and returns a 

263 SDCDataframe object with data loaded into the dataframe if data is available. 

264 :param filter_: Specific filters to narrow down queried data. 

265 :param limit_: How many records to return per page. Default (and endpoint max) is 200. 

266 

267 https://connect.ultipro.com/documentation#/api/1468 

268 :return: Dataframe 

269 """ 

270 

271 data_schema = json.loads(open(SDCFileHelpers.get_file_path( 

272 'schema', "Ultipro/rest_apis/employee-job-history-details.json")).read()) 

273 validation = SchemaValidation() 

274 validated_schema = validation.validate_schema(data_schema) 

275 validated_source_endpoint_schema = SchemaToolbox.get_endpoint_data_from_schema(validated_schema, "main_source") 

276 self.base_url = validated_source_endpoint_schema["info"]["access"]["base_url"] 

277 df = Dataframe(SDCDFTypes.PANDAS, validated_schema) 

278 

279 base_endpoint_url = self.base_url + '/employee-job-history-details' 

280 

281 data = self.process_endpoint(base_endpoint_url, filter_, limit_) 

282 

283 if len(data) >= 1: 

284 df.load_data(data) 

285 return df 

286 else: 

287 logging.warning("Received no data") 

288 return None