1 

2import datetime 

3from dateutil import parser 

4import json 

5import logging 

6import requests 

7import time 

8from sdc_etl_libs.api_helpers.apis.Ultipro.Ultipro import Ultipro 

9from sdc_etl_libs.sdc_dataframe.Dataframe import Dataframe 

10from sdc_etl_libs.sdc_dataframe.SDCDataframeEnums import SDCDFTypes 

11from sdc_etl_libs.sdc_file_helpers.SDCFileHelpers import SDCFileHelpers 

12from sdc_etl_libs.sdc_data_schema.schema_validation import SchemaValidation 

13from sdc_etl_libs.sdc_data_schema.schema_toolbox import SchemaToolbox 

14 

15logging.basicConfig(level=logging.INFO) 

16 

17 

18class UltiproTimeManagement(Ultipro): 

19 

20 def __init__(self): 

21 super().__init__() 

22 self.rest_authenticate("tm_username", "tm_password") 

23 

24 def get_daily_filter(self, datetime_, days_, filter_field_list_): 

25 """ 

26 Constructs a filter for Time Management calls using the list of supplied 

27 fields. A start date and end date is set with the fields used with 

28 "greater than or equal to" start date and "less than" end date. When 

29 there is more than one field they are combined with 'or'. 

30 

31 :param datetime_: Datetime object tp serve as end date. 

32 :param days_: Number of days to go back from datetime_ to set as start date. 

33 :param filter_field_list_: List of fields to create complex filter with. 

34 :return: URL filter as string. 

35 """ 

36 

37 if not isinstance(filter_field_list_, list): 

38 raise Exception("fields_ must be a list for Time Management " 

39 "get_daily_filter()") 

40 

41 if type(datetime_) == str: 

42 datetime_ = parser.parse(datetime_) 

43 

44 startdate = (datetime_ - datetime.timedelta(days_)).strftime("%Y-%m-%d") 

45 enddate = datetime_.strftime("%Y-%m-%d") 

46 url_filter = \ 

47 " or ".join(f'({field} ge {startdate} ' 

48 f'and {field} le {enddate})' for field in filter_field_list_) 

49 return url_filter 

50 

51 def process_endpoint(self, base_endpoint_url_, filter_, limit_=1000): 

52 """ 

53 This function handles the pagination of the Ultipro API calls for 

54 Time Management UTMOData Service. 

55 

56 :param base_endpoint_url_: base url for the api : string 

57 :param filter_: filter for api endpoint: string 

58 :param limit_: Number of records to return with each call. Default is 1,000. 

59 :return: List of JSON values 

60 """ 

61 

62 data = [] 

63 

64 if filter_ is not None: 

65 requests_url = f"{base_endpoint_url_}?$filter={filter_}&$count=true" 

66 else: 

67 requests_url = f"{base_endpoint_url_}?&$count=true" 

68 

69 total_records = 1 

70 skip = 0 

71 

72 while skip < total_records: 

73 

74 requests_url_with_pagination = requests_url + \ 

75 f"&$skip={skip}&$top={limit_}" 

76 

77 logging.info(requests_url_with_pagination) 

78 

79 response = requests.get(requests_url_with_pagination, auth=self.auth) 

80 

81 if response.status_code == 200: 

82 try: 

83 data_json = json.loads(response.content) 

84 total_records = data_json["@odata.count"] 

85 records = data_json["value"] 

86 if skip == 0: 

87 logging.info(f"Total records to grab: {total_records:,}") 

88 skip += limit_ 

89 except Exception as e: 

90 logging.error(e) 

91 raise Exception(f"Unable to process data: {response.content}") 

92 

93 for item in records: 

94 data.append(item) 

95 

96 logging.info(f"Grabbed {len(records):,} record(s) from page " 

97 f"{int(skip/limit_)}. Progress: " 

98 f"{len(data):,}/{total_records:,}") 

99 

100 elif response.status_code == 429: 

101 seconds_to_wait = 120 

102 logging.info(f"Too many requests made to API. " 

103 f"Waiting {seconds_to_wait/60:,} minute(s) before retrying.") 

104 time.sleep(seconds_to_wait) 

105 continue 

106 

107 else: 

108 raise Exception( 

109 f"Failed to get access group data from api. " 

110 f"Status: {response.status_code}") 

111 

112 return data 

113 

114 def get_data_from_endpoint(self, schema_name_, endpoint_name_, 

115 filter_=None, limit_=1000): 

116 """ 

117 Grabs data from API endpoint and returns a SDCDataframe object with 

118 data loaded into the dataframe if data is avaliable. 

119 

120 :param schema_name_: Schema file name for SDCDataframe instance.. 

121 :param endpoint_name_: Name of API URL endpoint. 

122 :param filter_: Filter to apply to API call. 

123 :return: SDCDataframe object with data in dataframe. 

124 """ 

125 

126 data_schema = json.loads(open(SDCFileHelpers.get_file_path( 

127 'schema', f"Ultipro/time_management/{schema_name_}.json")).read()) 

128 validation = SchemaValidation() 

129 validated_schema = validation.validate_schema(data_schema) 

130 validated_source_endpoint_schema = SchemaToolbox.get_endpoint_data_from_schema(validated_schema, "main_source") 

131 self.base_url = validated_source_endpoint_schema["info"]["access"]["base_url"] 

132 df = Dataframe(SDCDFTypes.PANDAS, validated_schema) 

133 

134 base_endpoint_url = self.base_url + f'/{endpoint_name_}' 

135 

136 data = self.process_endpoint(base_endpoint_url, filter_, limit_) 

137 

138 if len(data) >= 1: 

139 df.load_data(data) 

140 return df 

141 else: 

142 logging.warning("Received no data") 

143 return None