Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import logging 

2from ast import literal_eval 

3from collections import namedtuple 

4 

5import pandas as pd 

6 

7from sdc_etl_libs.sdc_file_helpers.TechnicalStandards.EDI.SDCEDIConverter import \ 

8 SDCEDIConverter 

9from sdc_etl_libs.sdc_file_helpers.TechnicalStandards.EDI.SDCEDIExceptions import \ 

10 EDITransactionNotFound 

11from sdc_etl_libs.sdc_file_helpers.TechnicalStandards.EDI.SDCEDIObject import \ 

12 SDCEDIObject 

13 

14 

15class SDCEDIPandasConverter(SDCEDIConverter): 

16 

17 DataFrame = pd.DataFrame 

18 

19 def __init__(self, edi_object_: SDCEDIObject, object_id_: str): 

20 """ 

21 Converts between Pandas Dataframe and EDI 

22 :param edi_object_: SDCEDIObject 

23 :param object_id_: str 

24 :return: SDCEDIPandasConverter 

25 """ 

26 self.edi_object = edi_object_ 

27 self.object_id = object_id_ 

28 

29 def convert_transactions_to_dataframe(self) -> DataFrame: 

30 """ 

31 Creates a Pandas Dataframe 

32 :return: DataFrame 

33 """ 

34 

35 def _aux_convert_transactions_to_dataframe(transaction_group): 

36 """ 

37 Converts a list of edi segments to pandas dataframe 

38 :param transaction_group: list(Segment) 

39 :return: DataFrame 

40 """ 

41 position, group = transaction_group 

42 dataframe = pd.DataFrame(data=group) 

43 dataframe['INVOICE_ID'] = position + 1 

44 dataframe['EDI_TYPE'] = self.edi_object.get_edi_type() 

45 return dataframe 

46 

47 # Get a list of EDI segments type ST (transactions) 

48 transaction_groups = self.edi_object.get_transactions_groups() 

49 # Converts each group to a dataframe 

50 list_df_transactions = map(_aux_convert_transactions_to_dataframe, enumerate(transaction_groups)) 

51 # Make an unified dataframe containing all of the EDI transactions 

52 df = pd.concat(list_df_transactions) 

53 # Add column date 

54 df = self._add_date_column_to_dataframe(df) 

55 # Add column file id 

56 df = self._add_file_id_column(df) 

57 return df 

58 

59 def convert_headers_to_dataframe(self, headers) -> DataFrame: 

60 """ 

61 Converts headers to dataframe 

62 :param headers: list(Segment) 

63 :return: DataFrame 

64 """ 

65 # Build dataframe 

66 df = pd.DataFrame(data=headers) 

67 # Add column invoice id 

68 # headers does not correspond to any invoice 

69 df['INVOICE_ID'] = None 

70 # Add column date 

71 df = self._add_date_column_to_dataframe(df) 

72 # Add column file id 

73 df = self._add_file_id_column(df) 

74 return df 

75 

76 def convert_request_headers_to_dataframe(self): 

77 """ 

78 Converts request headers to dataframe 

79 :return: DataFrame 

80 """ 

81 df = self.convert_headers_to_dataframe(self.edi_object.get_headers()) 

82 # Add column edi type to mark the msg as edi 210 

83 df['EDI_TYPE'] = self.edi_object.get_edi_type() 

84 return df 

85 

86 def convert_response_headers_to_dataframe(self): 

87 """ 

88 Converts response headers to dataframe 

89 :return: DataFrame 

90 """ 

91 df = self.convert_headers_to_dataframe(self.edi_object.get_ack_997_msg()) 

92 # Add column edi type to mark the msg as edi 997 

93 df['EDI_TYPE'] = 997 

94 return df 

95 

96 def _add_date_column_to_dataframe(self, df: DataFrame) -> DataFrame: 

97 """ 

98 Add column the origin date of transmission 

99 :return: DataFrame 

100 """ 

101 # Add column 

102 data_date_obj = self.edi_object.get_data_date() 

103 df['DATA_DATE'] = data_date_obj 

104 return df 

105 

106 def _add_file_id_column(self, df: DataFrame) -> DataFrame: 

107 """ 

108 Add column file identifier 

109 :return: DataFrame 

110 """ 

111 # Add column file id 

112 df['FILE_ID'] = self.object_id 

113 return df 

114 

115 def convert_edi_object_to_dataframe(self): 

116 """ 

117 Converts an edi object to a dataframe 

118 :return: DataFrame 

119 """ 

120 try: 

121 # Converts transactions (invoices) 

122 df_transactions = self.convert_transactions_to_dataframe() 

123 # Converts response headers 

124 df_response_headers = self.convert_response_headers_to_dataframe() 

125 # Converts request headers 

126 df_request_headers = self.convert_request_headers_to_dataframe() 

127 # Combine dataframes into a single one 

128 df = pd.concat([df_transactions, df_response_headers, df_request_headers], sort=True) 

129 return df 

130 except Exception as e: 

131 raise EDITransactionNotFound("Failed loading EDI data to Pandas Dataframe") 

132 

133 @staticmethod 

134 def convert_dataframe_to_edi_object(df): 

135 """ 

136 Converts dataframe to edi_object 

137 :param df: Pandas Dataframe 

138 :return: str 

139 """ 

140 # Segment definition using a Data Class 

141 Segment = namedtuple('Segment', ['SEGMENT_ID', 'POSITION', 'DETAIL']) 

142 

143 # Functional decomposition 

144 def aux_convert_dataframe_to_edi_object(segment_id): 

145 """ 

146 Given a segment id, a Segment object is created 

147 :param segment_id: Pandas Dataframe 

148 :return: Segment 

149 """ 

150 row = df[df['SEGMENT_ID'] == segment_id] 

151 segment_id = row.iloc[0]['SEGMENT_ID'] 

152 position = row.iloc[0]['POSITION'] 

153 str_detail = row.iloc[0]['DETAIL'] 

154 detail = literal_eval(str_detail) 

155 return Segment(SEGMENT_ID=segment_id, POSITION=position, DETAIL=detail) 

156 

157 logging.info("Converting Dataframe to EDI Object") 

158 # Create a list of segments that constitute a EDI 997 message 

159 list_segments = [ 

160 aux_convert_dataframe_to_edi_object('ISA'), 

161 aux_convert_dataframe_to_edi_object('GS'), 

162 aux_convert_dataframe_to_edi_object('ST'), 

163 aux_convert_dataframe_to_edi_object('AK1'), 

164 aux_convert_dataframe_to_edi_object('AK9'), 

165 aux_convert_dataframe_to_edi_object('SE'), 

166 aux_convert_dataframe_to_edi_object('GE'), 

167 aux_convert_dataframe_to_edi_object('IEA') 

168 ] 

169 # Return edi object in string format 

170 return SDCEDIObject.object_to_edi(list_segments) 

171 

172 @staticmethod 

173 def get_empty_dataframe(): 

174 """ 

175 Generate an empty pandas dataframe 

176 :return: DataFrame 

177 """ 

178 columns = ['DATA_DATE', 'DETAIL', 'EDI_TYPE', 'FILE_ID', 'INVOICE_ID', 'POSITION', 'SEGMENT_ID'] 

179 return pd.DataFrame(data={}, columns=columns)