Coverage for libs/sdc_etl_libs/sdc_file_helpers/TechnicalStandards/EDI/SDCEDIPandasConverter.py : 84%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import logging
2from ast import literal_eval
3from collections import namedtuple
5import pandas as pd
7from sdc_etl_libs.sdc_file_helpers.TechnicalStandards.EDI.SDCEDIConverter import \
8 SDCEDIConverter
9from sdc_etl_libs.sdc_file_helpers.TechnicalStandards.EDI.SDCEDIExceptions import \
10 EDITransactionNotFound
11from sdc_etl_libs.sdc_file_helpers.TechnicalStandards.EDI.SDCEDIObject import \
12 SDCEDIObject
15class SDCEDIPandasConverter(SDCEDIConverter):
17 DataFrame = pd.DataFrame
19 def __init__(self, edi_object_: SDCEDIObject, object_id_: str):
20 """
21 Converts between Pandas Dataframe and EDI
22 :param edi_object_: SDCEDIObject
23 :param object_id_: str
24 :return: SDCEDIPandasConverter
25 """
26 self.edi_object = edi_object_
27 self.object_id = object_id_
29 def convert_transactions_to_dataframe(self) -> DataFrame:
30 """
31 Creates a Pandas Dataframe
32 :return: DataFrame
33 """
35 def _aux_convert_transactions_to_dataframe(transaction_group):
36 """
37 Converts a list of edi segments to pandas dataframe
38 :param transaction_group: list(Segment)
39 :return: DataFrame
40 """
41 position, group = transaction_group
42 dataframe = pd.DataFrame(data=group)
43 dataframe['INVOICE_ID'] = position + 1
44 dataframe['EDI_TYPE'] = self.edi_object.get_edi_type()
45 return dataframe
47 # Get a list of EDI segments type ST (transactions)
48 transaction_groups = self.edi_object.get_transactions_groups()
49 # Converts each group to a dataframe
50 list_df_transactions = map(_aux_convert_transactions_to_dataframe, enumerate(transaction_groups))
51 # Make an unified dataframe containing all of the EDI transactions
52 df = pd.concat(list_df_transactions)
53 # Add column date
54 df = self._add_date_column_to_dataframe(df)
55 # Add column file id
56 df = self._add_file_id_column(df)
57 return df
59 def convert_headers_to_dataframe(self, headers) -> DataFrame:
60 """
61 Converts headers to dataframe
62 :param headers: list(Segment)
63 :return: DataFrame
64 """
65 # Build dataframe
66 df = pd.DataFrame(data=headers)
67 # Add column invoice id
68 # headers does not correspond to any invoice
69 df['INVOICE_ID'] = None
70 # Add column date
71 df = self._add_date_column_to_dataframe(df)
72 # Add column file id
73 df = self._add_file_id_column(df)
74 return df
76 def convert_request_headers_to_dataframe(self):
77 """
78 Converts request headers to dataframe
79 :return: DataFrame
80 """
81 df = self.convert_headers_to_dataframe(self.edi_object.get_headers())
82 # Add column edi type to mark the msg as edi 210
83 df['EDI_TYPE'] = self.edi_object.get_edi_type()
84 return df
86 def convert_response_headers_to_dataframe(self):
87 """
88 Converts response headers to dataframe
89 :return: DataFrame
90 """
91 df = self.convert_headers_to_dataframe(self.edi_object.get_ack_997_msg())
92 # Add column edi type to mark the msg as edi 997
93 df['EDI_TYPE'] = 997
94 return df
96 def _add_date_column_to_dataframe(self, df: DataFrame) -> DataFrame:
97 """
98 Add column the origin date of transmission
99 :return: DataFrame
100 """
101 # Add column
102 data_date_obj = self.edi_object.get_data_date()
103 df['DATA_DATE'] = data_date_obj
104 return df
106 def _add_file_id_column(self, df: DataFrame) -> DataFrame:
107 """
108 Add column file identifier
109 :return: DataFrame
110 """
111 # Add column file id
112 df['FILE_ID'] = self.object_id
113 return df
115 def convert_edi_object_to_dataframe(self):
116 """
117 Converts an edi object to a dataframe
118 :return: DataFrame
119 """
120 try:
121 # Converts transactions (invoices)
122 df_transactions = self.convert_transactions_to_dataframe()
123 # Converts response headers
124 df_response_headers = self.convert_response_headers_to_dataframe()
125 # Converts request headers
126 df_request_headers = self.convert_request_headers_to_dataframe()
127 # Combine dataframes into a single one
128 df = pd.concat([df_transactions, df_response_headers, df_request_headers], sort=True)
129 return df
130 except Exception as e:
131 raise EDITransactionNotFound("Failed loading EDI data to Pandas Dataframe")
133 @staticmethod
134 def convert_dataframe_to_edi_object(df):
135 """
136 Converts dataframe to edi_object
137 :param df: Pandas Dataframe
138 :return: str
139 """
140 # Segment definition using a Data Class
141 Segment = namedtuple('Segment', ['SEGMENT_ID', 'POSITION', 'DETAIL'])
143 # Functional decomposition
144 def aux_convert_dataframe_to_edi_object(segment_id):
145 """
146 Given a segment id, a Segment object is created
147 :param segment_id: Pandas Dataframe
148 :return: Segment
149 """
150 row = df[df['SEGMENT_ID'] == segment_id]
151 segment_id = row.iloc[0]['SEGMENT_ID']
152 position = row.iloc[0]['POSITION']
153 str_detail = row.iloc[0]['DETAIL']
154 detail = literal_eval(str_detail)
155 return Segment(SEGMENT_ID=segment_id, POSITION=position, DETAIL=detail)
157 logging.info("Converting Dataframe to EDI Object")
158 # Create a list of segments that constitute a EDI 997 message
159 list_segments = [
160 aux_convert_dataframe_to_edi_object('ISA'),
161 aux_convert_dataframe_to_edi_object('GS'),
162 aux_convert_dataframe_to_edi_object('ST'),
163 aux_convert_dataframe_to_edi_object('AK1'),
164 aux_convert_dataframe_to_edi_object('AK9'),
165 aux_convert_dataframe_to_edi_object('SE'),
166 aux_convert_dataframe_to_edi_object('GE'),
167 aux_convert_dataframe_to_edi_object('IEA')
168 ]
169 # Return edi object in string format
170 return SDCEDIObject.object_to_edi(list_segments)
172 @staticmethod
173 def get_empty_dataframe():
174 """
175 Generate an empty pandas dataframe
176 :return: DataFrame
177 """
178 columns = ['DATA_DATE', 'DETAIL', 'EDI_TYPE', 'FILE_ID', 'INVOICE_ID', 'POSITION', 'SEGMENT_ID']
179 return pd.DataFrame(data={}, columns=columns)