Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import logging 

2from datetime import datetime 

3 

4from dateutil.parser import parse 

5 

6from sdc_etl_libs.sdc_dataframe.Dataframe import Dataframe 

7from sdc_etl_libs.sdc_file_helpers.SDCFile import SDCFile 

8from sdc_etl_libs.sdc_file_helpers.TechnicalStandards.EDI.SDCEDIConverterFactory import \ 

9 SDCEDIConverterFactory 

10from sdc_etl_libs.sdc_file_helpers.TechnicalStandards.EDI.SDCEDIObject import \ 

11 SDCEDIObject 

12from sdc_etl_libs.sdc_file_helpers.TechnicalStandards.EDI.SDCEDIPandasConverter import \ 

13 SDCEDIPandasConverter 

14 

15 

16class SDCEDIFile(SDCFile): 

17 type = None 

18 file_name = None 

19 file_path = None 

20 file_obj = None 

21 schema = None 

22 endpoint_schema = None 

23 df = None 

24 endpoint_type = None 

25 

26 def __init__(self, schema_, endpoint_schema_, file_name_, file_path_, file_obj_, **kwargs): 

27 """ 

28 Creates an SDCEDIFile object representing a EDI file. 

29 :param schema_: Json schema of the data. 

30 :param endpoint_schema_: The endpoint schema of the source 

31 :param file_name_: Name of the file 

32 :param file_path_: Path you wish to write too 

33 :param file_obj_: Raw file object 

34 :return: SDCEDIFile 

35 """ 

36 super(SDCEDIFile, self).__init__(schema_, endpoint_schema_, file_name_, file_path_, file_obj_) 

37 

38 if self.args is None or self.args == {}: 

39 raise Exception("Missing EDI Args.") 

40 # Load optional attributes 

41 self.field_separator = self.args.get('field_separator') 

42 self.segment_separator = self.args.get('segment_separator') 

43 self.dataframe_type = self.args.get('dataframe_type') 

44 # Create python object from edi text file 

45 logging.info("Prefetching EDI file") 

46 self.edi_file_obj = SDCEDIObject( 

47 file_obj_, field_separator_=self.field_separator, segment_separator_=self.segment_separator) 

48 # Determine the type of Converter to use 

49 self.edi_dataframe_converter = SDCEDIConverterFactory.get_edi_converter(self.dataframe_type, self.edi_file_obj, 

50 self.file_name) 

51 

52 def get_file_as_dataframe(self): 

53 """ 

54 Converts a edi file object to an SDCDataframe. 

55 :return: A fully processed SDCDataframe. 

56 """ 

57 try: 

58 self.df = Dataframe(None, self.schema) 

59 logging.info("Converting EDI file to Object") 

60 pandas_df = self.edi_dataframe_converter.convert_edi_object_to_dataframe() 

61 logging.info("Converting EDI text to Object DONE") 

62 logging.info("Converting EDI Object to Dataframe") 

63 self.df.process_df(pandas_df) 

64 logging.info("Converting EDI Object to Dataframe DONE") 

65 return self.df 

66 

67 except Exception as e: 

68 logging.error(f"Failed loading EDI data to SDCDataframe. {e}") 

69 # Return a empty dataframe wheh an error happens 

70 pandas_df = SDCEDIPandasConverter.get_empty_dataframe() 

71 self.df.process_df(pandas_df) 

72 return self.df 

73 

74 def get_file_as_object(self): 

75 return self.file_obj