Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import io 

2import gzip 

3 

4from sdc_etl_libs.sdc_file_helpers.SDCJsonFileHelpers import SDCJsonFileHelpers 

5 

6def test_sdc_json_file_helpers_normalize_json_with_one_level(): 

7 """ 

8 Ensure loading of JSON with one level generate a SDCDataframe. 

9 """ 

10 json_string = '{"asof":"2020-01-20T18:34:19.858666+00:00", '\ 

11 '"events":[{"ship_date":"2020-06-05","tracking_id":"65177a1e234178d4504c07"},'\ 

12 ' {"ship_date":"2020-06-05","tracking_id":"65177a1e234178d4504c07"}]}' 

13 file_obj = io.BytesIO(gzip.compress(json_string.encode('utf-8'))) 

14 function_name = "normalize_json" 

15 opts= {'max_level': 1, 'record_path': "events"} 

16 

17 df = SDCJsonFileHelpers.apply_custom_pre_processing_function( 

18 file_obj_=gzip.open(file_obj), 

19 function_name_=function_name, 

20 opts_=opts) 

21 

22 assert df is not None 

23 assert len(df) == 2 

24 assert len(df.columns) == 2 

25 assert (list(df.columns.values)) == ['ship_date', 'tracking_id'] 

26 

27def test_sdc_json_file_helpers_normalize_json_with_one_level_uncompressed(): 

28 """ 

29 Ensure loading of JSON with one level generate a SDCDataframe. 

30 """ 

31 json_string = '{"asof":"2020-01-20T18:34:19.858666+00:00", '\ 

32 '"events":[{"ship_date":"2020-06-05","tracking_id":"65177a1e234178d4504c07"},'\ 

33 ' {"ship_date":"2020-06-05","tracking_id":"65177a1e234178d4504c07"}]}' 

34 file_obj = io.BytesIO(json_string.encode('utf-8')) 

35 function_name = "normalize_json" 

36 opts= {'max_level': 1, 'record_path': "events"} 

37 df = SDCJsonFileHelpers.apply_custom_pre_processing_function( 

38 file_obj_=file_obj, 

39 function_name_=function_name, 

40 opts_=opts) 

41 assert df is not None 

42 assert len(df) == 2 

43 assert len(df.columns) == 2 

44 assert (list(df.columns.values)) == ['ship_date', 'tracking_id']