Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import pandas as pd 

2from paramiko import SFTPAttributes 

3 

4from sdc_etl_libs.sdc_exchange_endpoints.database_endpoints.SDCSnowflakeEndpoint import SDCSnowflakeEndpoint 

5from sdc_etl_libs.sdc_filetransfer.utils.delete_files_mechanism.ExistsInTableDeleteCriteria import \ 

6 ExistsInTableDeleteCriteria 

7 

8 

9def test_get_files_to_delete_all_files(mocker): 

10 # endpoint config 

11 endpoint = SDCSnowflakeEndpoint() 

12 endpoint.query = "query" 

13 endpoint.database = "database" 

14 

15 # snowflake files 

16 data = ['first_file', 'second_file'] 

17 df = pd.DataFrame(data, columns=['filename']) 

18 mocker.patch('pandas.read_sql', return_value=df) 

19 

20 # sftp files 

21 first_ftp_file = SFTPAttributes() 

22 first_ftp_file.filename = "first_file" 

23 second_ftp_file = SFTPAttributes() 

24 second_ftp_file.filename = "second_file" 

25 files = [first_ftp_file, second_ftp_file] 

26 

27 delete_criteria = ExistsInTableDeleteCriteria(endpoint) 

28 assert len(delete_criteria.get_files_to_delete(files)) == 2 

29 

30 

31def test_get_files_to_delete_one_file(mocker): 

32 # endpoint config 

33 endpoint = SDCSnowflakeEndpoint() 

34 endpoint.query = "query" 

35 endpoint.database = "database" 

36 

37 # snowflake files 

38 data = ['first_', 'second_file', 'third_'] 

39 df = pd.DataFrame(data, columns=['filename']) 

40 mocker.patch('pandas.read_sql', return_value=df) 

41 

42 # sftp files 

43 first_ftp_file = SFTPAttributes() 

44 first_ftp_file.filename = "first_file" 

45 second_ftp_file = SFTPAttributes() 

46 second_ftp_file.filename = "second_file" 

47 files = [first_ftp_file, second_ftp_file] 

48 

49 delete_criteria = ExistsInTableDeleteCriteria(endpoint) 

50 assert len(delete_criteria.get_files_to_delete(files)) == 1 

51 

52 

53def test_get_files_to_delete_no_files(mocker): 

54 # endpoint config 

55 endpoint = SDCSnowflakeEndpoint() 

56 endpoint.query = "query" 

57 endpoint.database = "database" 

58 

59 # snowflake files 

60 data = ['file_', 'file_2', 'file_3'] 

61 df = pd.DataFrame(data, columns=['filename']) 

62 mocker.patch('pandas.read_sql', return_value=df) 

63 

64 # sftp files 

65 first_ftp_file = SFTPAttributes() 

66 first_ftp_file.filename = "first_file" 

67 second_ftp_file = SFTPAttributes() 

68 second_ftp_file.filename = "second_file" 

69 files = [first_ftp_file, second_ftp_file] 

70 

71 delete_criteria = ExistsInTableDeleteCriteria(endpoint) 

72 assert len(delete_criteria.get_files_to_delete(files)) == 0