Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1 

2import pytest 

3from enum import Enum 

4import sdc_etl_libs 

5from sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLogger import SDCSnowflakeLogger 

6 

7 

8def test_successful_logger_setup_from_logger_section_of_schema(mocker): 

9 """ 

10 Logger is initialized with endpoint schema's logger section. 

11 Credentials from AWS and actual connection to Snowflake mocked. 

12 """ 

13 

14 mocker.patch('sdc_etl_libs.aws_helpers.aws_helpers.AWSHelpers.get_secrets', return_value={ 

15 "username": None, 

16 "account": None, 

17 "warehouse": None, 

18 "role": None, 

19 "password": None 

20 }) 

21 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.connect', return_value=True) 

22 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query', return_value=True) 

23 

24 logger = SDCSnowflakeLogger({ 

25 "type": "snowflake", 

26 "opts": { 

27 "environment": "development", 

28 "processing": None 

29 }, 

30 "access": { 

31 "credentials": { 

32 "type": "aws_secrets", 

33 "opts": { 

34 "name": "snowflake/service_account/logger" 

35 } 

36 }, 

37 } 

38 }) 

39 

40 assert logger != None 

41 

42 return logger 

43 

44 

45def test_logger_setup_from_logger_section_of_schema_fails_on_missing_parameter(mocker): 

46 """ 

47 Logger intilization should fail due to a missing mandatory "environment" parameter. 

48 Credentials from AWS and actual connection to Snowflake mocked. 

49 """ 

50 

51 mocker.patch('sdc_etl_libs.aws_helpers.aws_helpers.AWSHelpers.get_secrets', return_value={ 

52 "username": None, 

53 "account": None, 

54 "warehouse": None, 

55 "role": None, 

56 "password": None 

57 }) 

58 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.connect', return_value=True) 

59 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query', return_value=True) 

60 

61 with pytest.raises(Exception): 

62 logger = SDCSnowflakeLogger({ 

63 "type": "snowflake", 

64 "opts": { 

65 "processing": None 

66 }, 

67 "access": { 

68 "credentials": { 

69 "type": "aws_secrets", 

70 "opts": { 

71 "name": "snowflake/service_account/logger" 

72 } 

73 }, 

74 } 

75 }) 

76 

77 

78def test_uuid_generation_stays_same(mocker): 

79 """ 

80 The UUID generated from UUID Generation func needs to stay the same under different dictionary pass methods. 

81 """ 

82 

83 logger = test_successful_logger_setup_from_logger_section_of_schema(mocker) 

84 assert logger != None 

85 

86 # Test perfect setup 

87 data_for_uuid = { 

88 "ACCOUNT": "sd7777", 

89 "DATABASE": "Logistics", 

90 "SCHEMA": "Fedex", 

91 "TABLE_NAME": "tracking_events" 

92 } 

93 uuid = logger.generate_uuid(data_for_uuid) 

94 assert uuid == 'c798f7c03f77787941151175a56a71ac' 

95 

96 # Test different case types in values 

97 data_for_uuid = { 

98 "ACCOUNT": "sd7777", 

99 "DATABASE": "LOGISTICS", 

100 "SCHEMA": "FEDEX", 

101 "TABLE_NAME": "tracking_events" 

102 } 

103 uuid = logger.generate_uuid(data_for_uuid) 

104 assert uuid == 'c798f7c03f77787941151175a56a71ac' 

105 

106 # Test keys in different order 

107 data_for_uuid = { 

108 "DATABASE": "LOGISTICS", 

109 "TABLE_NAME": "tracking_events", 

110 "ACCOUNT": "sd7777", 

111 "SCHEMA": "FEDEX" 

112 } 

113 uuid = logger.generate_uuid(data_for_uuid) 

114 assert uuid == 'c798f7c03f77787941151175a56a71ac' 

115 

116 # Test differ case type keys 

117 data_for_uuid = { 

118 "database": "LOGISTICS", 

119 "TABLE_name": "tracking_events", 

120 "account": "sd7777", 

121 "SCHEMA": "FEDEX" 

122 } 

123 uuid = logger.generate_uuid(data_for_uuid) 

124 assert uuid == 'c798f7c03f77787941151175a56a71ac' 

125 

126 # Test all possible variations 

127 data_for_uuid = { 

128 "account": "sd7777 ", 

129 "SCHEMA": "FEDEX ", 

130 "DATABASE": "LOGISTICs", 

131 "TABLE_name": " tracking_events " 

132 } 

133 uuid = logger.generate_uuid(data_for_uuid) 

134 assert uuid == 'c798f7c03f77787941151175a56a71ac' 

135 

136 

137def test_logging_environment_setup(mocker): 

138 """ 

139 Running the logging environment setup should: 

140 - Trigger execute_query to build the tables. 

141 """ 

142 

143 logger = test_successful_logger_setup_from_logger_section_of_schema(mocker) 

144 #logger.schema_path = '"DATAENGINEERING"."MARKETING"' 

145 assert logger != None 

146 

147 #TODO: Build out tests further to asssert sql code generated 

148 #class EventsSchema(Enum): 

149 # ID = {"NAME": "ID", "TYPE": "INT", "CONSTRAINTS": "IDENTITY PRIMARY KEY"} 

150 # DATA_ITEM_UUID = {"NAME": "DATA_ITEM_UUID", "TYPE": "VARCHAR", "CONSTRAINTS": "NOT NULL", 

151 # "REQUIRED_ON_INSERT": True, "REQUIRED_FOR_UUID": True} 

152 # EVENT_DATE = {"NAME": "EVENT_DATE", "TYPE": "DATETIME", "CONSTRAINTS": "NOT NULL", "REQUIRED_ON_INSERT": True, 

153 # "REQUIRED_FOR_UUID": True} 

154 

155 #class LoggingTables(Enum): 

156 # EVENTS_TABLE = {"TABLE_NAME": "LOGGING_EXCHANGE_EVENTS", "DATA_SCHEMA": EventsSchema} 

157 

158 #mocker.patch('sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLoggerEnums.LoggingTables', 

159 # return_value=LoggingTables, autospec=True) 

160 #mocker.patch('sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLoggerEnums.EventsSchema', 

161 # return_value=EventsSchema, autospec=True) 

162 

163 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query') 

164 logger.setup_logging_environment() 

165 sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query.assert_called() 

166 

167def test_generate_endpoint_uuid(mocker): 

168 """ 

169 Endpoint UUID should be generated consistently and should only be written to database table is explicitly 

170 instructed to do so. 

171 """ 

172 

173 logger = test_successful_logger_setup_from_logger_section_of_schema(mocker) 

174 assert logger != None 

175 

176 logger.endpoint_detail_table_mapping["S3"]["TABLE_NAME"] = "S3" 

177 logger.required_for_uuid["S3"] = ['BUCKET', 'PREFIX'] 

178 

179 endpoint_schema = { 

180 "type": "sink", 

181 "tag": "S3_new_dev_sink", 

182 "info": { 

183 "type": "s3", 

184 "access": { 

185 "bucket": "sdc-marketing-vendors", 

186 "prefix": "dataxu/", 

187 "region": "us-east-2" 

188 }, 

189 "opts": {}, 

190 "file_info": { 

191 "type": "csv", 

192 "delimiter": ",", 

193 "headers": True, 

194 "file_regex": ".*" 

195 }, 

196 "logger": { 

197 "type": "snowflake", 

198 "opts": { 

199 "environment": "development", 

200 "access": { 

201 "credentials": { 

202 "type": "aws_secrets", 

203 "opts": { 

204 "name": "snowflake/service_account/logger" 

205 } 

206 } 

207 } 

208 } 

209 } 

210 } 

211 } 

212 

213 # Assert UUId is generated as expected. 

214 uuid = logger.generate_endpoint_uuid(endpoint_schema, log_=False) 

215 assert uuid == '32d7b2618ae0910e8bae4d75162a974d' 

216 

217 # Assert that log_=True results in update_base_table being called. 

218 mocker.patch('sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLogger.SDCSnowflakeLogger') 

219 uuid = logger.generate_endpoint_uuid(endpoint_schema, log_=True) 

220 assert uuid == '32d7b2618ae0910e8bae4d75162a974d' 

221 uuid_ident = {'BUCKET': 'sdc-marketing-vendors', 'PREFIX': 'dataxu/', 'UUID': '32d7b2618ae0910e8bae4d75162a974d'} 

222 sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLogger.SDCSnowflakeLogger.update_base_table.\ 

223 asset_called_once_with(endpoint_schema, uuid_ident) 

224 

225 

226def test_insert_into_writes_table(mocker): 

227 """ 

228 Inserting into Writes Table should: 

229 - Succeed only when all required columns/values are provided 

230 - Fail if EVENT_UUID is not passed in write_stats 

231 - Fail if the TYPE given does not exist 

232 - Fail if any column/values are missing from the individual write records 

233 - Not call execute_query if no write stats exist 

234 """ 

235 

236 logger = test_successful_logger_setup_from_logger_section_of_schema(mocker) 

237 assert logger != None 

238 

239 class SnowflakeTableSchema(Enum): 

240 ID = {"NAME": "ID", "TYPE": "INT", "CONSTRAINTS": "IDENTITY PRIMARY KEY"} 

241 TABLE_NAME = {"NAME": "TABLE_NAME", "TYPE": "VARCHAR", "CONSTRAINTS": "NOT NULL", "REQUIRED_ON_INSERT": True, 

242 "REQUIRED_FOR_UUID": True} 

243 WRITE_DATE = {"NAME": "WRITE_DATE", "TYPE": "DATETIME", "CONSTRAINTS": "NOT NULL", "REQUIRED_ON_INSERT": True, 

244 "REQUIRED_FOR_UUID": True} 

245 EVENT_UUID = {"NAME": "EVENT_UUID", "TYPE": "VARCHAR", "CONSTRAINTS": "NOT NULL", "REQUIRED_ON_INSERT": True, 

246 "REQUIRED_FOR_UUID": True} 

247 UUID = {"NAME": "UUID", "TYPE": "VARCHAR", "CONSTRAINTS": ""} 

248 

249 logger.endpoint_write_table_mapping["SNOWFLAKE"]["TABLE_NAME"] = "SNOWFLAKE_TABLE" 

250 logger.endpoint_write_table_mapping["SNOWFLAKE"]["DATA_SCHEMA"] = SnowflakeTableSchema 

251 logger.required_on_insert["SNOWFLAKE_TABLE"] = ["TABLE_NAME", "WRITE_DATE"] 

252 

253 # Succeed and call execute_query to write to table 

254 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query') 

255 logger.insert_into_writes_table( 

256 { 

257 'TYPE': 'SNOWFLAKE', 

258 'EVENT_UUID': '68cf82f7eafc95a56106bc9b61ec8a2b', 

259 'STATS': [ 

260 {'TABLE_NAME': 'ORDERS_TEMP', 'WRITE_DATE': 'TEMP'}, 

261 {'TABLE_NAME': 'ORDERS_TEMP_DEDUPED', 'WRITE_DATE': '2019-10-01'}, 

262 {'TABLE_NAME': 'ORDERS', 'WRITE_DATE': '2019-10-01'} 

263 ] 

264 } 

265 ) 

266 sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query.assert_called() 

267 

268 # Fail if EVENT_UUID is missing 

269 with pytest.raises(Exception): 

270 logger.insert_into_writes_table( 

271 { 

272 'TYPE': 'SNOWFLAKE', 

273 'STATS': [ 

274 {'TABLE_NAME': 'ORDERS_TEMP', 'WRITE_DATE': 'TEMP'}, 

275 {'TABLE_NAME': 'ORDERS_TEMP_DEDUPED', 'WRITE_DATE': '2019-10-01'}, 

276 {'TABLE_NAME': 'ORDERS', 'WRITE_DATE': '2019-10-01'} 

277 ] 

278 } 

279 ) 

280 

281 # Fail if TYPE given does not exist 

282 with pytest.raises(Exception): 

283 logger.insert_into_writes_table( 

284 { 

285 'TYPE': 'STREAM', 

286 'EVENT_UUID': '68cf82f7eafc95a56106bc9b61ec8a2b', 

287 'STATS': [ 

288 {'TABLE_NAME': 'ORDERS_TEMP', 'WRITE_DATE': 'TEMP'}, 

289 {'TABLE_NAME': 'ORDERS_TEMP_DEDUPED', 'WRITE_DATE': '2019-10-01'}, 

290 {'TABLE_NAME': 'ORDERS', 'WRITE_DATE': '2019-10-01'} 

291 ] 

292 } 

293 ) 

294 

295 # Fail if any write stat records are missing required items 

296 with pytest.raises(Exception): 

297 logger.insert_into_writes_table( 

298 { 

299 'TYPE': 'SNOWFLAKE', 

300 'EVENT_UUID': '68cf82f7eafc95a56106bc9b61ec8a2b', 

301 'STATS': [ 

302 {'TABLE_NAME': 'ORDERS_TEMP'}, 

303 {'TABLE_NAME': 'ORDERS_TEMP_DEDUPED', 'WRITE_DATE': '2019-10-01'}, 

304 {'WRITE_DATE': '2019-10-01'} 

305 ] 

306 } 

307 ) 

308 

309 # Assert that EXECUTE_QUERY is not called when STATS portion of write_status is empty 

310 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query') 

311 logger.insert_into_writes_table( 

312 { 

313 'TYPE': 'SNOWFLAKE', 

314 'EVENT_UUID': '68cf82f7eafc95a56106bc9b61ec8a2b', 

315 'STATS': [] 

316 } 

317 ) 

318 sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query.assert_not_called() 

319 

320 

321def test_insert_into_events_table(mocker): 

322 """ 

323 Inserting into the Events Table should: 

324 - Call execute_query 

325 - Return a String (event uuid) 

326 """ 

327 

328 logger = test_successful_logger_setup_from_logger_section_of_schema(mocker) 

329 

330 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query') 

331 event_uuid = logger.insert_into_events_table({ 

332 "UUID": "d3d2af251a51a805b0d8c51d73a9b105", 

333 "TOTAL_RECORDS_WRITTEN": 15000, 

334 "CURRENT_EXCHANGE_STATUS": "PROCESSING", 

335 "CURRENT_EXCHANGE_REASON": None, 

336 "EXCHANGE_MESSAGE": "Processing..." 

337 }) 

338 

339 # Assert execute_query was called 

340 sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query.assert_called() 

341 

342 # Assert return value is a list 

343 assert isinstance(event_uuid, str) 

344 

345 

346 

347def test_update_base_table_with_write_stats(mocker): 

348 """ 

349 Updating the base table, with included write stats, should: 

350 - Trigger execute_query to update the base table 

351 - Trigger insert_into_events_table to update the events table 

352 - Trigger insert_into_writes table to update the associated writes table 

353 """ 

354 

355 logger = test_successful_logger_setup_from_logger_section_of_schema(mocker) 

356 assert logger != None 

357 

358 class BaseTableSchema(Enum): 

359 ID = {"NAME": "ID", "TYPE": "INT", "CONSTRAINTS": "IDENTITY PRIMARY KEY"} 

360 TABLE_NAME = {"NAME": "TABLE_NAME", "TYPE": "VARCHAR", "CONSTRAINTS": "NOT NULL", "REQUIRED_ON_INSERT": True, "REQUIRED_FOR_UUID": True} 

361 WRITE_DATE = {"NAME": "WRITE_DATE", "TYPE": "DATETIME", "CONSTRAINTS": "NOT NULL", "REQUIRED_ON_INSERT": True, "REQUIRED_FOR_UUID": True} 

362 UUID = {"NAME": "UUID", "TYPE": "VARCHAR", "CONSTRAINTS": ""} 

363 

364 mocker.patch('sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLoggerEnums.BaseTableSchema', 

365 return_value=BaseTableSchema, autospec=True) 

366 mocker.patch('sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLogger.SDCSnowflakeLogger.insert_into_events_table') 

367 mocker.patch('sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLogger.SDCSnowflakeLogger.insert_into_writes_table') 

368 

369 logger.update_base_table({ 

370 "UUID": "88afc0ef995984df47025eb0f9e99cbf", 

371 "TOTAL_RECORDS_WRITTEN": 15000, 

372 "CURRENT_EXCHANGE_STATUS": "PROCESSING", 

373 "CURRENT_EXCHANGE_REASON": None, 

374 "EXCHANGE_MESSAGE": "Processed another 15,000 records.", 

375 "WRITE_STATS": { 

376 'TYPE': 'SNOWFLAKE', 

377 'EVENT_UUID': '68cf82f7eafc95a56106bc9b61ec8a2b', 

378 'STATS': [ 

379 {'TABLE_NAME': 'ORDERS_TEMP'}, 

380 {'TABLE_NAME': 'ORDERS_TEMP_DEDUPED', 'WRITE_DATE': '2019-10-01'}, 

381 {'WRITE_DATE': '2019-10-01'} 

382 ] 

383 } 

384 }) 

385 

386 # Assert that execute_query was called to write to Base Table 

387 sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query.assert_called() 

388 

389 # Assert that insert_into_events_table was called to write to Events Table 

390 sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLogger.SDCSnowflakeLogger.insert_into_events_table.assert_called() 

391 

392 # Assert that insert_into_writes_table was called to write to Writes Table 

393 sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLogger.SDCSnowflakeLogger.insert_into_writes_table.assert_called() 

394 

395 

396def test_update_base_table_without_write_stats(mocker): 

397 """ 

398 Updating the base table, with included write stats, should: 

399 - Trigger execute_query to update the base table 

400 - Trigger insert_into_events_table to update the events table 

401 - NOT trigger insert_into_writes 

402 """ 

403 

404 logger = test_successful_logger_setup_from_logger_section_of_schema(mocker) 

405 assert logger != None 

406 

407 class BaseTableSchema(Enum): 

408 ID = {"NAME": "ID", "TYPE": "INT", "CONSTRAINTS": "IDENTITY PRIMARY KEY"} 

409 TABLE_NAME = {"NAME": "TABLE_NAME", "TYPE": "VARCHAR", "CONSTRAINTS": "NOT NULL", "REQUIRED_ON_INSERT": True, "REQUIRED_FOR_UUID": True} 

410 WRITE_DATE = {"NAME": "WRITE_DATE", "TYPE": "DATETIME", "CONSTRAINTS": "NOT NULL", "REQUIRED_ON_INSERT": True, "REQUIRED_FOR_UUID": True} 

411 UUID = {"NAME": "UUID", "TYPE": "VARCHAR", "CONSTRAINTS": ""} 

412 

413 mocker.patch('sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLoggerEnums.BaseTableSchema', 

414 return_value=BaseTableSchema, autospec=True) 

415 mocker.patch('sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLogger.SDCSnowflakeLogger.insert_into_writes_table') 

416 

417 logger.update_base_table({ 

418 "UUID": "88afc0ef995984df47025eb0f9e99cbf", 

419 "TOTAL_RECORDS_WRITTEN": 15000, 

420 "CURRENT_EXCHANGE_STATUS": "PROCESSING", 

421 "CURRENT_EXCHANGE_REASON": None, 

422 "EXCHANGE_MESSAGE": "Processed another 15,000 records." 

423 }) 

424 

425 # Assert that insert_into_writes_table was not called since write_stats was not given 

426 sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLogger.SDCSnowflakeLogger.insert_into_writes_table.assert_not_called() 

427 

428def test_retrieve_successful_runs(mocker): 

429 """ 

430 Retrieving successful runs should: 

431 - Trigger execute_query to run the SELECT statement 

432 - Return a List (data item names) 

433 """ 

434 

435 logger = test_successful_logger_setup_from_logger_section_of_schema(mocker) 

436 assert logger != None 

437 

438 data_sink_endpoint_uuid = "x2040" 

439 

440 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query', return_value=True) 

441 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.get_results', 

442 return_value=[("data_item_name.csv")]) 

443 

444 data_items = logger.retrieve_successful_runs(data_sink_endpoint_uuid, empty_is_success_=False) 

445 # Assert return value is a list 

446 assert isinstance(data_items, list) 

447 

448 # Assert execute_query was called 

449 sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query.assert_called() 

450 

451 

452def test_retrieve_failed_runs(mocker): 

453 """ 

454 Retrieving failed runs should: 

455 - Trigger execute_query to run the SELECT statement 

456 - Return a Dict (data item names with stats) 

457 """ 

458 

459 logger = test_successful_logger_setup_from_logger_section_of_schema(mocker) 

460 assert logger != None 

461 

462 data_sink_endpoint_uuid = "x2040" 

463 min_till_abandoned = 90 

464 

465 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query', return_value=True) 

466 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.get_results', 

467 return_value=[("data_item_name", "dataItemUUIDx", 200)]) 

468 mocker.patch('sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLogger.SDCSnowflakeLogger.handle_abandoned_runs') 

469 

470 data_items = logger.retrieve_failed_runs(data_sink_endpoint_uuid, handle_abandoned_runs_=False, mins_until_abandoned_=min_till_abandoned) 

471 # Assert return value is a dict 

472 assert isinstance(data_items, dict) 

473 # Assert execute_query was called 

474 sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query.assert_called() 

475 

476 # Assert handle_abandoned_runs runs if handle_abandoned_runs_=True 

477 logger.retrieve_failed_runs(data_sink_endpoint_uuid, handle_abandoned_runs_=True, mins_until_abandoned_=min_till_abandoned) 

478 sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLogger.SDCSnowflakeLogger.handle_abandoned_runs. \ 

479 asset_called_once_with(data_sink_endpoint_uuid, min_till_abandoned) 

480 

481 

482def test_retrieve_non_abandoned_runs(mocker): 

483 """ 

484 Retrieving non abandoned runs should: 

485 - Trigger execute_query to run the SELECT statement 

486 - Return a List (data item names) 

487 """ 

488 logger = test_successful_logger_setup_from_logger_section_of_schema(mocker) 

489 assert logger != None 

490 

491 data_sink_endpoint_uuid = "x2040" 

492 min_till_abandoned = 90 

493 

494 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query', return_value=True) 

495 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.get_results', 

496 return_value=[("data_item_name.csv")]) 

497 

498 data_items = logger.retrieve_non_abandoned_runs(data_sink_endpoint_uuid, mins_until_abandoned_=min_till_abandoned) 

499 # Assert return value is a list 

500 assert isinstance(data_items, list) 

501 

502 # Assert execute_query was called 

503 sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query.assert_called() 

504 

505 

506def test_handle_abandoned_runs(mocker): 

507 """ 

508 Retrieving non abandoned runs should: 

509 - Trigger execute_query to run the SELECT statement 

510 - Trigger update_base_table to update the items found 

511 """ 

512 

513 logger = test_successful_logger_setup_from_logger_section_of_schema(mocker) 

514 assert logger != None 

515 

516 data_sink_endpoint_uuid = "x2040" 

517 min_till_abandoned = 90 

518 

519 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query', return_value=True) 

520 mocker.patch('sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.get_results', 

521 return_value=[("dataItemUUID2", "data_item_name.csv")]) 

522 mocker.patch('sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLogger.SDCSnowflakeLogger.update_base_table') 

523 

524 logger.handle_abandoned_runs(data_sink_endpoint_uuid, min_till_abandoned) 

525 

526 # Assert execute_query was called 

527 sdc_etl_libs.database_helpers.SnowflakeDatabase.SnowflakeDatabase.execute_query.assert_called() 

528 # Assert update_base_table was called at least once 

529 sdc_etl_libs.sdc_data_exchange_loggers.SDCSnowflakeLogger.SDCSnowflakeLogger.SDCSnowflakeLogger.update_base_table. \ 

530 assert_called()