Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import json 

2import os 

3import sys 

4 

5import pytest 

6 

7sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../../../../") 

8from sdc_etl_libs.sdc_data_schema.schema_exceptions import (DataSchemaCriticalError, DataSchemaFailedValidation) 

9from sdc_etl_libs.sdc_data_schema.schema_validation import (ResultTypeStatuses, SchemaValidation) 

10 

11def test_validation__key_data_type__correct_type(): 

12 """ 

13 Ensure that if a key has a set "data_type", that a given value or list of values passes if they match that type. 

14 #################################################################################################### 

15 # key_dt_1 = {"required": False, "data_type": str} # 

16 # key_dt_2 = {"required": False, "data_type": list, "list_value_opts": {"data_type": int}} # 

17 #################################################################################################### 

18 """ 

19 

20 # Test single value with the correct data type 

21 ep_schema_raw = """ 

22 { 

23 "type": "test", 

24 "tag": "main_source", 

25 "info": { 

26 "type": "test", 

27 "access": { 

28 "type": "test_data_type", 

29 "test_details": { 

30 "key_dt_1": "dog" 

31 } 

32 }, 

33 "opts": { 

34 } 

35 } 

36 } 

37 """ 

38 

39 ep_schema = json.loads(ep_schema_raw) 

40 validation = SchemaValidation() 

41 schema = validation.validate_schema(ep_schema, validation_type_="endpoint") 

42 

43 assert validation.ttl_errors == 0 

44 assert { 

45 'status': 'ERROR', 

46 'reason': 'BAD_DATA_TYPE', 

47 'section': 'endpoint:info:access:test_details', 

48 'note': '"key_dt_1" is not of type <class \'str\'>.' 

49 } not in validation.results 

50 

51 # Test a list of values with the correct data types 

52 ep_schema_raw = """ 

53 { 

54 "type": "test", 

55 "tag": "main_source", 

56 "info": { 

57 "type": "test", 

58 "access": { 

59 "type": "test_data_type", 

60 "test_details": { 

61 "key_dt_1": "dog", 

62 "key_dt_2": [1, 2, 3] 

63 } 

64 }, 

65 "opts": { 

66 } 

67 } 

68 } 

69 """ 

70 

71 ep_schema = json.loads(ep_schema_raw) 

72 validation = SchemaValidation() 

73 schema = validation.validate_schema(ep_schema, validation_type_="endpoint") 

74 

75 assert validation.ttl_errors == 0 

76 assert { 

77 'status': 'ERROR', 

78 'reason': 'BAD_DATA_TYPE', 

79 'section': 'endpoint:info:access:test_details', 

80 'note': '"1" within list "key_dt_2" is not of type <class \'int\'>.' 

81 } not in validation.results 

82 assert { 

83 'status': 'ERROR', 

84 'reason': 'BAD_DATA_TYPE', 

85 'section': 'endpoint:info:access:test_details', 

86 'note': '"2" within list "key_dt_2" is not of type <class \'int\'>.' 

87 } not in validation.results 

88 assert { 

89 'status': 'ERROR', 

90 'reason': 'BAD_DATA_TYPE', 

91 'section': 'endpoint:info:access:test_details', 

92 'note': '"3" within list "key_dt_2" is not of type <class \'int\'>.' 

93 } not in validation.results 

94 

95 

96def test_validation__key_data_type__incorrect_type(): 

97 """ 

98 Ensure that if a key has a set "data_type" and that value is not that type, an ERROR is recorded and 

99 DataSchemaFailedValidation is raised. If "data_type" is list, checks if value is of the "data_type" from "list_value_opts". 

100 #################################################################################################### 

101 # key_dt_1 = {"required": False, "data_type": str} # 

102 # key_dt_2 = {"required": False, "data_type": list, "list_value_opts": {"data_type": int}} # 

103 #################################################################################################### 

104 """ 

105 

106 # Test single value with an incorrect data type 

107 ep_schema_raw = """ 

108 { 

109 "type": "test", 

110 "tag": "main_source", 

111 "info": { 

112 "type": "test", 

113 "access": { 

114 "type": "test_data_type", 

115 "test_details": { 

116 "key_dt_1": 1 

117 } 

118 }, 

119 "opts": { 

120 } 

121 } 

122 } 

123 """ 

124 

125 ep_schema = json.loads(ep_schema_raw) 

126 validation = SchemaValidation() 

127 

128 with pytest.raises(DataSchemaFailedValidation): 

129 schema = validation.validate_schema(ep_schema, validation_type_="endpoint") 

130 

131 assert { 

132 'status': 'ERROR', 

133 'reason': 'BAD_DATA_TYPE', 

134 'section': 'endpoint:info:access:test_details', 

135 'note': '"key_dt_1" is not of type <class \'str\'>.' 

136 } in validation.results 

137 

138 # Test a list that includes a value with an incorrect data type 

139 ep_schema_raw = """ 

140 { 

141 "type": "test", 

142 "tag": "main_source", 

143 "info": { 

144 "type": "test", 

145 "access": { 

146 "type": "test_data_type", 

147 "test_details": { 

148 "key_dt_1": "dog", 

149 "key_dt_2": [1, 2, "dog"] 

150 } 

151 }, 

152 "opts": { 

153 } 

154 } 

155 } 

156 """ 

157 

158 ep_schema = json.loads(ep_schema_raw) 

159 validation = SchemaValidation() 

160 

161 with pytest.raises(DataSchemaFailedValidation): 

162 schema = validation.validate_schema(ep_schema, validation_type_="endpoint") 

163 

164 assert { 

165 'status': 'ERROR', 

166 'reason': 'BAD_DATA_TYPE', 

167 'section': 'endpoint:info:access:test_details', 

168 'note': '"dog" within list "key_dt_2" is not of type <class \'int\'>.' 

169 } in validation.results 

170 

171 

172def test_validation__key_data_type__default_to_string_if_not_specified(): 

173 """ 

174 Ensure that if a key does not have a "data_type" set, that it defaults to a type of str. If "data_type" is "list" 

175 and enum_key["list_value_opts"]["data_type"] is not set, then the list value data type defaults to str. 

176 #################################################################################################### 

177 # key_dt_6 = {"required": False, "optional": True} # 

178 # key_dt_7 = {"required": False, "optional": True, "data_type": list} # 

179 #################################################################################################### 

180 """ 

181 

182 # Test single value when "data_type" is undefined 

183 ep_schema_raw = """ 

184 { 

185 "type": "test", 

186 "tag": "main_source", 

187 "info": { 

188 "type": "test", 

189 "access": { 

190 "type": "test_data_type", 

191 "test_details": { 

192 "key_dt_6": "should-be-string" 

193 } 

194 }, 

195 "opts": { 

196 } 

197 } 

198 } 

199 """ 

200 

201 ep_schema = json.loads(ep_schema_raw) 

202 

203 validation = SchemaValidation() 

204 schema = validation.validate_schema(ep_schema, validation_type_="endpoint") 

205 

206 assert isinstance(schema["info"]["access"]["test_details"]["key_dt_6"], str) 

207 

208 # Test list when enum_key["list_value_opts"]["data_type"] is undefined 

209 ep_schema_raw = """ 

210 { 

211 "type": "test", 

212 "tag": "main_source", 

213 "info": { 

214 "type": "test", 

215 "access": { 

216 "type": "test_data_type", 

217 "test_details": { 

218 "key_dt_7": ["should", "be", "string"] 

219 } 

220 }, 

221 "opts": { 

222 } 

223 } 

224 } 

225 """ 

226 

227 ep_schema = json.loads(ep_schema_raw) 

228 

229 validation = SchemaValidation() 

230 schema = validation.validate_schema(ep_schema, validation_type_="endpoint") 

231 

232 for value in schema["info"]["access"]["test_details"]["key_dt_7"]: 

233 assert isinstance(value, str) 

234 

235 

236 

237 

238 

239def test_validation__key_data_type__asterisk_allows_all_data_types(): 

240 """ 

241 Ensure that if a key has a set "data_type" or enum_key["list_value_opts"]["data_type"] of "*", that a value of any 

242 data type is allowed. 

243 #################################################################################################### 

244 # key_dt_4 = {"required": ["test|test2"], "data_type": "*"} # 

245 # key_dt_5 = {"required": True, "data_type": list, "list_value_opts": {"data_type": "*"}} # 

246 #################################################################################################### 

247 """ 

248 

249 # Test a single Int value 

250 ep_schema_raw = """ 

251 { 

252 "type": "test", 

253 "tag": "main_source", 

254 "info": { 

255 "type": "test", 

256 "access": { 

257 "type": "test_data_type", 

258 "test_details": { 

259 "key_dt_4": 1 

260 } 

261 }, 

262 "opts": { 

263 } 

264 } 

265 } 

266 """ 

267 

268 ep_schema = json.loads(ep_schema_raw) 

269 

270 validation = SchemaValidation() 

271 schema = validation.validate_schema(ep_schema, validation_type_="endpoint") 

272 

273 assert schema["info"]["access"]["test_details"]["key_dt_4"] == 1 

274 

275 # Test a single Boolean value 

276 ep_schema_raw = """ 

277 { 

278 "type": "test", 

279 "tag": "main_source", 

280 "info": { 

281 "type": "test", 

282 "access": { 

283 "type": "test_data_type", 

284 "test_details": { 

285 "key_dt_4": true 

286 } 

287 }, 

288 "opts": { 

289 } 

290 } 

291 } 

292 """ 

293 

294 ep_schema = json.loads(ep_schema_raw) 

295 

296 validation = SchemaValidation() 

297 schema = validation.validate_schema(ep_schema, validation_type_="endpoint") 

298 

299 assert schema["info"]["access"]["test_details"]["key_dt_4"] == True 

300 

301 # Test a single String value 

302 ep_schema_raw = """ 

303 { 

304 "type": "test", 

305 "tag": "main_source", 

306 "info": { 

307 "type": "test", 

308 "access": { 

309 "type": "test_data_type", 

310 "test_details": { 

311 "key_dt_4": "this-is-a-string" 

312 } 

313 }, 

314 "opts": { 

315 } 

316 } 

317 } 

318 """ 

319 

320 ep_schema = json.loads(ep_schema_raw) 

321 

322 validation = SchemaValidation() 

323 schema = validation.validate_schema(ep_schema, validation_type_="endpoint") 

324 

325 assert schema["info"]["access"]["test_details"]["key_dt_4"] == "this-is-a-string" 

326 

327 # Test a single Null value 

328 ep_schema_raw = """ 

329 { 

330 "type": "test", 

331 "tag": "main_source", 

332 "info": { 

333 "type": "test", 

334 "access": { 

335 "type": "test_data_type", 

336 "test_details": { 

337 "key_dt_4": null 

338 } 

339 }, 

340 "opts": { 

341 } 

342 } 

343 } 

344 """ 

345 

346 ep_schema = json.loads(ep_schema_raw) 

347 

348 validation = SchemaValidation() 

349 schema = validation.validate_schema(ep_schema, validation_type_="endpoint") 

350 

351 assert schema["info"]["access"]["test_details"]["key_dt_4"] == None 

352 

353 # Test list of values 

354 ep_schema_raw = """ 

355 { 

356 "type": "test", 

357 "tag": "main_source", 

358 "info": { 

359 "type": "test", 

360 "access": { 

361 "type": "test_data_type", 

362 "test_details": { 

363 "key_dt_5": [1, "a", null] 

364 } 

365 }, 

366 "opts": { 

367 } 

368 } 

369 } 

370 """ 

371 

372 ep_schema = json.loads(ep_schema_raw) 

373 

374 validation = SchemaValidation() 

375 schema = validation.validate_schema(ep_schema, validation_type_="endpoint") 

376 

377 assert schema["info"]["access"]["test_details"]["key_dt_5"] == [1, "a", None] 

378 

379 

380def test_validation__key_allowed_values__list__critical_error(): 

381 """ 

382 Ensure that a DataSchemaCriticalError is raised if a key has "list_value_opts" when "data_type" is not "list". 

383 ################################################################################################################### 

384 # key_dt_3 = {"required": True, "data_type": str, "list_value_opts": {"data_type": str}} # 

385 ################################################################################################################### 

386 """ 

387 

388 ep_schema_raw = """ 

389 { 

390 "type": "test", 

391 "tag": "main_source", 

392 "info": { 

393 "type": "test", 

394 "access": { 

395 "type": "test_data_type", 

396 "test_details": { 

397 "key_dt_3": "value" 

398 } 

399 }, 

400 "opts": { 

401 } 

402 } 

403 } 

404 """ 

405 

406 ep_schema = json.loads(ep_schema_raw) 

407 validation = SchemaValidation() 

408 

409 with pytest.raises(DataSchemaCriticalError): 

410 schema = validation.validate_schema(ep_schema, validation_type_="endpoint")