Coverage for src/bioimageio/core/_resource_tests.py: 69%

511 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 09:46 +0000

1from __future__ import annotations 

2 

3import hashlib 

4import os 

5import platform 

6import subprocess 

7import sys 

8import warnings 

9from collections.abc import Callable, Hashable, Sequence 

10from contextlib import nullcontext 

11from copy import deepcopy 

12from io import StringIO 

13from itertools import product 

14from pathlib import Path 

15from tempfile import TemporaryDirectory 

16from typing import ( 

17 Any, 

18 Literal, 

19 get_args, 

20 overload, 

21) 

22 

23import numpy as np 

24import xarray as xr 

25from loguru import logger 

26from numpy.typing import NDArray 

27from typing_extensions import NotRequired, TypedDict, Unpack, assert_never 

28 

29from bioimageio.spec import ( 

30 AnyDatasetDescr, 

31 AnyModelDescr, 

32 BioimageioCondaEnv, 

33 DatasetDescr, 

34 InvalidDescr, 

35 LatestResourceDescr, 

36 ModelDescr, 

37 ResourceDescr, 

38 ValidationContext, 

39 build_description, 

40 dump_description, 

41 get_conda_env, 

42 load_description, 

43 save_bioimageio_package, 

44) 

45from bioimageio.spec._description_impl import DISCOVER 

46from bioimageio.spec._internal.common_nodes import ResourceDescrBase 

47from bioimageio.spec._internal.io import is_yaml_value 

48from bioimageio.spec._internal.io_utils import read_yaml, write_yaml 

49from bioimageio.spec._internal.types import ( 

50 AbsoluteTolerance, 

51 FormatVersionPlaceholder, 

52 MismatchedElementsPerMillion, 

53 RelativeTolerance, 

54) 

55from bioimageio.spec._internal.validation_context import get_validation_context 

56from bioimageio.spec._internal.warning_levels import INFO, WARNING, WarningSeverity 

57from bioimageio.spec.common import BioimageioYamlContent, PermissiveFileSource, Sha256 

58from bioimageio.spec.model import v0_4, v0_5 

59from bioimageio.spec.model.v0_5 import WeightsFormat 

60from bioimageio.spec.summary import ( 

61 ErrorEntry, 

62 InstalledPackage, 

63 ValidationDetail, 

64 ValidationSummary, 

65 WarningEntry, 

66) 

67 

68from . import __version__ 

69from ._prediction_pipeline import create_prediction_pipeline 

70from ._settings import settings 

71from .axis import BATCH_AXIS_ID, AxisId, BatchSize 

72from .common import MemberId, SupportedWeightsFormat 

73from .digest_spec import ( 

74 get_input_halo, 

75 get_member_id, 

76 get_test_input_sample, 

77 get_test_output_sample, 

78) 

79from .io import save_tensor 

80from .sample import Sample 

81from .tensor import Tensor 

82 

83CONDA_CMD = "conda.bat" if platform.system() == "Windows" else "conda" 

84 

85 

86class DeprecatedKwargs(TypedDict): 

87 absolute_tolerance: NotRequired[AbsoluteTolerance] 

88 relative_tolerance: NotRequired[RelativeTolerance] 

89 decimal: NotRequired[int | None] 

90 

91 

92def enable_determinism( 

93 mode: Literal["seed_only", "full"] = "full", 

94 weight_formats: Sequence[SupportedWeightsFormat] | None = None, 

95): 

96 """Seed and configure ML frameworks for maximum reproducibility. 

97 May degrade performance. Only recommended for testing reproducibility! 

98 

99 Seed any random generators and (if **mode**=="full") request ML frameworks to use 

100 deterministic algorithms. 

101 

102 Args: 

103 mode: determinism mode 

104 - 'seed_only' -- only set seeds, or 

105 - 'full' determinsm features (might degrade performance or throw exceptions) 

106 weight_formats: Limit deep learning importing deep learning frameworks 

107 based on weight_formats. 

108 E.g. this allows to avoid importing tensorflow when testing with pytorch. 

109 

110 Notes: 

111 - **mode** == "full" might degrade performance or throw exceptions. 

112 - Subsequent inference calls might still differ. Call before each function 

113 (sequence) that is expected to be reproducible. 

114 - Degraded performance: Use for testing reproducibility only! 

115 - Recipes: 

116 - [PyTorch](https://pytorch.org/docs/stable/notes/randomness.html) 

117 - [Keras](https://keras.io/examples/keras_recipes/reproducibility_recipes/) 

118 - [NumPy](https://numpy.org/doc/2.0/reference/random/generated/numpy.random.seed.html) 

119 """ 

120 try: 

121 try: 

122 import numpy.random 

123 except ImportError: 

124 pass 

125 else: 

126 numpy.random.seed(0) 

127 except Exception as e: 

128 logger.debug(str(e)) 

129 

130 if ( 

131 weight_formats is None 

132 or "pytorch_state_dict" in weight_formats 

133 or "torchscript" in weight_formats 

134 ): 

135 try: 

136 try: 

137 import torch 

138 except ImportError: 

139 pass 

140 else: 

141 _ = torch.manual_seed(0) 

142 torch.use_deterministic_algorithms(mode == "full") 

143 except Exception as e: 

144 logger.debug(str(e)) 

145 

146 if ( 

147 weight_formats is None 

148 or "tensorflow_saved_model_bundle" in weight_formats 

149 or "keras_hdf5" in weight_formats 

150 ): 

151 try: 

152 os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0" 

153 try: 

154 import tensorflow as tf 

155 except ImportError: 

156 pass 

157 else: 

158 tf.random.set_seed(0) 

159 if mode == "full": 

160 tf.config.experimental.enable_op_determinism() 

161 # TODO: find possibility to switch it off again?? 

162 except Exception as e: 

163 logger.debug(str(e)) 

164 

165 if weight_formats is None or "keras_hdf5" in weight_formats: 

166 try: 

167 try: 

168 import keras # pyright: ignore[reportMissingTypeStubs] 

169 except ImportError: 

170 pass 

171 else: 

172 keras.utils.set_random_seed(0) 

173 except Exception as e: 

174 logger.debug(str(e)) 

175 

176 

177def test_model( 

178 source: v0_4.ModelDescr | v0_5.ModelDescr | PermissiveFileSource, 

179 weight_format: SupportedWeightsFormat | None = None, 

180 devices: list[str] | None = None, 

181 *, 

182 determinism: Literal["seed_only", "full"] = "seed_only", 

183 sha256: Sha256 | None = None, 

184 stop_early: bool = False, 

185 working_dir: os.PathLike[str] | str | None = None, 

186 **deprecated: Unpack[DeprecatedKwargs], 

187) -> ValidationSummary: 

188 """Test model inference""" 

189 return test_description( 

190 source, 

191 weight_format=weight_format, 

192 devices=devices, 

193 determinism=determinism, 

194 expected_type="model", 

195 sha256=sha256, 

196 stop_early=stop_early, 

197 working_dir=working_dir, 

198 **deprecated, 

199 ) 

200 

201 

202def default_run_command(args: Sequence[str]): 

203 logger.info("running '{}'...", " ".join(args)) 

204 _ = subprocess.check_call(args) 

205 

206 

207def test_description( 

208 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent, 

209 *, 

210 format_version: FormatVersionPlaceholder | str = "discover", 

211 weight_format: SupportedWeightsFormat | None = None, 

212 devices: Sequence[str] | None = None, 

213 determinism: Literal["seed_only", "full"] = "seed_only", 

214 expected_type: str | None = None, 

215 sha256: Sha256 | None = None, 

216 stop_early: bool = False, 

217 runtime_env: Literal["currently-active", "as-described"] 

218 | Path 

219 | BioimageioCondaEnv = ("currently-active"), 

220 run_command: Callable[[Sequence[str]], None] = default_run_command, 

221 working_dir: os.PathLike[str] | str | None = None, 

222 **deprecated: Unpack[DeprecatedKwargs], 

223) -> ValidationSummary: 

224 """Test a bioimage.io resource dynamically, 

225 for example run prediction of test tensors for models. 

226 

227 Args: 

228 source: model description source. 

229 weight_format: Weight format to test. 

230 Default: All weight formats present in **source**. 

231 devices: Devices to test with, e.g. 'cpu', 'cuda'. 

232 Default (may be weight format dependent): ['cuda'] if available, ['cpu'] otherwise. 

233 determinism: Modes to improve reproducibility of test outputs. 

234 expected_type: Assert an expected resource description `type`. 

235 sha256: Expected SHA256 value of **source**. 

236 (Ignored if **source** already is a loaded `ResourceDescr` object.) 

237 stop_early: Do not run further subtests after a failed one. 

238 runtime_env: (Experimental feature!) The Python environment to run the tests in 

239 - `"currently-active"`: Use active Python interpreter. 

240 - `"as-described"`: Use `bioimageio.spec.get_conda_env` to generate a conda 

241 environment YAML file based on the model weights description. 

242 - A `BioimageioCondaEnv` or a path to a conda environment YAML file. 

243 Note: The `bioimageio.core` dependency will be added automatically if not present. 

244 run_command: (Experimental feature!) Function to execute (conda) terminal commands in a subprocess. 

245 The function should raise an exception if the command fails. 

246 **run_command** is ignored if **runtime_env** is `"currently-active"`. 

247 working_dir: (for debugging) directory to save any temporary files 

248 (model packages, conda environments, test summaries). 

249 Defaults to a temporary directory. 

250 """ 

251 if runtime_env == "currently-active": 

252 rd = load_description_and_test( 

253 source, 

254 format_version=format_version, 

255 weight_format=weight_format, 

256 devices=devices, 

257 determinism=determinism, 

258 expected_type=expected_type, 

259 sha256=sha256, 

260 stop_early=stop_early, 

261 working_dir=working_dir, 

262 **deprecated, 

263 ) 

264 return rd.validation_summary 

265 

266 if runtime_env == "as-described": 266 ↛ 268line 266 didn't jump to line 268 because the condition on line 266 was always true

267 conda_env = None 

268 elif isinstance(runtime_env, (str, Path)): 

269 conda_env = BioimageioCondaEnv.model_validate(read_yaml(Path(runtime_env))) 

270 elif isinstance(runtime_env, BioimageioCondaEnv): 

271 conda_env = runtime_env 

272 else: 

273 assert_never(runtime_env) 

274 

275 if run_command is not default_run_command: 275 ↛ 276line 275 didn't jump to line 276 because the condition on line 275 was never true

276 try: 

277 run_command(["thiscommandshouldalwaysfail", "please"]) 

278 except Exception as e: 

279 logger.trace("run_command failed successfully :) {}", e) 

280 else: 

281 raise RuntimeError( 

282 "given run_command does not raise an exception for a failing command" 

283 ) 

284 

285 verbose = working_dir is not None 

286 if working_dir is None: 286 ↛ 292line 286 didn't jump to line 292 because the condition on line 286 was always true

287 td_kwargs: dict[str, Any] = ( 

288 {"ignore_cleanup_errors": True} if sys.version_info >= (3, 10) else {} 

289 ) 

290 working_dir_ctxt = TemporaryDirectory(**td_kwargs) 

291 else: 

292 working_dir_ctxt = nullcontext(working_dir) 

293 

294 with working_dir_ctxt as _d: 

295 working_dir = Path(_d) 

296 

297 if isinstance(source, ResourceDescrBase): 297 ↛ 298line 297 didn't jump to line 298 because the condition on line 297 was never true

298 descr = source 

299 elif isinstance(source, dict): 299 ↛ 300line 299 didn't jump to line 300 because the condition on line 299 was never true

300 context = get_validation_context().replace( 

301 perform_io_checks=True # make sure we perform io checks though 

302 ) 

303 

304 descr = build_description(source, context=context) 

305 else: 

306 descr = load_description(source, perform_io_checks=True) 

307 

308 if isinstance(descr, InvalidDescr): 308 ↛ 309line 308 didn't jump to line 309 because the condition on line 308 was never true

309 return descr.validation_summary 

310 elif isinstance(source, (dict, ResourceDescrBase)): 310 ↛ 311line 310 didn't jump to line 311 because the condition on line 310 was never true

311 file_source = save_bioimageio_package( 

312 descr, output_path=working_dir / "package.zip" 

313 ) 

314 else: 

315 file_source = source 

316 

317 # elevate status valid-format to passed and start testing 

318 descr.validation_summary.status = "passed" 

319 try: 

320 _test_in_env( 

321 file_source, 

322 descr=descr, 

323 working_dir=working_dir, 

324 weight_format=weight_format, 

325 conda_env=conda_env, 

326 devices=devices, 

327 determinism=determinism, 

328 expected_type=expected_type, 

329 sha256=sha256, 

330 stop_early=stop_early, 

331 run_command=run_command, 

332 verbose=verbose, 

333 **deprecated, 

334 ) 

335 except Exception as e: 

336 descr.validation_summary.add_detail( 

337 ValidationDetail( 

338 name="Test in dedicated environment", 

339 status="failed", 

340 loc=(), 

341 errors=[ 

342 ErrorEntry( 

343 loc=(), 

344 msg=str(e), 

345 type="bioimageio.core", 

346 with_traceback=True, 

347 ) 

348 ], 

349 ) 

350 ) 

351 

352 return descr.validation_summary 

353 

354 

355def _test_in_env( 

356 source: PermissiveFileSource, 

357 *, 

358 descr: ResourceDescr, 

359 working_dir: Path, 

360 weight_format: SupportedWeightsFormat | None, 

361 conda_env: BioimageioCondaEnv | None, 

362 devices: Sequence[str] | None, 

363 determinism: Literal["seed_only", "full"], 

364 run_command: Callable[[Sequence[str]], None], 

365 stop_early: bool, 

366 expected_type: str | None, 

367 sha256: Sha256 | None, 

368 verbose: bool, 

369 **deprecated: Unpack[DeprecatedKwargs], 

370): 

371 """Test a bioimage.io resource in a given conda environment. 

372 Adds details to the existing validation summary of **descr**. 

373 """ 

374 if isinstance(descr, (v0_4.ModelDescr, v0_5.ModelDescr)): 374 ↛ 435line 374 didn't jump to line 435 because the condition on line 374 was always true

375 if weight_format is None: 

376 # run tests for all present weight formats 

377 all_present_wfs = [ 

378 wf for wf in get_args(WeightsFormat) if getattr(descr.weights, wf, None) 

379 ] 

380 ignore_wfs = [wf for wf in all_present_wfs if wf in ["tensorflow_js"]] 

381 logger.info( 

382 "Found weight formats {}. Start testing all{}...", 

383 all_present_wfs, 

384 f" (except: {', '.join(ignore_wfs)}) " if ignore_wfs else "", 

385 ) 

386 for wf in all_present_wfs: 386 ↛ 403line 386 didn't jump to line 403 because the loop on line 386 didn't complete

387 _test_in_env( 

388 source, 

389 descr=descr, 

390 working_dir=working_dir / wf, 

391 weight_format=wf, 

392 devices=devices, 

393 determinism=determinism, 

394 conda_env=conda_env, 

395 run_command=run_command, 

396 expected_type=expected_type, 

397 sha256=sha256, 

398 stop_early=stop_early, 

399 verbose=verbose, 

400 **deprecated, 

401 ) 

402 

403 return 

404 

405 if weight_format == "pytorch_state_dict": 405 ↛ 406line 405 didn't jump to line 406 because the condition on line 405 was never true

406 wf = descr.weights.pytorch_state_dict 

407 elif weight_format == "torchscript": 407 ↛ 408line 407 didn't jump to line 408 because the condition on line 407 was never true

408 wf = descr.weights.torchscript 

409 elif weight_format == "keras_hdf5": 409 ↛ 410line 409 didn't jump to line 410 because the condition on line 409 was never true

410 wf = descr.weights.keras_hdf5 

411 elif weight_format == "onnx": 411 ↛ 413line 411 didn't jump to line 413 because the condition on line 411 was always true

412 wf = descr.weights.onnx 

413 elif weight_format == "tensorflow_saved_model_bundle": 

414 wf = descr.weights.tensorflow_saved_model_bundle 

415 elif weight_format == "keras_v3": 

416 if isinstance(descr, v0_4.ModelDescr): 

417 raise ValueError( 

418 "Weight format 'keras_v3' is not supported in v0.4 model descriptions. use format version >= 0.5" 

419 ) 

420 

421 wf = descr.weights.keras_v3 

422 elif weight_format == "tensorflow_js": 

423 raise RuntimeError( 

424 "testing 'tensorflow_js' is not supported by bioimageio.core" 

425 ) 

426 else: 

427 assert_never(weight_format) 

428 

429 assert wf is not None 

430 if conda_env is None: 430 ↛ 433line 430 didn't jump to line 433 because the condition on line 430 was always true

431 conda_env = get_conda_env(entry=wf) 

432 

433 test_loc = ("weights", weight_format) 

434 else: 

435 if conda_env is None: 

436 warnings.warn( 

437 "No conda environment description given for testing (And no default conda envs available for non-model descriptions)." 

438 ) 

439 return 

440 

441 test_loc = () 

442 

443 # remove name as we create a name based on the env description hash value 

444 conda_env.name = None 

445 

446 dumped_env = conda_env.model_dump(mode="json", exclude_none=True) 

447 if not is_yaml_value(dumped_env): 447 ↛ 448line 447 didn't jump to line 448 because the condition on line 447 was never true

448 raise ValueError(f"Failed to dump conda env to valid YAML {conda_env}") 

449 

450 env_io = StringIO() 

451 write_yaml(dumped_env, file=env_io) 

452 encoded_env = env_io.getvalue().encode() 

453 env_name = hashlib.sha256(encoded_env).hexdigest() 

454 

455 try: 

456 run_command(["where" if platform.system() == "Windows" else "which", CONDA_CMD]) 

457 except Exception as e: 

458 raise RuntimeError("Conda not available") from e 

459 

460 try: 

461 run_command([CONDA_CMD, "run", "-n", env_name, "python", "--version"]) 

462 except Exception: 

463 working_dir.mkdir(parents=True, exist_ok=True) 

464 path = working_dir / "env.yaml" 

465 try: 

466 _ = path.write_bytes(encoded_env) 

467 logger.debug("written conda env to {}", path) 

468 run_command( 

469 [ 

470 CONDA_CMD, 

471 "env", 

472 "create", 

473 "--yes", 

474 f"--file={path}", 

475 f"--name={env_name}", 

476 ] 

477 + (["--quiet"] if settings.CI else []) 

478 ) 

479 # double check that environment was created successfully 

480 run_command([CONDA_CMD, "run", "-n", env_name, "python", "--version"]) 

481 except Exception as e: 

482 descr.validation_summary.add_detail( 

483 ValidationDetail( 

484 name="Conda environment creation", 

485 status="failed", 

486 loc=test_loc, 

487 recommended_env=conda_env, 

488 errors=[ 

489 ErrorEntry( 

490 loc=test_loc, 

491 msg=str(e), 

492 type="conda", 

493 with_traceback=True, 

494 ) 

495 ], 

496 ) 

497 ) 

498 return 

499 else: 

500 descr.validation_summary.add_detail( 

501 ValidationDetail( 

502 name=f"Created conda environment '{env_name}'", 

503 status="passed", 

504 loc=test_loc, 

505 ) 

506 ) 

507 else: 

508 descr.validation_summary.add_detail( 

509 ValidationDetail( 

510 name=f"Found existing conda environment '{env_name}'", 

511 status="passed", 

512 loc=test_loc, 

513 ) 

514 ) 

515 

516 working_dir.mkdir(parents=True, exist_ok=True) 

517 summary_path = working_dir / "summary.json" 

518 assert not summary_path.exists(), "Summary file already exists" 

519 cmd = [] 

520 cmd_error = None 

521 for summary_path_arg_name in ("summary", "summary-path"): 

522 try: 

523 run_command( 

524 cmd := ( 

525 [ 

526 CONDA_CMD, 

527 "run", 

528 "-n", 

529 env_name, 

530 "bioimageio", 

531 "test", 

532 str(source), 

533 f"--{summary_path_arg_name}={summary_path.as_posix()}", 

534 f"--determinism={determinism}", 

535 ] 

536 + ([f"--weight-format={weight_format}"] if weight_format else []) 

537 + ([f"--expected-type={expected_type}"] if expected_type else []) 

538 + (["--stop-early"] if stop_early else []) 

539 ) 

540 ) 

541 except Exception as e: 

542 cmd_error = f"Command '{' '.join(cmd)}' returned with error: {e}." 

543 

544 if summary_path.exists(): 

545 break 

546 else: 

547 if cmd_error is not None: 

548 logger.warning(cmd_error) 

549 

550 descr.validation_summary.add_detail( 

551 ValidationDetail( 

552 name="run 'bioimageio test' command", 

553 recommended_env=conda_env, 

554 errors=[ 

555 ErrorEntry( 

556 loc=(), 

557 type="bioimageio cli", 

558 msg=f"test command '{' '.join(cmd)}' did not produce a summary file at {summary_path}", 

559 ) 

560 ], 

561 status="failed", 

562 ) 

563 ) 

564 return 

565 

566 # add relevant details from command summary 

567 command_summary = ValidationSummary.load_json(summary_path) 

568 for detail in command_summary.details: 

569 if detail.loc[: len(test_loc)] == test_loc or detail.status == "failed": 

570 descr.validation_summary.add_detail(detail) 

571 

572 

573@overload 

574def load_description_and_test( 

575 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent, 

576 *, 

577 format_version: Literal["latest"], 

578 weight_format: SupportedWeightsFormat | None = None, 

579 devices: Sequence[str] | None = None, 

580 determinism: Literal["seed_only", "full"] = "seed_only", 

581 expected_type: Literal["model"], 

582 sha256: Sha256 | None = None, 

583 stop_early: bool = False, 

584 working_dir: os.PathLike[str] | str | None = None, 

585 **deprecated: Unpack[DeprecatedKwargs], 

586) -> ModelDescr | InvalidDescr: ... 

587 

588 

589@overload 

590def load_description_and_test( 

591 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent, 

592 *, 

593 format_version: Literal["latest"], 

594 weight_format: SupportedWeightsFormat | None = None, 

595 devices: Sequence[str] | None = None, 

596 determinism: Literal["seed_only", "full"] = "seed_only", 

597 expected_type: Literal["dataset"], 

598 sha256: Sha256 | None = None, 

599 stop_early: bool = False, 

600 working_dir: os.PathLike[str] | str | None = None, 

601 **deprecated: Unpack[DeprecatedKwargs], 

602) -> DatasetDescr | InvalidDescr: ... 

603 

604 

605@overload 

606def load_description_and_test( 

607 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent, 

608 *, 

609 format_version: Literal["latest"], 

610 weight_format: SupportedWeightsFormat | None = None, 

611 devices: Sequence[str] | None = None, 

612 determinism: Literal["seed_only", "full"] = "seed_only", 

613 expected_type: str | None = None, 

614 sha256: Sha256 | None = None, 

615 stop_early: bool = False, 

616 working_dir: os.PathLike[str] | str | None = None, 

617 **deprecated: Unpack[DeprecatedKwargs], 

618) -> LatestResourceDescr | InvalidDescr: ... 

619 

620 

621@overload 

622def load_description_and_test( 

623 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent, 

624 *, 

625 format_version: FormatVersionPlaceholder | str = DISCOVER, 

626 weight_format: SupportedWeightsFormat | None = None, 

627 devices: Sequence[str] | None = None, 

628 determinism: Literal["seed_only", "full"] = "seed_only", 

629 expected_type: Literal["model"], 

630 sha256: Sha256 | None = None, 

631 stop_early: bool = False, 

632 working_dir: os.PathLike[str] | str | None = None, 

633 **deprecated: Unpack[DeprecatedKwargs], 

634) -> AnyModelDescr | InvalidDescr: ... 

635 

636 

637@overload 

638def load_description_and_test( 

639 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent, 

640 *, 

641 format_version: FormatVersionPlaceholder | str = DISCOVER, 

642 weight_format: SupportedWeightsFormat | None = None, 

643 devices: Sequence[str] | None = None, 

644 determinism: Literal["seed_only", "full"] = "seed_only", 

645 expected_type: Literal["dataset"], 

646 sha256: Sha256 | None = None, 

647 stop_early: bool = False, 

648 working_dir: os.PathLike[str] | str | None = None, 

649 **deprecated: Unpack[DeprecatedKwargs], 

650) -> AnyDatasetDescr | InvalidDescr: ... 

651 

652 

653@overload 

654def load_description_and_test( 

655 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent, 

656 *, 

657 format_version: FormatVersionPlaceholder | str = DISCOVER, 

658 weight_format: SupportedWeightsFormat | None = None, 

659 devices: Sequence[str] | None = None, 

660 determinism: Literal["seed_only", "full"] = "seed_only", 

661 expected_type: str | None = None, 

662 sha256: Sha256 | None = None, 

663 stop_early: bool = False, 

664 working_dir: os.PathLike[str] | str | None = None, 

665 **deprecated: Unpack[DeprecatedKwargs], 

666) -> ResourceDescr | InvalidDescr: ... 

667 

668 

669def load_description_and_test( 

670 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent, 

671 *, 

672 format_version: FormatVersionPlaceholder | str = DISCOVER, 

673 weight_format: SupportedWeightsFormat | None = None, 

674 devices: Sequence[str] | None = None, 

675 determinism: Literal["seed_only", "full"] = "seed_only", 

676 expected_type: str | None = None, 

677 sha256: Sha256 | None = None, 

678 stop_early: bool = False, 

679 working_dir: os.PathLike[str] | str | None = None, 

680 **deprecated: Unpack[DeprecatedKwargs], 

681) -> ResourceDescr | InvalidDescr: 

682 """Test a bioimage.io resource dynamically, 

683 for example run prediction of test tensors for models. 

684 

685 See `test_description` for more details. 

686 

687 Returns: 

688 A (possibly invalid) resource description object 

689 with a populated `.validation_summary` attribute. 

690 """ 

691 if isinstance(source, ResourceDescrBase): 

692 root = source.root 

693 file_name = source.file_name 

694 if ( 

695 ( 

696 format_version 

697 not in ( 

698 DISCOVER, 

699 source.format_version, 

700 ".".join(source.format_version.split(".")[:2]), 

701 ) 

702 ) 

703 or (c := source.validation_summary.details[0].context) is None 

704 or not c.perform_io_checks 

705 ): 

706 logger.debug( 

707 "deserializing source to ensure we validate and test using format {} and perform io checks", 

708 format_version, 

709 ) 

710 source = dump_description(source) 

711 else: 

712 root = Path() 

713 file_name = None 

714 

715 if isinstance(source, ResourceDescrBase): 

716 rd = source 

717 elif isinstance(source, dict): 

718 # check context for a given root; default to root of source 

719 context = get_validation_context( 

720 ValidationContext(root=root, file_name=file_name) 

721 ).replace( 

722 perform_io_checks=True # make sure we perform io checks though 

723 ) 

724 

725 rd = build_description( 

726 source, 

727 format_version=format_version, 

728 context=context, 

729 ) 

730 else: 

731 rd = load_description( 

732 source, format_version=format_version, sha256=sha256, perform_io_checks=True 

733 ) 

734 

735 rd.validation_summary.env.add( 

736 InstalledPackage(name="bioimageio.core", version=__version__) 

737 ) 

738 

739 if expected_type is not None: 

740 has_expected_type = _test_expected_resource_type(rd, expected_type) 

741 if not has_expected_type: 741 ↛ 743line 741 didn't jump to line 743 because the condition on line 741 was never true

742 # unexpected type -> invalid format 

743 rd.validation_summary.status = "failed" 

744 return rd 

745 

746 # elevate status valid-format to passed and start testing 

747 if rd.validation_summary.status == "valid-format": 747 ↛ 750line 747 didn't jump to line 750 because the condition on line 747 was always true

748 rd.validation_summary.status = "passed" 

749 

750 if isinstance(rd, (v0_4.ModelDescr, v0_5.ModelDescr)): 750 ↛ 783line 750 didn't jump to line 783 because the condition on line 750 was always true

751 if weight_format is None: 

752 weight_formats: list[SupportedWeightsFormat] = [ 

753 w for w, we in rd.weights if we is not None 

754 ] # pyright: ignore[reportAssignmentType] 

755 else: 

756 weight_formats = [weight_format] 

757 

758 enable_determinism(determinism, weight_formats=weight_formats) 

759 for w in weight_formats: 

760 passed_recreate_test_outputs = _test_recreate_test_outputs( 

761 rd, 

762 w, 

763 devices, 

764 stop_early=stop_early, 

765 working_dir=working_dir, 

766 verbose=working_dir is not None, 

767 **deprecated, 

768 ) 

769 

770 if stop_early and not passed_recreate_test_outputs: 770 ↛ 771line 770 didn't jump to line 771 because the condition on line 770 was never true

771 break 

772 

773 if not isinstance(rd, v0_4.ModelDescr): 

774 passed_parametrized_inference = _test_parametrized_inference( 

775 rd, w, devices, stop_early=stop_early 

776 ) 

777 if stop_early and not passed_parametrized_inference: 777 ↛ 778line 777 didn't jump to line 778 because the condition on line 777 was never true

778 break 

779 

780 # TODO: add execution of jupyter notebooks 

781 # TODO: add more tests 

782 

783 return rd 

784 

785 

786def _get_tolerance( 

787 model: v0_4.ModelDescr | v0_5.ModelDescr, 

788 wf: SupportedWeightsFormat, 

789 m: MemberId, 

790 **deprecated: Unpack[DeprecatedKwargs], 

791) -> tuple[RelativeTolerance, AbsoluteTolerance, MismatchedElementsPerMillion]: 

792 out_descr = next((out for out in model.outputs if get_member_id(out) == m), None) 

793 if out_descr is None: 793 ↛ 794line 793 didn't jump to line 794 because the condition on line 793 was never true

794 raise ValueError(f"Output '{m}' not found in model outputs") 

795 if isinstance(model, v0_5.ModelDescr): 

796 applicable = None 

797 # check legacy test kwargs for weight format specific tolerance 

798 if model.config.bioimageio.model_extra is not None: 798 ↛ 810line 798 didn't jump to line 810 because the condition on line 798 was always true

799 for weights_format, test_kwargs in model.config.bioimageio.model_extra.get( 799 ↛ 802line 799 didn't jump to line 802 because the loop on line 799 never started

800 "test_kwargs", {} 

801 ).items(): 

802 if wf == weights_format: 

803 applicable = v0_5.ReproducibilityTolerance( 

804 relative_tolerance=test_kwargs.get("relative_tolerance", 1e-3), 

805 absolute_tolerance=test_kwargs.get("absolute_tolerance", 1e-3), 

806 ) 

807 break 

808 

809 # check for weights format and output tensor specific tolerance 

810 for a in model.config.bioimageio.reproducibility_tolerance: 810 ↛ 842line 810 didn't jump to line 842 because the loop on line 810 didn't complete

811 if (not a.weights_formats or wf in a.weights_formats) and ( 

812 not a.output_ids 

813 or m in a.output_ids 

814 or ( 

815 a.output_ids == "any_continuous_output" 

816 and next( 

817 ( 

818 o.dtype.startswith("float") 

819 for o in model.outputs 

820 if o.id == m 

821 ), 

822 False, 

823 ) 

824 ) 

825 or ( 

826 a.output_ids == "any_discrete_output" 

827 and next( 

828 ( 

829 o.dtype.startswith("int") 

830 or o.dtype.startswith("uint") 

831 or o.dtype.startswith("bool") 

832 for o in model.outputs 

833 if o.id == m 

834 ), 

835 False, 

836 ) 

837 ) 

838 ): 

839 applicable = a 

840 break 

841 

842 if applicable is None: 842 ↛ 843line 842 didn't jump to line 843 because the condition on line 842 was never true

843 if out_descr.dtype.startswith("float"): 

844 applicable = v0_5.ContinuousReproducibilityTolerance.model_construct() 

845 else: 

846 applicable = v0_5.DiscreteReproducibilityTolerance.model_construct() 

847 

848 rtol = applicable.relative_tolerance 

849 atol = applicable.absolute_tolerance 

850 mismatched_tol = applicable.mismatched_elements_per_million 

851 logger.debug( 

852 "Using reproducibility tolerance: rtol={}, atol={}, mismatched_tol_ppm={}", 

853 rtol, 

854 atol, 

855 mismatched_tol, 

856 ) 

857 elif (decimal := deprecated.get("decimal")) is not None: 857 ↛ 858line 857 didn't jump to line 858 because the condition on line 857 was never true

858 warnings.warn( 

859 "The argument `decimal` has been deprecated in favour of" 

860 + " `relative_tolerance` and `absolute_tolerance`, with different" 

861 + " validation logic, using `numpy.testing.assert_allclose, see" 

862 + " 'https://numpy.org/doc/stable/reference/generated/" 

863 + " numpy.testing.assert_allclose.html'. Passing a value for `decimal`" 

864 + " will cause validation to revert to the old behaviour." 

865 ) 

866 atol = 1.5 * 10 ** (-decimal) 

867 rtol = 0 

868 mismatched_tol = 0 

869 logger.debug( 

870 "Using given (deprecated) reproducibility tolerance from 'decimal': atol={}", 

871 atol, 

872 ) 

873 else: 

874 is_continuous = out_descr.dtype.startswith("float") 

875 rtol = deprecated.get("relative_tolerance", 1e-3 if is_continuous else 0) 

876 atol = deprecated.get("absolute_tolerance", 1e-3 if is_continuous else 0) 

877 mismatched_tol = 0 if is_continuous else 1000 

878 logger.debug( 

879 "Using reproducibility tolerance: rtol={}, atol={}, mismatched_tol_ppm={}", 

880 rtol, 

881 atol, 

882 mismatched_tol, 

883 ) 

884 

885 return rtol, atol, mismatched_tol 

886 

887 

888def evaluate_mismatched_elements( 

889 actual: Tensor, 

890 expected: Tensor, 

891 rtol: float, 

892 atol: float, 

893 mismatched_tol: float, 

894 name: str, 

895) -> tuple[float, str, str | None]: 

896 try: 

897 expected_np = expected.data.to_numpy().astype(np.float32) 

898 dims = expected.dims 

899 del expected 

900 actual_np: NDArray[Any] = actual.data.to_numpy().astype(np.float32) 

901 del actual 

902 

903 rtol_value = rtol * abs(expected_np) 

904 abs_diff = abs(actual_np - expected_np) 

905 mismatched = ~np.isclose(actual_np, expected_np, rtol=rtol, atol=atol) 

906 mismatched_elements = mismatched.sum().item() 

907 

908 mismatched_ppm = mismatched_elements / expected_np.size * 1e6 

909 abs_diff[~mismatched] = 0 # ignore non-mismatched elements 

910 

911 r_max_idx_flat = (r_diff := (abs_diff / (abs(expected_np) + 1e-6))).argmax() 

912 r_max_idx = np.unravel_index(r_max_idx_flat, r_diff.shape) 

913 r_max = r_diff[r_max_idx].item() 

914 r_actual = actual_np[r_max_idx].item() 

915 r_expected = expected_np[r_max_idx].item() 

916 

917 # Calculate the max absolute difference with the relative tolerance subtracted 

918 abs_diff_wo_rtol = abs_diff - rtol_value 

919 a_max_idx = np.unravel_index(abs_diff_wo_rtol.argmax(), abs_diff_wo_rtol.shape) 

920 

921 a_max = abs_diff[a_max_idx].item() 

922 a_actual = actual_np[a_max_idx].item() 

923 a_expected = expected_np[a_max_idx].item() 

924 except Exception as e: 

925 mismatched_ppm = -1 

926 msg = "" 

927 error_msg = ( 

928 f"Error while checking if '{name}' disagrees with expected values: {e}" 

929 ) 

930 else: 

931 error_msg = None 

932 if mismatched_elements: 

933 msg = ( 

934 f"Output '{name}': {mismatched_elements} of " 

935 + f"{expected_np.size} elements (" 

936 + ( 

937 f"{mismatched_ppm / 10_000:.1f}%" 

938 if mismatched_ppm >= 1_000 

939 else f"{mismatched_ppm:.1f} ppm" 

940 ) 

941 + ") disagree with expected values (tolerance: " 

942 + ( 

943 f"{mismatched_tol / 10_000:.1f}%" 

944 if mismatched_tol >= 1_000 

945 else f"{mismatched_tol:.1f} ppm" 

946 ) 

947 + ")." 

948 ) 

949 else: 

950 msg = f"Output `{name}`: all elements agree with expected values. " 

951 

952 if rtol > 0 and r_max != 0: 

953 msg += ( 

954 f"\nMax relative difference not accounted for by absolute tolerance ({atol:.2e}): {r_max:.2e} " 

955 + rf"(= \|{r_actual:.2e} - {r_expected:.2e}\|/\|{r_expected:.2e} + 1e-6\|)" 

956 + f" at {dict(zip(dims, map(int, r_max_idx)))} " 

957 ) 

958 if atol > 0 and a_max != 0: 

959 msg += ( 

960 f"\nMax absolute difference not accounted for by relative tolerance ({rtol:.2e}): {a_max:.2e}" 

961 + rf" (= \|{a_actual:.7e} - {a_expected:.7e}\|) at {dict(zip(dims, map(int, a_max_idx)))}" 

962 ) 

963 

964 return mismatched_ppm, msg, error_msg 

965 

966 

967def _test_recreate_test_outputs( 

968 model: v0_4.ModelDescr | v0_5.ModelDescr, 

969 weight_format: SupportedWeightsFormat, 

970 devices: Sequence[str] | None, 

971 stop_early: bool, 

972 *, 

973 working_dir: os.PathLike[str] | str | None, 

974 verbose: bool, 

975 **deprecated: Unpack[DeprecatedKwargs], 

976) -> bool: 

977 passed = _test_recreate_test_outputs_impl( 

978 model, 

979 weight_format, 

980 devices, 

981 stop_early, 

982 working_dir=working_dir, 

983 verbose=verbose, 

984 blockwise=False, 

985 **deprecated, 

986 ) 

987 if ( 

988 (passed or not stop_early) 

989 and isinstance(model, v0_5.ModelDescr) 

990 and any(a.concatenable == "reproducible" for t in model.inputs for a in t.axes) 

991 ): 

992 blockwise_passed = _test_recreate_test_outputs_blockwise( 

993 model, 

994 weight_format, 

995 devices, 

996 stop_early, 

997 working_dir=working_dir, 

998 verbose=verbose, 

999 **deprecated, 

1000 ) 

1001 passed = passed and blockwise_passed 

1002 

1003 return passed 

1004 

1005 

1006def _test_recreate_test_outputs_blockwise( 

1007 model: v0_5.ModelDescr, 

1008 weight_format: SupportedWeightsFormat, 

1009 devices: Sequence[str] | None, 

1010 stop_early: bool, 

1011 *, 

1012 working_dir: os.PathLike[str] | str | None, 

1013 verbose: bool, 

1014 **deprecated: Unpack[DeprecatedKwargs], 

1015) -> bool: 

1016 return _test_recreate_test_outputs_impl( 

1017 model, 

1018 weight_format, 

1019 devices, 

1020 stop_early, 

1021 working_dir=working_dir, 

1022 verbose=verbose, 

1023 blockwise=True, 

1024 **deprecated, 

1025 ) 

1026 

1027 

1028def _test_recreate_test_outputs_impl( 

1029 model: v0_4.ModelDescr | v0_5.ModelDescr, 

1030 weight_format: SupportedWeightsFormat, 

1031 devices: Sequence[str] | None, 

1032 stop_early: bool, 

1033 *, 

1034 working_dir: os.PathLike[str] | str | None, 

1035 verbose: bool, 

1036 blockwise: bool, 

1037 **deprecated: Unpack[DeprecatedKwargs], 

1038) -> bool: 

1039 test_name = ( 

1040 f"Reproduce test outputs from test inputs" 

1041 f"{' blockwise' if blockwise else ''} ({weight_format})" 

1042 ) 

1043 logger.debug("starting '{}'", test_name) 

1044 error_entries: list[ErrorEntry] = [] 

1045 warning_entries: list[WarningEntry] = [] 

1046 

1047 def add_error_entry(msg: str, with_traceback: bool = False): 

1048 error_entries.append( 

1049 ErrorEntry( 

1050 loc=("weights", weight_format), 

1051 msg=msg, 

1052 type="bioimageio.core", 

1053 with_traceback=with_traceback, 

1054 ) 

1055 ) 

1056 

1057 def add_warning_entry(msg: str, severity: WarningSeverity): 

1058 warning_entries.append( 

1059 WarningEntry( 

1060 loc=("weights", weight_format), 

1061 msg=msg, 

1062 type="bioimageio.core", 

1063 severity=severity, 

1064 ) 

1065 ) 

1066 

1067 def save_to_working_dir(name: str, tensor: Tensor) -> list[Path]: 

1068 saved_paths: list[Path] = [] 

1069 if working_dir is not None and verbose: 

1070 for p in [ 

1071 Path(working_dir) / f"{name}_{weight_format}{suffix}" 

1072 for suffix in (".npy", ".tiff") 

1073 ]: 

1074 try: 

1075 save_tensor(p, tensor) 

1076 except Exception as e: 

1077 logger.error( 

1078 "Failed to save tensor {}: {}", 

1079 p, 

1080 e, 

1081 ) 

1082 else: 

1083 saved_paths.append(p) 

1084 

1085 return saved_paths 

1086 

1087 try: 

1088 test_input = get_test_input_sample(model) 

1089 expected_sample = get_test_output_sample(model) 

1090 

1091 if blockwise and isinstance(model, v0_5.ModelDescr): 

1092 concatenable_axes = [ 

1093 a 

1094 for t in model.inputs 

1095 for a in t.axes 

1096 if a.concatenable == "reproducible" 

1097 ] 

1098 if ( 

1099 concatenable_axes 

1100 and all(isinstance(a, v0_5.BatchAxis) for a in concatenable_axes) 

1101 and all( 

1102 t.tagged_shape[BATCH_AXIS_ID] == 1 

1103 for t in test_input.members.values() 

1104 if BATCH_AXIS_ID in t.dims 

1105 ) 

1106 ): 

1107 for sample in (test_input, expected_sample): 

1108 sample.members = { 

1109 m: Tensor.from_xarray( 

1110 xr.concat([t.data, t.data], dim=BATCH_AXIS_ID) 

1111 ) 

1112 if BATCH_AXIS_ID in t.dims 

1113 else t 

1114 for m, t in sample.members.items() 

1115 } 

1116 add_warning_entry( 

1117 "Duplicated test inputs along the singleton batch dimension for blockwise reproduction.", 

1118 severity=INFO, 

1119 ) 

1120 

1121 with create_prediction_pipeline( 

1122 bioimageio_model=model, 

1123 devices=devices, 

1124 weight_format=weight_format, 

1125 dataset_for_initial_statistics=[test_input], 

1126 ) as prediction_pipeline: 

1127 prediction_pipeline.apply_preprocessing(test_input) 

1128 test_input_preprocessed = deepcopy(test_input) 

1129 if blockwise: 

1130 assert isinstance(model, v0_5.ModelDescr) 

1131 input_halo = get_input_halo(model) 

1132 ns = { 

1133 (t.id, a.id): ( 

1134 max( 

1135 0, 

1136 ( 

1137 sum(input_halo.get(t.id, {}).get(a.id, (0, 0))) 

1138 - a.size.min 

1139 ) 

1140 // a.size.step 

1141 + 1, 

1142 ) 

1143 if a.concatenable == "reproducible" 

1144 else a.size.get_n(test_input.members[t.id].tagged_shape[a.id]) 

1145 ) 

1146 for t in model.inputs 

1147 for a in t.axes 

1148 if isinstance(a.size, v0_5.ParameterizedSize) 

1149 } 

1150 axes = {(t.id, a.id): a for t in model.inputs for a in t.axes} 

1151 for axis in axes.values(): 

1152 if axis.concatenable == "reproducible": 

1153 continue 

1154 

1155 seen: set[tuple[MemberId, AxisId]] = set() 

1156 while isinstance(axis.size, v0_5.SizeReference): 

1157 key = (axis.size.tensor_id, axis.size.axis_id) 

1158 if key in seen: 1158 ↛ 1159line 1158 didn't jump to line 1159 because the condition on line 1158 was never true

1159 raise ValueError( 

1160 f"Circular input axis size reference: {key}" 

1161 ) 

1162 seen.add(key) 

1163 axis = axes[key] 

1164 if isinstance(axis.size, v0_5.ParameterizedSize): 1164 ↛ 1156line 1164 didn't jump to line 1156 because the condition on line 1164 was always true

1165 ns[key] = axis.size.get_n( 

1166 test_input.members[key[0]].tagged_shape[key[1]] 

1167 ) 

1168 

1169 block_shapes = model.get_tensor_sizes( 

1170 ns, batch_size=1, max_input_shape=test_input.shape 

1171 ).inputs 

1172 concatenable_sizes = { 

1173 (t.id, a.id): ( 

1174 test_input.members[t.id].tagged_shape[a.id], 

1175 block_shapes[t.id][a.id], 

1176 ) 

1177 for t in model.inputs 

1178 for a in t.axes 

1179 if a.concatenable == "reproducible" 

1180 } 

1181 if not any(size > block for size, block in concatenable_sizes.values()): 

1182 raise ValueError( 

1183 "A test input must be larger than the minimal block shape " 

1184 + "along at least one reproducibly concatenable axis to test blockwise " 

1185 + "reproducibility. Input and block sizes: " 

1186 + f"{concatenable_sizes}" 

1187 ) 

1188 results_not_postprocessed = prediction_pipeline.predict_with_blocking( 

1189 test_input, 

1190 ns=ns, 

1191 batch_size=1, 

1192 skip_postprocessing=True, 

1193 skip_preprocessing=True, 

1194 skip_input_padding=True, 

1195 skip_output_cropping=True, 

1196 ) 

1197 else: 

1198 results_not_postprocessed = ( 

1199 prediction_pipeline.predict_without_blocking( 

1200 test_input, 

1201 skip_postprocessing=True, 

1202 skip_preprocessing=True, 

1203 skip_input_padding=True, 

1204 skip_output_cropping=True, 

1205 ) 

1206 ) 

1207 results_not_postprocessed.members = { 

1208 m: tensor.as_numpy() 

1209 for m, tensor in results_not_postprocessed.members.items() 

1210 } 

1211 results = deepcopy(results_not_postprocessed) 

1212 prediction_pipeline.apply_postprocessing(results) 

1213 

1214 if len(results.members) != len(expected_sample.members): 1214 ↛ 1215line 1214 didn't jump to line 1215 because the condition on line 1214 was never true

1215 add_error_entry( 

1216 f"Expected {len(expected_sample.members)} outputs, but got {len(results.members)}" 

1217 ) 

1218 

1219 else: 

1220 intermediate_paths: list[Path] = [] 

1221 for m, expected in test_input_preprocessed.members.items(): 

1222 intermediate_paths.extend( 

1223 save_to_working_dir(f"test_input_preprocessed_{m}", expected) 

1224 ) 

1225 if intermediate_paths: 

1226 logger.debug("Saved preprocessed test inputs to {}", intermediate_paths) 

1227 

1228 for m, expected in expected_sample.members.items(): 

1229 actual = results.members.get(m) 

1230 if actual is None: 1230 ↛ 1231line 1230 didn't jump to line 1231 because the condition on line 1230 was never true

1231 add_error_entry("Output tensors for test case may not be None") 

1232 if stop_early: 

1233 break 

1234 else: 

1235 continue 

1236 

1237 if actual.dims != expected.dims: 1237 ↛ 1238line 1237 didn't jump to line 1238 because the condition on line 1237 was never true

1238 add_error_entry( 

1239 f"Output '{m}' has dims {actual.dims}, but expected {expected.dims}" 

1240 ) 

1241 if stop_early: 

1242 break 

1243 else: 

1244 continue 

1245 

1246 if actual.tagged_shape != expected.tagged_shape: 1246 ↛ 1247line 1246 didn't jump to line 1247 because the condition on line 1246 was never true

1247 add_error_entry( 

1248 f"Output '{m}' has shape {actual.tagged_shape}, but expected {expected.tagged_shape}" 

1249 ) 

1250 if stop_early: 

1251 break 

1252 else: 

1253 continue 

1254 

1255 try: 

1256 output_prefix = ( 

1257 "actual_output_blockwise" if blockwise else "actual_output" 

1258 ) 

1259 output_paths = save_to_working_dir(f"{output_prefix}_{m}", actual) 

1260 if m in results_not_postprocessed.members: 1260 ↛ 1271line 1260 didn't jump to line 1271 because the condition on line 1260 was always true

1261 output_paths.extend( 

1262 save_to_working_dir( 

1263 f"{output_prefix}_{m}_not_postprocessed", 

1264 results_not_postprocessed.members[m], 

1265 ) 

1266 ) 

1267 except Exception as e: 

1268 logger.error(f"Failed to save actual output tensor for '{m}': {e}") 

1269 output_paths = None 

1270 

1271 if blockwise: 

1272 assert isinstance(model, v0_5.ModelDescr) 

1273 output = next(o for o in model.outputs if o.id == m) 

1274 halo = { 

1275 a.id: a.halo 

1276 for a in output.axes 

1277 if isinstance(a, v0_5.WithHalo) 

1278 } 

1279 for axis, width in halo.items(): 

1280 if actual.tagged_shape[axis] <= 2 * width: 1280 ↛ 1281line 1280 didn't jump to line 1281 because the condition on line 1280 was never true

1281 raise ValueError( 

1282 f"Output '{m}' is too small to compare after cropping " 

1283 + f"halo {width} from axis '{axis}' " 

1284 + f"(size {actual.tagged_shape[axis]})." 

1285 ) 

1286 actual = actual.crop_halo(halo) 

1287 expected = expected.crop_halo(halo) 

1288 

1289 rtol, atol, mismatched_tol = _get_tolerance( 

1290 model, wf=weight_format, m=m, **deprecated 

1291 ) 

1292 mismatched_ppm, msg, error_msg = evaluate_mismatched_elements( 

1293 actual, expected, rtol, atol, mismatched_tol, m 

1294 ) 

1295 if error_msg is not None: 1295 ↛ 1296line 1295 didn't jump to line 1296 because the condition on line 1295 was never true

1296 add_error_entry(error_msg) 

1297 if stop_early: 

1298 break 

1299 

1300 if output_paths: 

1301 msg += f"\n Saved (intermediate) outputs to {output_paths}." 

1302 

1303 if mismatched_ppm > mismatched_tol: 

1304 add_error_entry(msg) 

1305 if stop_early: 

1306 break 

1307 else: 

1308 add_warning_entry( 

1309 msg, severity=WARNING if mismatched_ppm != 0 else INFO 

1310 ) 

1311 

1312 except Exception as e: 

1313 if get_validation_context().raise_errors: 1313 ↛ 1314line 1313 didn't jump to line 1314 because the condition on line 1313 was never true

1314 raise 

1315 

1316 add_error_entry(str(e), with_traceback=True) 

1317 

1318 model.validation_summary.add_detail( 

1319 ValidationDetail( 

1320 name=test_name, 

1321 loc=("weights", weight_format), 

1322 status="failed" if error_entries else "passed", 

1323 recommended_env=get_conda_env(entry=dict(model.weights)[weight_format]), 

1324 errors=error_entries, 

1325 warnings=warning_entries, 

1326 ) 

1327 ) 

1328 return not error_entries 

1329 

1330 

1331def _test_parametrized_inference( 

1332 model: v0_5.ModelDescr, 

1333 weight_format: SupportedWeightsFormat, 

1334 devices: Sequence[str] | None, 

1335 *, 

1336 stop_early: bool, 

1337) -> bool: 

1338 if not any( 

1339 isinstance(a.size, v0_5.ParameterizedSize) 

1340 for ipt in model.inputs 

1341 for a in ipt.axes 

1342 ): 

1343 # no parameterized sizes => set n=0 

1344 ns: set[v0_5.ParameterizedSize_N] = {0} 

1345 else: 

1346 ns = {0, 1, 2} 

1347 

1348 given_batch_sizes = { 

1349 a.size 

1350 for ipt in model.inputs 

1351 for a in ipt.axes 

1352 if isinstance(a, v0_5.BatchAxis) 

1353 } 

1354 if given_batch_sizes: 

1355 batch_sizes = {gbs for gbs in given_batch_sizes if gbs is not None} 

1356 if not batch_sizes: 

1357 # only arbitrary batch sizes 

1358 batch_sizes = {1, 2} 

1359 else: 

1360 # no batch axis 

1361 batch_sizes = {1} 

1362 

1363 test_cases: set[tuple[BatchSize, v0_5.ParameterizedSize_N]] = { 

1364 (b, n) for b, n in product(sorted(batch_sizes), sorted(ns)) 

1365 } 

1366 logger.info( 

1367 "Testing inference with '{}' for {} different inputs (B, N): {}", 

1368 weight_format, 

1369 len(test_cases), 

1370 test_cases, 

1371 ) 

1372 

1373 def generate_test_cases(): 

1374 tested: set[Hashable] = set() 

1375 

1376 def get_ns(n: int): 

1377 return { 

1378 (t.id, a.id): n 

1379 for t in model.inputs 

1380 for a in t.axes 

1381 if isinstance(a.size, v0_5.ParameterizedSize) 

1382 } 

1383 

1384 for batch_size, n in sorted(test_cases): 

1385 input_target_sizes, expected_output_sizes = model.get_axis_sizes( 

1386 get_ns(n), batch_size=batch_size 

1387 ) 

1388 hashable_target_size = tuple( 

1389 (k, input_target_sizes[k]) for k in sorted(input_target_sizes) 

1390 ) 

1391 if hashable_target_size in tested: 1391 ↛ 1392line 1391 didn't jump to line 1392 because the condition on line 1391 was never true

1392 continue 

1393 else: 

1394 tested.add(hashable_target_size) 

1395 

1396 resized_test_inputs = Sample( 

1397 members={ 

1398 t.id: ( 

1399 test_input.members[t.id].resize_to( 

1400 { 

1401 aid: s 

1402 for (tid, aid), s in input_target_sizes.items() 

1403 if tid == t.id 

1404 }, 

1405 ) 

1406 ) 

1407 for t in model.inputs 

1408 }, 

1409 stat=test_input.stat, 

1410 id=test_input.id, 

1411 ) 

1412 expected_output_shapes = { 

1413 t.id: { 

1414 aid: s 

1415 for (tid, aid), s in expected_output_sizes.items() 

1416 if tid == t.id 

1417 } 

1418 for t in model.outputs 

1419 } 

1420 yield n, batch_size, resized_test_inputs, expected_output_shapes 

1421 

1422 passed = True 

1423 try: 

1424 test_input = get_test_input_sample(model) 

1425 

1426 with create_prediction_pipeline( 

1427 bioimageio_model=model, 

1428 devices=devices, 

1429 weight_format=weight_format, 

1430 dataset_for_initial_statistics=[test_input], 

1431 ) as prediction_pipeline: 

1432 for n, batch_size, inputs, exptected_output_shape in generate_test_cases(): 

1433 error: str | None = None 

1434 try: 

1435 result = prediction_pipeline.predict_without_blocking( 

1436 inputs, skip_input_padding=True, skip_output_cropping=True 

1437 ) 

1438 except Exception as e: 

1439 error = str(e) 

1440 else: 

1441 if len(result.members) != len(exptected_output_shape): 1441 ↛ 1442line 1441 didn't jump to line 1442 because the condition on line 1441 was never true

1442 error = ( 

1443 f"Expected {len(exptected_output_shape)} outputs," 

1444 + f" but got {len(result.members)}" 

1445 ) 

1446 

1447 else: 

1448 for m, exp in exptected_output_shape.items(): 

1449 res = result.members.get(m) 

1450 if res is None: 1450 ↛ 1451line 1450 didn't jump to line 1451 because the condition on line 1450 was never true

1451 error = "Output tensors may not be None for test case" 

1452 break 

1453 

1454 diff: dict[AxisId, int] = {} 

1455 for a, s in res.sizes.items(): 

1456 if isinstance((e_aid := exp[AxisId(a)]), int): 1456 ↛ 1459line 1456 didn't jump to line 1459 because the condition on line 1456 was always true

1457 if s != e_aid: 1457 ↛ 1458line 1457 didn't jump to line 1458 because the condition on line 1457 was never true

1458 diff[AxisId(a)] = s 

1459 elif ( 

1460 s < e_aid.min 

1461 or e_aid.max is not None 

1462 and s > e_aid.max 

1463 ): 

1464 diff[AxisId(a)] = s 

1465 if diff: 1465 ↛ 1466line 1465 didn't jump to line 1466 because the condition on line 1465 was never true

1466 error = ( 

1467 f"(n={n}) Expected output shape {exp}," 

1468 + f" but got {res.sizes} (diff: {diff})" 

1469 ) 

1470 break 

1471 

1472 model.validation_summary.add_detail( 

1473 ValidationDetail( 

1474 name=f"Run {weight_format} inference for inputs with" 

1475 + f" batch_size: {batch_size} and size parameter n: {n}", 

1476 loc=("weights", weight_format), 

1477 status="passed" if error is None else "failed", 

1478 errors=( 

1479 [] 

1480 if error is None 

1481 else [ 

1482 ErrorEntry( 

1483 loc=("weights", weight_format), 

1484 msg=error, 

1485 type="bioimageio.core", 

1486 ) 

1487 ] 

1488 ), 

1489 ) 

1490 ) 

1491 if error is not None: 1491 ↛ 1492line 1491 didn't jump to line 1492 because the condition on line 1491 was never true

1492 passed = False 

1493 if stop_early and error is not None: 1493 ↛ 1494line 1493 didn't jump to line 1494 because the condition on line 1493 was never true

1494 break 

1495 except Exception as e: 

1496 if get_validation_context().raise_errors: 

1497 raise 

1498 

1499 model.validation_summary.add_detail( 

1500 ValidationDetail( 

1501 name=f"Run {weight_format} inference for parametrized inputs", 

1502 status="failed", 

1503 loc=("weights", weight_format), 

1504 errors=[ 

1505 ErrorEntry( 

1506 loc=("weights", weight_format), 

1507 msg=str(e), 

1508 type="bioimageio.core", 

1509 with_traceback=True, 

1510 ) 

1511 ], 

1512 ) 

1513 ) 

1514 passed = False 

1515 

1516 return passed 

1517 

1518 

1519def _test_expected_resource_type(rd: InvalidDescr | ResourceDescr, expected_type: str): 

1520 has_expected_type = rd.type is expected_type 

1521 rd.validation_summary.details.append( 

1522 ValidationDetail( 

1523 name="Has expected resource type", 

1524 status="passed" if has_expected_type else "failed", 

1525 loc=("type",), 

1526 errors=( 

1527 [] 

1528 if has_expected_type 

1529 else [ 

1530 ErrorEntry( 

1531 loc=("type",), 

1532 type="type", 

1533 msg=f"Expected type {expected_type}, found {rd.type}", 

1534 ) 

1535 ] 

1536 ), 

1537 ) 

1538 ) 

1539 return has_expected_type 

1540 

1541 

1542# TODO: Implement `debug_model()` 

1543# def debug_model( 

1544# model_rdf: Union[RawResourceDescr, ResourceDescr, URI, Path, str], 

1545# *, 

1546# weight_format: Optional[WeightsFormat] = None, 

1547# devices: Optional[List[str]] = None, 

1548# ): 

1549# """Run the model test and return dict with inputs, results, expected results and intermediates. 

1550 

1551# Returns dict with tensors "inputs", "inputs_processed", "outputs_raw", "outputs", "expected" and "diff". 

1552# """ 

1553# inputs_raw: Optional = None 

1554# inputs_processed: Optional = None 

1555# outputs_raw: Optional = None 

1556# outputs: Optional = None 

1557# expected: Optional = None 

1558# diff: Optional = None 

1559 

1560# model = load_description( 

1561# model_rdf, weights_priority_order=None if weight_format is None else [weight_format] 

1562# ) 

1563# if not isinstance(model, Model): 

1564# raise ValueError(f"Not a bioimageio.model: {model_rdf}") 

1565 

1566# prediction_pipeline = create_prediction_pipeline( 

1567# bioimageio_model=model, devices=devices, weight_format=weight_format 

1568# ) 

1569# inputs = [ 

1570# xr.DataArray(load_array(str(in_path)), dims=input_spec.axes) 

1571# for in_path, input_spec in zip(model.test_inputs, model.inputs) 

1572# ] 

1573# input_dict = {input_spec.name: input for input_spec, input in zip(model.inputs, inputs)} 

1574 

1575# # keep track of the non-processed inputs 

1576# inputs_raw = [deepcopy(input) for input in inputs] 

1577 

1578# computed_measures = {} 

1579 

1580# prediction_pipeline.apply_preprocessing(input_dict, computed_measures) 

1581# inputs_processed = list(input_dict.values()) 

1582# outputs_raw = prediction_pipeline.predict(*inputs_processed) 

1583# output_dict = {output_spec.name: deepcopy(output) for output_spec, output in zip(model.outputs, outputs_raw)} 

1584# prediction_pipeline.apply_postprocessing(output_dict, computed_measures) 

1585# outputs = list(output_dict.values()) 

1586 

1587# if isinstance(outputs, (np.ndarray, xr.DataArray)): 

1588# outputs = [outputs] 

1589 

1590# expected = [ 

1591# xr.DataArray(load_array(str(out_path)), dims=output_spec.axes) 

1592# for out_path, output_spec in zip(model.test_outputs, model.outputs) 

1593# ] 

1594# if len(outputs) != len(expected): 

1595# error = f"Number of outputs and number of expected outputs disagree: {len(outputs)} != {len(expected)}" 

1596# print(error) 

1597# else: 

1598# diff = [] 

1599# for res, exp in zip(outputs, expected): 

1600# diff.append(res - exp) 

1601 

1602# return { 

1603# "inputs": inputs_raw, 

1604# "inputs_processed": inputs_processed, 

1605# "outputs_raw": outputs_raw, 

1606# "outputs": outputs, 

1607# "expected": expected, 

1608# "diff": diff, 

1609# }