Coverage for src/bioimageio/core/_resource_tests.py: 69%
511 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 09:46 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 09:46 +0000
1from __future__ import annotations
3import hashlib
4import os
5import platform
6import subprocess
7import sys
8import warnings
9from collections.abc import Callable, Hashable, Sequence
10from contextlib import nullcontext
11from copy import deepcopy
12from io import StringIO
13from itertools import product
14from pathlib import Path
15from tempfile import TemporaryDirectory
16from typing import (
17 Any,
18 Literal,
19 get_args,
20 overload,
21)
23import numpy as np
24import xarray as xr
25from loguru import logger
26from numpy.typing import NDArray
27from typing_extensions import NotRequired, TypedDict, Unpack, assert_never
29from bioimageio.spec import (
30 AnyDatasetDescr,
31 AnyModelDescr,
32 BioimageioCondaEnv,
33 DatasetDescr,
34 InvalidDescr,
35 LatestResourceDescr,
36 ModelDescr,
37 ResourceDescr,
38 ValidationContext,
39 build_description,
40 dump_description,
41 get_conda_env,
42 load_description,
43 save_bioimageio_package,
44)
45from bioimageio.spec._description_impl import DISCOVER
46from bioimageio.spec._internal.common_nodes import ResourceDescrBase
47from bioimageio.spec._internal.io import is_yaml_value
48from bioimageio.spec._internal.io_utils import read_yaml, write_yaml
49from bioimageio.spec._internal.types import (
50 AbsoluteTolerance,
51 FormatVersionPlaceholder,
52 MismatchedElementsPerMillion,
53 RelativeTolerance,
54)
55from bioimageio.spec._internal.validation_context import get_validation_context
56from bioimageio.spec._internal.warning_levels import INFO, WARNING, WarningSeverity
57from bioimageio.spec.common import BioimageioYamlContent, PermissiveFileSource, Sha256
58from bioimageio.spec.model import v0_4, v0_5
59from bioimageio.spec.model.v0_5 import WeightsFormat
60from bioimageio.spec.summary import (
61 ErrorEntry,
62 InstalledPackage,
63 ValidationDetail,
64 ValidationSummary,
65 WarningEntry,
66)
68from . import __version__
69from ._prediction_pipeline import create_prediction_pipeline
70from ._settings import settings
71from .axis import BATCH_AXIS_ID, AxisId, BatchSize
72from .common import MemberId, SupportedWeightsFormat
73from .digest_spec import (
74 get_input_halo,
75 get_member_id,
76 get_test_input_sample,
77 get_test_output_sample,
78)
79from .io import save_tensor
80from .sample import Sample
81from .tensor import Tensor
83CONDA_CMD = "conda.bat" if platform.system() == "Windows" else "conda"
86class DeprecatedKwargs(TypedDict):
87 absolute_tolerance: NotRequired[AbsoluteTolerance]
88 relative_tolerance: NotRequired[RelativeTolerance]
89 decimal: NotRequired[int | None]
92def enable_determinism(
93 mode: Literal["seed_only", "full"] = "full",
94 weight_formats: Sequence[SupportedWeightsFormat] | None = None,
95):
96 """Seed and configure ML frameworks for maximum reproducibility.
97 May degrade performance. Only recommended for testing reproducibility!
99 Seed any random generators and (if **mode**=="full") request ML frameworks to use
100 deterministic algorithms.
102 Args:
103 mode: determinism mode
104 - 'seed_only' -- only set seeds, or
105 - 'full' determinsm features (might degrade performance or throw exceptions)
106 weight_formats: Limit deep learning importing deep learning frameworks
107 based on weight_formats.
108 E.g. this allows to avoid importing tensorflow when testing with pytorch.
110 Notes:
111 - **mode** == "full" might degrade performance or throw exceptions.
112 - Subsequent inference calls might still differ. Call before each function
113 (sequence) that is expected to be reproducible.
114 - Degraded performance: Use for testing reproducibility only!
115 - Recipes:
116 - [PyTorch](https://pytorch.org/docs/stable/notes/randomness.html)
117 - [Keras](https://keras.io/examples/keras_recipes/reproducibility_recipes/)
118 - [NumPy](https://numpy.org/doc/2.0/reference/random/generated/numpy.random.seed.html)
119 """
120 try:
121 try:
122 import numpy.random
123 except ImportError:
124 pass
125 else:
126 numpy.random.seed(0)
127 except Exception as e:
128 logger.debug(str(e))
130 if (
131 weight_formats is None
132 or "pytorch_state_dict" in weight_formats
133 or "torchscript" in weight_formats
134 ):
135 try:
136 try:
137 import torch
138 except ImportError:
139 pass
140 else:
141 _ = torch.manual_seed(0)
142 torch.use_deterministic_algorithms(mode == "full")
143 except Exception as e:
144 logger.debug(str(e))
146 if (
147 weight_formats is None
148 or "tensorflow_saved_model_bundle" in weight_formats
149 or "keras_hdf5" in weight_formats
150 ):
151 try:
152 os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0"
153 try:
154 import tensorflow as tf
155 except ImportError:
156 pass
157 else:
158 tf.random.set_seed(0)
159 if mode == "full":
160 tf.config.experimental.enable_op_determinism()
161 # TODO: find possibility to switch it off again??
162 except Exception as e:
163 logger.debug(str(e))
165 if weight_formats is None or "keras_hdf5" in weight_formats:
166 try:
167 try:
168 import keras # pyright: ignore[reportMissingTypeStubs]
169 except ImportError:
170 pass
171 else:
172 keras.utils.set_random_seed(0)
173 except Exception as e:
174 logger.debug(str(e))
177def test_model(
178 source: v0_4.ModelDescr | v0_5.ModelDescr | PermissiveFileSource,
179 weight_format: SupportedWeightsFormat | None = None,
180 devices: list[str] | None = None,
181 *,
182 determinism: Literal["seed_only", "full"] = "seed_only",
183 sha256: Sha256 | None = None,
184 stop_early: bool = False,
185 working_dir: os.PathLike[str] | str | None = None,
186 **deprecated: Unpack[DeprecatedKwargs],
187) -> ValidationSummary:
188 """Test model inference"""
189 return test_description(
190 source,
191 weight_format=weight_format,
192 devices=devices,
193 determinism=determinism,
194 expected_type="model",
195 sha256=sha256,
196 stop_early=stop_early,
197 working_dir=working_dir,
198 **deprecated,
199 )
202def default_run_command(args: Sequence[str]):
203 logger.info("running '{}'...", " ".join(args))
204 _ = subprocess.check_call(args)
207def test_description(
208 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent,
209 *,
210 format_version: FormatVersionPlaceholder | str = "discover",
211 weight_format: SupportedWeightsFormat | None = None,
212 devices: Sequence[str] | None = None,
213 determinism: Literal["seed_only", "full"] = "seed_only",
214 expected_type: str | None = None,
215 sha256: Sha256 | None = None,
216 stop_early: bool = False,
217 runtime_env: Literal["currently-active", "as-described"]
218 | Path
219 | BioimageioCondaEnv = ("currently-active"),
220 run_command: Callable[[Sequence[str]], None] = default_run_command,
221 working_dir: os.PathLike[str] | str | None = None,
222 **deprecated: Unpack[DeprecatedKwargs],
223) -> ValidationSummary:
224 """Test a bioimage.io resource dynamically,
225 for example run prediction of test tensors for models.
227 Args:
228 source: model description source.
229 weight_format: Weight format to test.
230 Default: All weight formats present in **source**.
231 devices: Devices to test with, e.g. 'cpu', 'cuda'.
232 Default (may be weight format dependent): ['cuda'] if available, ['cpu'] otherwise.
233 determinism: Modes to improve reproducibility of test outputs.
234 expected_type: Assert an expected resource description `type`.
235 sha256: Expected SHA256 value of **source**.
236 (Ignored if **source** already is a loaded `ResourceDescr` object.)
237 stop_early: Do not run further subtests after a failed one.
238 runtime_env: (Experimental feature!) The Python environment to run the tests in
239 - `"currently-active"`: Use active Python interpreter.
240 - `"as-described"`: Use `bioimageio.spec.get_conda_env` to generate a conda
241 environment YAML file based on the model weights description.
242 - A `BioimageioCondaEnv` or a path to a conda environment YAML file.
243 Note: The `bioimageio.core` dependency will be added automatically if not present.
244 run_command: (Experimental feature!) Function to execute (conda) terminal commands in a subprocess.
245 The function should raise an exception if the command fails.
246 **run_command** is ignored if **runtime_env** is `"currently-active"`.
247 working_dir: (for debugging) directory to save any temporary files
248 (model packages, conda environments, test summaries).
249 Defaults to a temporary directory.
250 """
251 if runtime_env == "currently-active":
252 rd = load_description_and_test(
253 source,
254 format_version=format_version,
255 weight_format=weight_format,
256 devices=devices,
257 determinism=determinism,
258 expected_type=expected_type,
259 sha256=sha256,
260 stop_early=stop_early,
261 working_dir=working_dir,
262 **deprecated,
263 )
264 return rd.validation_summary
266 if runtime_env == "as-described": 266 ↛ 268line 266 didn't jump to line 268 because the condition on line 266 was always true
267 conda_env = None
268 elif isinstance(runtime_env, (str, Path)):
269 conda_env = BioimageioCondaEnv.model_validate(read_yaml(Path(runtime_env)))
270 elif isinstance(runtime_env, BioimageioCondaEnv):
271 conda_env = runtime_env
272 else:
273 assert_never(runtime_env)
275 if run_command is not default_run_command: 275 ↛ 276line 275 didn't jump to line 276 because the condition on line 275 was never true
276 try:
277 run_command(["thiscommandshouldalwaysfail", "please"])
278 except Exception as e:
279 logger.trace("run_command failed successfully :) {}", e)
280 else:
281 raise RuntimeError(
282 "given run_command does not raise an exception for a failing command"
283 )
285 verbose = working_dir is not None
286 if working_dir is None: 286 ↛ 292line 286 didn't jump to line 292 because the condition on line 286 was always true
287 td_kwargs: dict[str, Any] = (
288 {"ignore_cleanup_errors": True} if sys.version_info >= (3, 10) else {}
289 )
290 working_dir_ctxt = TemporaryDirectory(**td_kwargs)
291 else:
292 working_dir_ctxt = nullcontext(working_dir)
294 with working_dir_ctxt as _d:
295 working_dir = Path(_d)
297 if isinstance(source, ResourceDescrBase): 297 ↛ 298line 297 didn't jump to line 298 because the condition on line 297 was never true
298 descr = source
299 elif isinstance(source, dict): 299 ↛ 300line 299 didn't jump to line 300 because the condition on line 299 was never true
300 context = get_validation_context().replace(
301 perform_io_checks=True # make sure we perform io checks though
302 )
304 descr = build_description(source, context=context)
305 else:
306 descr = load_description(source, perform_io_checks=True)
308 if isinstance(descr, InvalidDescr): 308 ↛ 309line 308 didn't jump to line 309 because the condition on line 308 was never true
309 return descr.validation_summary
310 elif isinstance(source, (dict, ResourceDescrBase)): 310 ↛ 311line 310 didn't jump to line 311 because the condition on line 310 was never true
311 file_source = save_bioimageio_package(
312 descr, output_path=working_dir / "package.zip"
313 )
314 else:
315 file_source = source
317 # elevate status valid-format to passed and start testing
318 descr.validation_summary.status = "passed"
319 try:
320 _test_in_env(
321 file_source,
322 descr=descr,
323 working_dir=working_dir,
324 weight_format=weight_format,
325 conda_env=conda_env,
326 devices=devices,
327 determinism=determinism,
328 expected_type=expected_type,
329 sha256=sha256,
330 stop_early=stop_early,
331 run_command=run_command,
332 verbose=verbose,
333 **deprecated,
334 )
335 except Exception as e:
336 descr.validation_summary.add_detail(
337 ValidationDetail(
338 name="Test in dedicated environment",
339 status="failed",
340 loc=(),
341 errors=[
342 ErrorEntry(
343 loc=(),
344 msg=str(e),
345 type="bioimageio.core",
346 with_traceback=True,
347 )
348 ],
349 )
350 )
352 return descr.validation_summary
355def _test_in_env(
356 source: PermissiveFileSource,
357 *,
358 descr: ResourceDescr,
359 working_dir: Path,
360 weight_format: SupportedWeightsFormat | None,
361 conda_env: BioimageioCondaEnv | None,
362 devices: Sequence[str] | None,
363 determinism: Literal["seed_only", "full"],
364 run_command: Callable[[Sequence[str]], None],
365 stop_early: bool,
366 expected_type: str | None,
367 sha256: Sha256 | None,
368 verbose: bool,
369 **deprecated: Unpack[DeprecatedKwargs],
370):
371 """Test a bioimage.io resource in a given conda environment.
372 Adds details to the existing validation summary of **descr**.
373 """
374 if isinstance(descr, (v0_4.ModelDescr, v0_5.ModelDescr)): 374 ↛ 435line 374 didn't jump to line 435 because the condition on line 374 was always true
375 if weight_format is None:
376 # run tests for all present weight formats
377 all_present_wfs = [
378 wf for wf in get_args(WeightsFormat) if getattr(descr.weights, wf, None)
379 ]
380 ignore_wfs = [wf for wf in all_present_wfs if wf in ["tensorflow_js"]]
381 logger.info(
382 "Found weight formats {}. Start testing all{}...",
383 all_present_wfs,
384 f" (except: {', '.join(ignore_wfs)}) " if ignore_wfs else "",
385 )
386 for wf in all_present_wfs: 386 ↛ 403line 386 didn't jump to line 403 because the loop on line 386 didn't complete
387 _test_in_env(
388 source,
389 descr=descr,
390 working_dir=working_dir / wf,
391 weight_format=wf,
392 devices=devices,
393 determinism=determinism,
394 conda_env=conda_env,
395 run_command=run_command,
396 expected_type=expected_type,
397 sha256=sha256,
398 stop_early=stop_early,
399 verbose=verbose,
400 **deprecated,
401 )
403 return
405 if weight_format == "pytorch_state_dict": 405 ↛ 406line 405 didn't jump to line 406 because the condition on line 405 was never true
406 wf = descr.weights.pytorch_state_dict
407 elif weight_format == "torchscript": 407 ↛ 408line 407 didn't jump to line 408 because the condition on line 407 was never true
408 wf = descr.weights.torchscript
409 elif weight_format == "keras_hdf5": 409 ↛ 410line 409 didn't jump to line 410 because the condition on line 409 was never true
410 wf = descr.weights.keras_hdf5
411 elif weight_format == "onnx": 411 ↛ 413line 411 didn't jump to line 413 because the condition on line 411 was always true
412 wf = descr.weights.onnx
413 elif weight_format == "tensorflow_saved_model_bundle":
414 wf = descr.weights.tensorflow_saved_model_bundle
415 elif weight_format == "keras_v3":
416 if isinstance(descr, v0_4.ModelDescr):
417 raise ValueError(
418 "Weight format 'keras_v3' is not supported in v0.4 model descriptions. use format version >= 0.5"
419 )
421 wf = descr.weights.keras_v3
422 elif weight_format == "tensorflow_js":
423 raise RuntimeError(
424 "testing 'tensorflow_js' is not supported by bioimageio.core"
425 )
426 else:
427 assert_never(weight_format)
429 assert wf is not None
430 if conda_env is None: 430 ↛ 433line 430 didn't jump to line 433 because the condition on line 430 was always true
431 conda_env = get_conda_env(entry=wf)
433 test_loc = ("weights", weight_format)
434 else:
435 if conda_env is None:
436 warnings.warn(
437 "No conda environment description given for testing (And no default conda envs available for non-model descriptions)."
438 )
439 return
441 test_loc = ()
443 # remove name as we create a name based on the env description hash value
444 conda_env.name = None
446 dumped_env = conda_env.model_dump(mode="json", exclude_none=True)
447 if not is_yaml_value(dumped_env): 447 ↛ 448line 447 didn't jump to line 448 because the condition on line 447 was never true
448 raise ValueError(f"Failed to dump conda env to valid YAML {conda_env}")
450 env_io = StringIO()
451 write_yaml(dumped_env, file=env_io)
452 encoded_env = env_io.getvalue().encode()
453 env_name = hashlib.sha256(encoded_env).hexdigest()
455 try:
456 run_command(["where" if platform.system() == "Windows" else "which", CONDA_CMD])
457 except Exception as e:
458 raise RuntimeError("Conda not available") from e
460 try:
461 run_command([CONDA_CMD, "run", "-n", env_name, "python", "--version"])
462 except Exception:
463 working_dir.mkdir(parents=True, exist_ok=True)
464 path = working_dir / "env.yaml"
465 try:
466 _ = path.write_bytes(encoded_env)
467 logger.debug("written conda env to {}", path)
468 run_command(
469 [
470 CONDA_CMD,
471 "env",
472 "create",
473 "--yes",
474 f"--file={path}",
475 f"--name={env_name}",
476 ]
477 + (["--quiet"] if settings.CI else [])
478 )
479 # double check that environment was created successfully
480 run_command([CONDA_CMD, "run", "-n", env_name, "python", "--version"])
481 except Exception as e:
482 descr.validation_summary.add_detail(
483 ValidationDetail(
484 name="Conda environment creation",
485 status="failed",
486 loc=test_loc,
487 recommended_env=conda_env,
488 errors=[
489 ErrorEntry(
490 loc=test_loc,
491 msg=str(e),
492 type="conda",
493 with_traceback=True,
494 )
495 ],
496 )
497 )
498 return
499 else:
500 descr.validation_summary.add_detail(
501 ValidationDetail(
502 name=f"Created conda environment '{env_name}'",
503 status="passed",
504 loc=test_loc,
505 )
506 )
507 else:
508 descr.validation_summary.add_detail(
509 ValidationDetail(
510 name=f"Found existing conda environment '{env_name}'",
511 status="passed",
512 loc=test_loc,
513 )
514 )
516 working_dir.mkdir(parents=True, exist_ok=True)
517 summary_path = working_dir / "summary.json"
518 assert not summary_path.exists(), "Summary file already exists"
519 cmd = []
520 cmd_error = None
521 for summary_path_arg_name in ("summary", "summary-path"):
522 try:
523 run_command(
524 cmd := (
525 [
526 CONDA_CMD,
527 "run",
528 "-n",
529 env_name,
530 "bioimageio",
531 "test",
532 str(source),
533 f"--{summary_path_arg_name}={summary_path.as_posix()}",
534 f"--determinism={determinism}",
535 ]
536 + ([f"--weight-format={weight_format}"] if weight_format else [])
537 + ([f"--expected-type={expected_type}"] if expected_type else [])
538 + (["--stop-early"] if stop_early else [])
539 )
540 )
541 except Exception as e:
542 cmd_error = f"Command '{' '.join(cmd)}' returned with error: {e}."
544 if summary_path.exists():
545 break
546 else:
547 if cmd_error is not None:
548 logger.warning(cmd_error)
550 descr.validation_summary.add_detail(
551 ValidationDetail(
552 name="run 'bioimageio test' command",
553 recommended_env=conda_env,
554 errors=[
555 ErrorEntry(
556 loc=(),
557 type="bioimageio cli",
558 msg=f"test command '{' '.join(cmd)}' did not produce a summary file at {summary_path}",
559 )
560 ],
561 status="failed",
562 )
563 )
564 return
566 # add relevant details from command summary
567 command_summary = ValidationSummary.load_json(summary_path)
568 for detail in command_summary.details:
569 if detail.loc[: len(test_loc)] == test_loc or detail.status == "failed":
570 descr.validation_summary.add_detail(detail)
573@overload
574def load_description_and_test(
575 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent,
576 *,
577 format_version: Literal["latest"],
578 weight_format: SupportedWeightsFormat | None = None,
579 devices: Sequence[str] | None = None,
580 determinism: Literal["seed_only", "full"] = "seed_only",
581 expected_type: Literal["model"],
582 sha256: Sha256 | None = None,
583 stop_early: bool = False,
584 working_dir: os.PathLike[str] | str | None = None,
585 **deprecated: Unpack[DeprecatedKwargs],
586) -> ModelDescr | InvalidDescr: ...
589@overload
590def load_description_and_test(
591 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent,
592 *,
593 format_version: Literal["latest"],
594 weight_format: SupportedWeightsFormat | None = None,
595 devices: Sequence[str] | None = None,
596 determinism: Literal["seed_only", "full"] = "seed_only",
597 expected_type: Literal["dataset"],
598 sha256: Sha256 | None = None,
599 stop_early: bool = False,
600 working_dir: os.PathLike[str] | str | None = None,
601 **deprecated: Unpack[DeprecatedKwargs],
602) -> DatasetDescr | InvalidDescr: ...
605@overload
606def load_description_and_test(
607 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent,
608 *,
609 format_version: Literal["latest"],
610 weight_format: SupportedWeightsFormat | None = None,
611 devices: Sequence[str] | None = None,
612 determinism: Literal["seed_only", "full"] = "seed_only",
613 expected_type: str | None = None,
614 sha256: Sha256 | None = None,
615 stop_early: bool = False,
616 working_dir: os.PathLike[str] | str | None = None,
617 **deprecated: Unpack[DeprecatedKwargs],
618) -> LatestResourceDescr | InvalidDescr: ...
621@overload
622def load_description_and_test(
623 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent,
624 *,
625 format_version: FormatVersionPlaceholder | str = DISCOVER,
626 weight_format: SupportedWeightsFormat | None = None,
627 devices: Sequence[str] | None = None,
628 determinism: Literal["seed_only", "full"] = "seed_only",
629 expected_type: Literal["model"],
630 sha256: Sha256 | None = None,
631 stop_early: bool = False,
632 working_dir: os.PathLike[str] | str | None = None,
633 **deprecated: Unpack[DeprecatedKwargs],
634) -> AnyModelDescr | InvalidDescr: ...
637@overload
638def load_description_and_test(
639 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent,
640 *,
641 format_version: FormatVersionPlaceholder | str = DISCOVER,
642 weight_format: SupportedWeightsFormat | None = None,
643 devices: Sequence[str] | None = None,
644 determinism: Literal["seed_only", "full"] = "seed_only",
645 expected_type: Literal["dataset"],
646 sha256: Sha256 | None = None,
647 stop_early: bool = False,
648 working_dir: os.PathLike[str] | str | None = None,
649 **deprecated: Unpack[DeprecatedKwargs],
650) -> AnyDatasetDescr | InvalidDescr: ...
653@overload
654def load_description_and_test(
655 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent,
656 *,
657 format_version: FormatVersionPlaceholder | str = DISCOVER,
658 weight_format: SupportedWeightsFormat | None = None,
659 devices: Sequence[str] | None = None,
660 determinism: Literal["seed_only", "full"] = "seed_only",
661 expected_type: str | None = None,
662 sha256: Sha256 | None = None,
663 stop_early: bool = False,
664 working_dir: os.PathLike[str] | str | None = None,
665 **deprecated: Unpack[DeprecatedKwargs],
666) -> ResourceDescr | InvalidDescr: ...
669def load_description_and_test(
670 source: ResourceDescr | PermissiveFileSource | BioimageioYamlContent,
671 *,
672 format_version: FormatVersionPlaceholder | str = DISCOVER,
673 weight_format: SupportedWeightsFormat | None = None,
674 devices: Sequence[str] | None = None,
675 determinism: Literal["seed_only", "full"] = "seed_only",
676 expected_type: str | None = None,
677 sha256: Sha256 | None = None,
678 stop_early: bool = False,
679 working_dir: os.PathLike[str] | str | None = None,
680 **deprecated: Unpack[DeprecatedKwargs],
681) -> ResourceDescr | InvalidDescr:
682 """Test a bioimage.io resource dynamically,
683 for example run prediction of test tensors for models.
685 See `test_description` for more details.
687 Returns:
688 A (possibly invalid) resource description object
689 with a populated `.validation_summary` attribute.
690 """
691 if isinstance(source, ResourceDescrBase):
692 root = source.root
693 file_name = source.file_name
694 if (
695 (
696 format_version
697 not in (
698 DISCOVER,
699 source.format_version,
700 ".".join(source.format_version.split(".")[:2]),
701 )
702 )
703 or (c := source.validation_summary.details[0].context) is None
704 or not c.perform_io_checks
705 ):
706 logger.debug(
707 "deserializing source to ensure we validate and test using format {} and perform io checks",
708 format_version,
709 )
710 source = dump_description(source)
711 else:
712 root = Path()
713 file_name = None
715 if isinstance(source, ResourceDescrBase):
716 rd = source
717 elif isinstance(source, dict):
718 # check context for a given root; default to root of source
719 context = get_validation_context(
720 ValidationContext(root=root, file_name=file_name)
721 ).replace(
722 perform_io_checks=True # make sure we perform io checks though
723 )
725 rd = build_description(
726 source,
727 format_version=format_version,
728 context=context,
729 )
730 else:
731 rd = load_description(
732 source, format_version=format_version, sha256=sha256, perform_io_checks=True
733 )
735 rd.validation_summary.env.add(
736 InstalledPackage(name="bioimageio.core", version=__version__)
737 )
739 if expected_type is not None:
740 has_expected_type = _test_expected_resource_type(rd, expected_type)
741 if not has_expected_type: 741 ↛ 743line 741 didn't jump to line 743 because the condition on line 741 was never true
742 # unexpected type -> invalid format
743 rd.validation_summary.status = "failed"
744 return rd
746 # elevate status valid-format to passed and start testing
747 if rd.validation_summary.status == "valid-format": 747 ↛ 750line 747 didn't jump to line 750 because the condition on line 747 was always true
748 rd.validation_summary.status = "passed"
750 if isinstance(rd, (v0_4.ModelDescr, v0_5.ModelDescr)): 750 ↛ 783line 750 didn't jump to line 783 because the condition on line 750 was always true
751 if weight_format is None:
752 weight_formats: list[SupportedWeightsFormat] = [
753 w for w, we in rd.weights if we is not None
754 ] # pyright: ignore[reportAssignmentType]
755 else:
756 weight_formats = [weight_format]
758 enable_determinism(determinism, weight_formats=weight_formats)
759 for w in weight_formats:
760 passed_recreate_test_outputs = _test_recreate_test_outputs(
761 rd,
762 w,
763 devices,
764 stop_early=stop_early,
765 working_dir=working_dir,
766 verbose=working_dir is not None,
767 **deprecated,
768 )
770 if stop_early and not passed_recreate_test_outputs: 770 ↛ 771line 770 didn't jump to line 771 because the condition on line 770 was never true
771 break
773 if not isinstance(rd, v0_4.ModelDescr):
774 passed_parametrized_inference = _test_parametrized_inference(
775 rd, w, devices, stop_early=stop_early
776 )
777 if stop_early and not passed_parametrized_inference: 777 ↛ 778line 777 didn't jump to line 778 because the condition on line 777 was never true
778 break
780 # TODO: add execution of jupyter notebooks
781 # TODO: add more tests
783 return rd
786def _get_tolerance(
787 model: v0_4.ModelDescr | v0_5.ModelDescr,
788 wf: SupportedWeightsFormat,
789 m: MemberId,
790 **deprecated: Unpack[DeprecatedKwargs],
791) -> tuple[RelativeTolerance, AbsoluteTolerance, MismatchedElementsPerMillion]:
792 out_descr = next((out for out in model.outputs if get_member_id(out) == m), None)
793 if out_descr is None: 793 ↛ 794line 793 didn't jump to line 794 because the condition on line 793 was never true
794 raise ValueError(f"Output '{m}' not found in model outputs")
795 if isinstance(model, v0_5.ModelDescr):
796 applicable = None
797 # check legacy test kwargs for weight format specific tolerance
798 if model.config.bioimageio.model_extra is not None: 798 ↛ 810line 798 didn't jump to line 810 because the condition on line 798 was always true
799 for weights_format, test_kwargs in model.config.bioimageio.model_extra.get( 799 ↛ 802line 799 didn't jump to line 802 because the loop on line 799 never started
800 "test_kwargs", {}
801 ).items():
802 if wf == weights_format:
803 applicable = v0_5.ReproducibilityTolerance(
804 relative_tolerance=test_kwargs.get("relative_tolerance", 1e-3),
805 absolute_tolerance=test_kwargs.get("absolute_tolerance", 1e-3),
806 )
807 break
809 # check for weights format and output tensor specific tolerance
810 for a in model.config.bioimageio.reproducibility_tolerance: 810 ↛ 842line 810 didn't jump to line 842 because the loop on line 810 didn't complete
811 if (not a.weights_formats or wf in a.weights_formats) and (
812 not a.output_ids
813 or m in a.output_ids
814 or (
815 a.output_ids == "any_continuous_output"
816 and next(
817 (
818 o.dtype.startswith("float")
819 for o in model.outputs
820 if o.id == m
821 ),
822 False,
823 )
824 )
825 or (
826 a.output_ids == "any_discrete_output"
827 and next(
828 (
829 o.dtype.startswith("int")
830 or o.dtype.startswith("uint")
831 or o.dtype.startswith("bool")
832 for o in model.outputs
833 if o.id == m
834 ),
835 False,
836 )
837 )
838 ):
839 applicable = a
840 break
842 if applicable is None: 842 ↛ 843line 842 didn't jump to line 843 because the condition on line 842 was never true
843 if out_descr.dtype.startswith("float"):
844 applicable = v0_5.ContinuousReproducibilityTolerance.model_construct()
845 else:
846 applicable = v0_5.DiscreteReproducibilityTolerance.model_construct()
848 rtol = applicable.relative_tolerance
849 atol = applicable.absolute_tolerance
850 mismatched_tol = applicable.mismatched_elements_per_million
851 logger.debug(
852 "Using reproducibility tolerance: rtol={}, atol={}, mismatched_tol_ppm={}",
853 rtol,
854 atol,
855 mismatched_tol,
856 )
857 elif (decimal := deprecated.get("decimal")) is not None: 857 ↛ 858line 857 didn't jump to line 858 because the condition on line 857 was never true
858 warnings.warn(
859 "The argument `decimal` has been deprecated in favour of"
860 + " `relative_tolerance` and `absolute_tolerance`, with different"
861 + " validation logic, using `numpy.testing.assert_allclose, see"
862 + " 'https://numpy.org/doc/stable/reference/generated/"
863 + " numpy.testing.assert_allclose.html'. Passing a value for `decimal`"
864 + " will cause validation to revert to the old behaviour."
865 )
866 atol = 1.5 * 10 ** (-decimal)
867 rtol = 0
868 mismatched_tol = 0
869 logger.debug(
870 "Using given (deprecated) reproducibility tolerance from 'decimal': atol={}",
871 atol,
872 )
873 else:
874 is_continuous = out_descr.dtype.startswith("float")
875 rtol = deprecated.get("relative_tolerance", 1e-3 if is_continuous else 0)
876 atol = deprecated.get("absolute_tolerance", 1e-3 if is_continuous else 0)
877 mismatched_tol = 0 if is_continuous else 1000
878 logger.debug(
879 "Using reproducibility tolerance: rtol={}, atol={}, mismatched_tol_ppm={}",
880 rtol,
881 atol,
882 mismatched_tol,
883 )
885 return rtol, atol, mismatched_tol
888def evaluate_mismatched_elements(
889 actual: Tensor,
890 expected: Tensor,
891 rtol: float,
892 atol: float,
893 mismatched_tol: float,
894 name: str,
895) -> tuple[float, str, str | None]:
896 try:
897 expected_np = expected.data.to_numpy().astype(np.float32)
898 dims = expected.dims
899 del expected
900 actual_np: NDArray[Any] = actual.data.to_numpy().astype(np.float32)
901 del actual
903 rtol_value = rtol * abs(expected_np)
904 abs_diff = abs(actual_np - expected_np)
905 mismatched = ~np.isclose(actual_np, expected_np, rtol=rtol, atol=atol)
906 mismatched_elements = mismatched.sum().item()
908 mismatched_ppm = mismatched_elements / expected_np.size * 1e6
909 abs_diff[~mismatched] = 0 # ignore non-mismatched elements
911 r_max_idx_flat = (r_diff := (abs_diff / (abs(expected_np) + 1e-6))).argmax()
912 r_max_idx = np.unravel_index(r_max_idx_flat, r_diff.shape)
913 r_max = r_diff[r_max_idx].item()
914 r_actual = actual_np[r_max_idx].item()
915 r_expected = expected_np[r_max_idx].item()
917 # Calculate the max absolute difference with the relative tolerance subtracted
918 abs_diff_wo_rtol = abs_diff - rtol_value
919 a_max_idx = np.unravel_index(abs_diff_wo_rtol.argmax(), abs_diff_wo_rtol.shape)
921 a_max = abs_diff[a_max_idx].item()
922 a_actual = actual_np[a_max_idx].item()
923 a_expected = expected_np[a_max_idx].item()
924 except Exception as e:
925 mismatched_ppm = -1
926 msg = ""
927 error_msg = (
928 f"Error while checking if '{name}' disagrees with expected values: {e}"
929 )
930 else:
931 error_msg = None
932 if mismatched_elements:
933 msg = (
934 f"Output '{name}': {mismatched_elements} of "
935 + f"{expected_np.size} elements ("
936 + (
937 f"{mismatched_ppm / 10_000:.1f}%"
938 if mismatched_ppm >= 1_000
939 else f"{mismatched_ppm:.1f} ppm"
940 )
941 + ") disagree with expected values (tolerance: "
942 + (
943 f"{mismatched_tol / 10_000:.1f}%"
944 if mismatched_tol >= 1_000
945 else f"{mismatched_tol:.1f} ppm"
946 )
947 + ")."
948 )
949 else:
950 msg = f"Output `{name}`: all elements agree with expected values. "
952 if rtol > 0 and r_max != 0:
953 msg += (
954 f"\nMax relative difference not accounted for by absolute tolerance ({atol:.2e}): {r_max:.2e} "
955 + rf"(= \|{r_actual:.2e} - {r_expected:.2e}\|/\|{r_expected:.2e} + 1e-6\|)"
956 + f" at {dict(zip(dims, map(int, r_max_idx)))} "
957 )
958 if atol > 0 and a_max != 0:
959 msg += (
960 f"\nMax absolute difference not accounted for by relative tolerance ({rtol:.2e}): {a_max:.2e}"
961 + rf" (= \|{a_actual:.7e} - {a_expected:.7e}\|) at {dict(zip(dims, map(int, a_max_idx)))}"
962 )
964 return mismatched_ppm, msg, error_msg
967def _test_recreate_test_outputs(
968 model: v0_4.ModelDescr | v0_5.ModelDescr,
969 weight_format: SupportedWeightsFormat,
970 devices: Sequence[str] | None,
971 stop_early: bool,
972 *,
973 working_dir: os.PathLike[str] | str | None,
974 verbose: bool,
975 **deprecated: Unpack[DeprecatedKwargs],
976) -> bool:
977 passed = _test_recreate_test_outputs_impl(
978 model,
979 weight_format,
980 devices,
981 stop_early,
982 working_dir=working_dir,
983 verbose=verbose,
984 blockwise=False,
985 **deprecated,
986 )
987 if (
988 (passed or not stop_early)
989 and isinstance(model, v0_5.ModelDescr)
990 and any(a.concatenable == "reproducible" for t in model.inputs for a in t.axes)
991 ):
992 blockwise_passed = _test_recreate_test_outputs_blockwise(
993 model,
994 weight_format,
995 devices,
996 stop_early,
997 working_dir=working_dir,
998 verbose=verbose,
999 **deprecated,
1000 )
1001 passed = passed and blockwise_passed
1003 return passed
1006def _test_recreate_test_outputs_blockwise(
1007 model: v0_5.ModelDescr,
1008 weight_format: SupportedWeightsFormat,
1009 devices: Sequence[str] | None,
1010 stop_early: bool,
1011 *,
1012 working_dir: os.PathLike[str] | str | None,
1013 verbose: bool,
1014 **deprecated: Unpack[DeprecatedKwargs],
1015) -> bool:
1016 return _test_recreate_test_outputs_impl(
1017 model,
1018 weight_format,
1019 devices,
1020 stop_early,
1021 working_dir=working_dir,
1022 verbose=verbose,
1023 blockwise=True,
1024 **deprecated,
1025 )
1028def _test_recreate_test_outputs_impl(
1029 model: v0_4.ModelDescr | v0_5.ModelDescr,
1030 weight_format: SupportedWeightsFormat,
1031 devices: Sequence[str] | None,
1032 stop_early: bool,
1033 *,
1034 working_dir: os.PathLike[str] | str | None,
1035 verbose: bool,
1036 blockwise: bool,
1037 **deprecated: Unpack[DeprecatedKwargs],
1038) -> bool:
1039 test_name = (
1040 f"Reproduce test outputs from test inputs"
1041 f"{' blockwise' if blockwise else ''} ({weight_format})"
1042 )
1043 logger.debug("starting '{}'", test_name)
1044 error_entries: list[ErrorEntry] = []
1045 warning_entries: list[WarningEntry] = []
1047 def add_error_entry(msg: str, with_traceback: bool = False):
1048 error_entries.append(
1049 ErrorEntry(
1050 loc=("weights", weight_format),
1051 msg=msg,
1052 type="bioimageio.core",
1053 with_traceback=with_traceback,
1054 )
1055 )
1057 def add_warning_entry(msg: str, severity: WarningSeverity):
1058 warning_entries.append(
1059 WarningEntry(
1060 loc=("weights", weight_format),
1061 msg=msg,
1062 type="bioimageio.core",
1063 severity=severity,
1064 )
1065 )
1067 def save_to_working_dir(name: str, tensor: Tensor) -> list[Path]:
1068 saved_paths: list[Path] = []
1069 if working_dir is not None and verbose:
1070 for p in [
1071 Path(working_dir) / f"{name}_{weight_format}{suffix}"
1072 for suffix in (".npy", ".tiff")
1073 ]:
1074 try:
1075 save_tensor(p, tensor)
1076 except Exception as e:
1077 logger.error(
1078 "Failed to save tensor {}: {}",
1079 p,
1080 e,
1081 )
1082 else:
1083 saved_paths.append(p)
1085 return saved_paths
1087 try:
1088 test_input = get_test_input_sample(model)
1089 expected_sample = get_test_output_sample(model)
1091 if blockwise and isinstance(model, v0_5.ModelDescr):
1092 concatenable_axes = [
1093 a
1094 for t in model.inputs
1095 for a in t.axes
1096 if a.concatenable == "reproducible"
1097 ]
1098 if (
1099 concatenable_axes
1100 and all(isinstance(a, v0_5.BatchAxis) for a in concatenable_axes)
1101 and all(
1102 t.tagged_shape[BATCH_AXIS_ID] == 1
1103 for t in test_input.members.values()
1104 if BATCH_AXIS_ID in t.dims
1105 )
1106 ):
1107 for sample in (test_input, expected_sample):
1108 sample.members = {
1109 m: Tensor.from_xarray(
1110 xr.concat([t.data, t.data], dim=BATCH_AXIS_ID)
1111 )
1112 if BATCH_AXIS_ID in t.dims
1113 else t
1114 for m, t in sample.members.items()
1115 }
1116 add_warning_entry(
1117 "Duplicated test inputs along the singleton batch dimension for blockwise reproduction.",
1118 severity=INFO,
1119 )
1121 with create_prediction_pipeline(
1122 bioimageio_model=model,
1123 devices=devices,
1124 weight_format=weight_format,
1125 dataset_for_initial_statistics=[test_input],
1126 ) as prediction_pipeline:
1127 prediction_pipeline.apply_preprocessing(test_input)
1128 test_input_preprocessed = deepcopy(test_input)
1129 if blockwise:
1130 assert isinstance(model, v0_5.ModelDescr)
1131 input_halo = get_input_halo(model)
1132 ns = {
1133 (t.id, a.id): (
1134 max(
1135 0,
1136 (
1137 sum(input_halo.get(t.id, {}).get(a.id, (0, 0)))
1138 - a.size.min
1139 )
1140 // a.size.step
1141 + 1,
1142 )
1143 if a.concatenable == "reproducible"
1144 else a.size.get_n(test_input.members[t.id].tagged_shape[a.id])
1145 )
1146 for t in model.inputs
1147 for a in t.axes
1148 if isinstance(a.size, v0_5.ParameterizedSize)
1149 }
1150 axes = {(t.id, a.id): a for t in model.inputs for a in t.axes}
1151 for axis in axes.values():
1152 if axis.concatenable == "reproducible":
1153 continue
1155 seen: set[tuple[MemberId, AxisId]] = set()
1156 while isinstance(axis.size, v0_5.SizeReference):
1157 key = (axis.size.tensor_id, axis.size.axis_id)
1158 if key in seen: 1158 ↛ 1159line 1158 didn't jump to line 1159 because the condition on line 1158 was never true
1159 raise ValueError(
1160 f"Circular input axis size reference: {key}"
1161 )
1162 seen.add(key)
1163 axis = axes[key]
1164 if isinstance(axis.size, v0_5.ParameterizedSize): 1164 ↛ 1156line 1164 didn't jump to line 1156 because the condition on line 1164 was always true
1165 ns[key] = axis.size.get_n(
1166 test_input.members[key[0]].tagged_shape[key[1]]
1167 )
1169 block_shapes = model.get_tensor_sizes(
1170 ns, batch_size=1, max_input_shape=test_input.shape
1171 ).inputs
1172 concatenable_sizes = {
1173 (t.id, a.id): (
1174 test_input.members[t.id].tagged_shape[a.id],
1175 block_shapes[t.id][a.id],
1176 )
1177 for t in model.inputs
1178 for a in t.axes
1179 if a.concatenable == "reproducible"
1180 }
1181 if not any(size > block for size, block in concatenable_sizes.values()):
1182 raise ValueError(
1183 "A test input must be larger than the minimal block shape "
1184 + "along at least one reproducibly concatenable axis to test blockwise "
1185 + "reproducibility. Input and block sizes: "
1186 + f"{concatenable_sizes}"
1187 )
1188 results_not_postprocessed = prediction_pipeline.predict_with_blocking(
1189 test_input,
1190 ns=ns,
1191 batch_size=1,
1192 skip_postprocessing=True,
1193 skip_preprocessing=True,
1194 skip_input_padding=True,
1195 skip_output_cropping=True,
1196 )
1197 else:
1198 results_not_postprocessed = (
1199 prediction_pipeline.predict_without_blocking(
1200 test_input,
1201 skip_postprocessing=True,
1202 skip_preprocessing=True,
1203 skip_input_padding=True,
1204 skip_output_cropping=True,
1205 )
1206 )
1207 results_not_postprocessed.members = {
1208 m: tensor.as_numpy()
1209 for m, tensor in results_not_postprocessed.members.items()
1210 }
1211 results = deepcopy(results_not_postprocessed)
1212 prediction_pipeline.apply_postprocessing(results)
1214 if len(results.members) != len(expected_sample.members): 1214 ↛ 1215line 1214 didn't jump to line 1215 because the condition on line 1214 was never true
1215 add_error_entry(
1216 f"Expected {len(expected_sample.members)} outputs, but got {len(results.members)}"
1217 )
1219 else:
1220 intermediate_paths: list[Path] = []
1221 for m, expected in test_input_preprocessed.members.items():
1222 intermediate_paths.extend(
1223 save_to_working_dir(f"test_input_preprocessed_{m}", expected)
1224 )
1225 if intermediate_paths:
1226 logger.debug("Saved preprocessed test inputs to {}", intermediate_paths)
1228 for m, expected in expected_sample.members.items():
1229 actual = results.members.get(m)
1230 if actual is None: 1230 ↛ 1231line 1230 didn't jump to line 1231 because the condition on line 1230 was never true
1231 add_error_entry("Output tensors for test case may not be None")
1232 if stop_early:
1233 break
1234 else:
1235 continue
1237 if actual.dims != expected.dims: 1237 ↛ 1238line 1237 didn't jump to line 1238 because the condition on line 1237 was never true
1238 add_error_entry(
1239 f"Output '{m}' has dims {actual.dims}, but expected {expected.dims}"
1240 )
1241 if stop_early:
1242 break
1243 else:
1244 continue
1246 if actual.tagged_shape != expected.tagged_shape: 1246 ↛ 1247line 1246 didn't jump to line 1247 because the condition on line 1246 was never true
1247 add_error_entry(
1248 f"Output '{m}' has shape {actual.tagged_shape}, but expected {expected.tagged_shape}"
1249 )
1250 if stop_early:
1251 break
1252 else:
1253 continue
1255 try:
1256 output_prefix = (
1257 "actual_output_blockwise" if blockwise else "actual_output"
1258 )
1259 output_paths = save_to_working_dir(f"{output_prefix}_{m}", actual)
1260 if m in results_not_postprocessed.members: 1260 ↛ 1271line 1260 didn't jump to line 1271 because the condition on line 1260 was always true
1261 output_paths.extend(
1262 save_to_working_dir(
1263 f"{output_prefix}_{m}_not_postprocessed",
1264 results_not_postprocessed.members[m],
1265 )
1266 )
1267 except Exception as e:
1268 logger.error(f"Failed to save actual output tensor for '{m}': {e}")
1269 output_paths = None
1271 if blockwise:
1272 assert isinstance(model, v0_5.ModelDescr)
1273 output = next(o for o in model.outputs if o.id == m)
1274 halo = {
1275 a.id: a.halo
1276 for a in output.axes
1277 if isinstance(a, v0_5.WithHalo)
1278 }
1279 for axis, width in halo.items():
1280 if actual.tagged_shape[axis] <= 2 * width: 1280 ↛ 1281line 1280 didn't jump to line 1281 because the condition on line 1280 was never true
1281 raise ValueError(
1282 f"Output '{m}' is too small to compare after cropping "
1283 + f"halo {width} from axis '{axis}' "
1284 + f"(size {actual.tagged_shape[axis]})."
1285 )
1286 actual = actual.crop_halo(halo)
1287 expected = expected.crop_halo(halo)
1289 rtol, atol, mismatched_tol = _get_tolerance(
1290 model, wf=weight_format, m=m, **deprecated
1291 )
1292 mismatched_ppm, msg, error_msg = evaluate_mismatched_elements(
1293 actual, expected, rtol, atol, mismatched_tol, m
1294 )
1295 if error_msg is not None: 1295 ↛ 1296line 1295 didn't jump to line 1296 because the condition on line 1295 was never true
1296 add_error_entry(error_msg)
1297 if stop_early:
1298 break
1300 if output_paths:
1301 msg += f"\n Saved (intermediate) outputs to {output_paths}."
1303 if mismatched_ppm > mismatched_tol:
1304 add_error_entry(msg)
1305 if stop_early:
1306 break
1307 else:
1308 add_warning_entry(
1309 msg, severity=WARNING if mismatched_ppm != 0 else INFO
1310 )
1312 except Exception as e:
1313 if get_validation_context().raise_errors: 1313 ↛ 1314line 1313 didn't jump to line 1314 because the condition on line 1313 was never true
1314 raise
1316 add_error_entry(str(e), with_traceback=True)
1318 model.validation_summary.add_detail(
1319 ValidationDetail(
1320 name=test_name,
1321 loc=("weights", weight_format),
1322 status="failed" if error_entries else "passed",
1323 recommended_env=get_conda_env(entry=dict(model.weights)[weight_format]),
1324 errors=error_entries,
1325 warnings=warning_entries,
1326 )
1327 )
1328 return not error_entries
1331def _test_parametrized_inference(
1332 model: v0_5.ModelDescr,
1333 weight_format: SupportedWeightsFormat,
1334 devices: Sequence[str] | None,
1335 *,
1336 stop_early: bool,
1337) -> bool:
1338 if not any(
1339 isinstance(a.size, v0_5.ParameterizedSize)
1340 for ipt in model.inputs
1341 for a in ipt.axes
1342 ):
1343 # no parameterized sizes => set n=0
1344 ns: set[v0_5.ParameterizedSize_N] = {0}
1345 else:
1346 ns = {0, 1, 2}
1348 given_batch_sizes = {
1349 a.size
1350 for ipt in model.inputs
1351 for a in ipt.axes
1352 if isinstance(a, v0_5.BatchAxis)
1353 }
1354 if given_batch_sizes:
1355 batch_sizes = {gbs for gbs in given_batch_sizes if gbs is not None}
1356 if not batch_sizes:
1357 # only arbitrary batch sizes
1358 batch_sizes = {1, 2}
1359 else:
1360 # no batch axis
1361 batch_sizes = {1}
1363 test_cases: set[tuple[BatchSize, v0_5.ParameterizedSize_N]] = {
1364 (b, n) for b, n in product(sorted(batch_sizes), sorted(ns))
1365 }
1366 logger.info(
1367 "Testing inference with '{}' for {} different inputs (B, N): {}",
1368 weight_format,
1369 len(test_cases),
1370 test_cases,
1371 )
1373 def generate_test_cases():
1374 tested: set[Hashable] = set()
1376 def get_ns(n: int):
1377 return {
1378 (t.id, a.id): n
1379 for t in model.inputs
1380 for a in t.axes
1381 if isinstance(a.size, v0_5.ParameterizedSize)
1382 }
1384 for batch_size, n in sorted(test_cases):
1385 input_target_sizes, expected_output_sizes = model.get_axis_sizes(
1386 get_ns(n), batch_size=batch_size
1387 )
1388 hashable_target_size = tuple(
1389 (k, input_target_sizes[k]) for k in sorted(input_target_sizes)
1390 )
1391 if hashable_target_size in tested: 1391 ↛ 1392line 1391 didn't jump to line 1392 because the condition on line 1391 was never true
1392 continue
1393 else:
1394 tested.add(hashable_target_size)
1396 resized_test_inputs = Sample(
1397 members={
1398 t.id: (
1399 test_input.members[t.id].resize_to(
1400 {
1401 aid: s
1402 for (tid, aid), s in input_target_sizes.items()
1403 if tid == t.id
1404 },
1405 )
1406 )
1407 for t in model.inputs
1408 },
1409 stat=test_input.stat,
1410 id=test_input.id,
1411 )
1412 expected_output_shapes = {
1413 t.id: {
1414 aid: s
1415 for (tid, aid), s in expected_output_sizes.items()
1416 if tid == t.id
1417 }
1418 for t in model.outputs
1419 }
1420 yield n, batch_size, resized_test_inputs, expected_output_shapes
1422 passed = True
1423 try:
1424 test_input = get_test_input_sample(model)
1426 with create_prediction_pipeline(
1427 bioimageio_model=model,
1428 devices=devices,
1429 weight_format=weight_format,
1430 dataset_for_initial_statistics=[test_input],
1431 ) as prediction_pipeline:
1432 for n, batch_size, inputs, exptected_output_shape in generate_test_cases():
1433 error: str | None = None
1434 try:
1435 result = prediction_pipeline.predict_without_blocking(
1436 inputs, skip_input_padding=True, skip_output_cropping=True
1437 )
1438 except Exception as e:
1439 error = str(e)
1440 else:
1441 if len(result.members) != len(exptected_output_shape): 1441 ↛ 1442line 1441 didn't jump to line 1442 because the condition on line 1441 was never true
1442 error = (
1443 f"Expected {len(exptected_output_shape)} outputs,"
1444 + f" but got {len(result.members)}"
1445 )
1447 else:
1448 for m, exp in exptected_output_shape.items():
1449 res = result.members.get(m)
1450 if res is None: 1450 ↛ 1451line 1450 didn't jump to line 1451 because the condition on line 1450 was never true
1451 error = "Output tensors may not be None for test case"
1452 break
1454 diff: dict[AxisId, int] = {}
1455 for a, s in res.sizes.items():
1456 if isinstance((e_aid := exp[AxisId(a)]), int): 1456 ↛ 1459line 1456 didn't jump to line 1459 because the condition on line 1456 was always true
1457 if s != e_aid: 1457 ↛ 1458line 1457 didn't jump to line 1458 because the condition on line 1457 was never true
1458 diff[AxisId(a)] = s
1459 elif (
1460 s < e_aid.min
1461 or e_aid.max is not None
1462 and s > e_aid.max
1463 ):
1464 diff[AxisId(a)] = s
1465 if diff: 1465 ↛ 1466line 1465 didn't jump to line 1466 because the condition on line 1465 was never true
1466 error = (
1467 f"(n={n}) Expected output shape {exp},"
1468 + f" but got {res.sizes} (diff: {diff})"
1469 )
1470 break
1472 model.validation_summary.add_detail(
1473 ValidationDetail(
1474 name=f"Run {weight_format} inference for inputs with"
1475 + f" batch_size: {batch_size} and size parameter n: {n}",
1476 loc=("weights", weight_format),
1477 status="passed" if error is None else "failed",
1478 errors=(
1479 []
1480 if error is None
1481 else [
1482 ErrorEntry(
1483 loc=("weights", weight_format),
1484 msg=error,
1485 type="bioimageio.core",
1486 )
1487 ]
1488 ),
1489 )
1490 )
1491 if error is not None: 1491 ↛ 1492line 1491 didn't jump to line 1492 because the condition on line 1491 was never true
1492 passed = False
1493 if stop_early and error is not None: 1493 ↛ 1494line 1493 didn't jump to line 1494 because the condition on line 1493 was never true
1494 break
1495 except Exception as e:
1496 if get_validation_context().raise_errors:
1497 raise
1499 model.validation_summary.add_detail(
1500 ValidationDetail(
1501 name=f"Run {weight_format} inference for parametrized inputs",
1502 status="failed",
1503 loc=("weights", weight_format),
1504 errors=[
1505 ErrorEntry(
1506 loc=("weights", weight_format),
1507 msg=str(e),
1508 type="bioimageio.core",
1509 with_traceback=True,
1510 )
1511 ],
1512 )
1513 )
1514 passed = False
1516 return passed
1519def _test_expected_resource_type(rd: InvalidDescr | ResourceDescr, expected_type: str):
1520 has_expected_type = rd.type is expected_type
1521 rd.validation_summary.details.append(
1522 ValidationDetail(
1523 name="Has expected resource type",
1524 status="passed" if has_expected_type else "failed",
1525 loc=("type",),
1526 errors=(
1527 []
1528 if has_expected_type
1529 else [
1530 ErrorEntry(
1531 loc=("type",),
1532 type="type",
1533 msg=f"Expected type {expected_type}, found {rd.type}",
1534 )
1535 ]
1536 ),
1537 )
1538 )
1539 return has_expected_type
1542# TODO: Implement `debug_model()`
1543# def debug_model(
1544# model_rdf: Union[RawResourceDescr, ResourceDescr, URI, Path, str],
1545# *,
1546# weight_format: Optional[WeightsFormat] = None,
1547# devices: Optional[List[str]] = None,
1548# ):
1549# """Run the model test and return dict with inputs, results, expected results and intermediates.
1551# Returns dict with tensors "inputs", "inputs_processed", "outputs_raw", "outputs", "expected" and "diff".
1552# """
1553# inputs_raw: Optional = None
1554# inputs_processed: Optional = None
1555# outputs_raw: Optional = None
1556# outputs: Optional = None
1557# expected: Optional = None
1558# diff: Optional = None
1560# model = load_description(
1561# model_rdf, weights_priority_order=None if weight_format is None else [weight_format]
1562# )
1563# if not isinstance(model, Model):
1564# raise ValueError(f"Not a bioimageio.model: {model_rdf}")
1566# prediction_pipeline = create_prediction_pipeline(
1567# bioimageio_model=model, devices=devices, weight_format=weight_format
1568# )
1569# inputs = [
1570# xr.DataArray(load_array(str(in_path)), dims=input_spec.axes)
1571# for in_path, input_spec in zip(model.test_inputs, model.inputs)
1572# ]
1573# input_dict = {input_spec.name: input for input_spec, input in zip(model.inputs, inputs)}
1575# # keep track of the non-processed inputs
1576# inputs_raw = [deepcopy(input) for input in inputs]
1578# computed_measures = {}
1580# prediction_pipeline.apply_preprocessing(input_dict, computed_measures)
1581# inputs_processed = list(input_dict.values())
1582# outputs_raw = prediction_pipeline.predict(*inputs_processed)
1583# output_dict = {output_spec.name: deepcopy(output) for output_spec, output in zip(model.outputs, outputs_raw)}
1584# prediction_pipeline.apply_postprocessing(output_dict, computed_measures)
1585# outputs = list(output_dict.values())
1587# if isinstance(outputs, (np.ndarray, xr.DataArray)):
1588# outputs = [outputs]
1590# expected = [
1591# xr.DataArray(load_array(str(out_path)), dims=output_spec.axes)
1592# for out_path, output_spec in zip(model.test_outputs, model.outputs)
1593# ]
1594# if len(outputs) != len(expected):
1595# error = f"Number of outputs and number of expected outputs disagree: {len(outputs)} != {len(expected)}"
1596# print(error)
1597# else:
1598# diff = []
1599# for res, exp in zip(outputs, expected):
1600# diff.append(res - exp)
1602# return {
1603# "inputs": inputs_raw,
1604# "inputs_processed": inputs_processed,
1605# "outputs_raw": outputs_raw,
1606# "outputs": outputs,
1607# "expected": expected,
1608# "diff": diff,
1609# }