Coverage for src/bioimageio/core/_resource_tests.py: 57%
442 statements
« prev ^ index » next coverage.py v7.15.0, created at 2026-07-08 15:59 +0000
1import hashlib
2import os
3import platform
4import subprocess
5import sys
6import warnings
7from contextlib import nullcontext
8from copy import deepcopy
9from io import StringIO
10from itertools import product
11from pathlib import Path
12from tempfile import TemporaryDirectory
13from typing import (
14 Any,
15 Callable,
16 Dict,
17 Hashable,
18 List,
19 Literal,
20 Optional,
21 Sequence,
22 Set,
23 Tuple,
24 Union,
25 overload,
26)
28import numpy as np
29from loguru import logger
30from numpy.typing import NDArray
31from typing_extensions import NotRequired, TypedDict, Unpack, assert_never, get_args
33from bioimageio.spec import (
34 AnyDatasetDescr,
35 AnyModelDescr,
36 BioimageioCondaEnv,
37 DatasetDescr,
38 InvalidDescr,
39 LatestResourceDescr,
40 ModelDescr,
41 ResourceDescr,
42 ValidationContext,
43 build_description,
44 dump_description,
45 get_conda_env,
46 load_description,
47 save_bioimageio_package,
48)
49from bioimageio.spec._description_impl import DISCOVER
50from bioimageio.spec._internal.common_nodes import ResourceDescrBase
51from bioimageio.spec._internal.io import is_yaml_value
52from bioimageio.spec._internal.io_utils import read_yaml, write_yaml
53from bioimageio.spec._internal.types import (
54 AbsoluteTolerance,
55 FormatVersionPlaceholder,
56 MismatchedElementsPerMillion,
57 RelativeTolerance,
58)
59from bioimageio.spec._internal.validation_context import get_validation_context
60from bioimageio.spec._internal.warning_levels import INFO, WARNING, WarningSeverity
61from bioimageio.spec.common import BioimageioYamlContent, PermissiveFileSource, Sha256
62from bioimageio.spec.model import v0_4, v0_5
63from bioimageio.spec.model.v0_5 import WeightsFormat
64from bioimageio.spec.summary import (
65 ErrorEntry,
66 InstalledPackage,
67 ValidationDetail,
68 ValidationSummary,
69 WarningEntry,
70)
72from . import __version__
73from ._prediction_pipeline import create_prediction_pipeline
74from ._settings import settings
75from .axis import AxisId, BatchSize
76from .common import MemberId, SupportedWeightsFormat
77from .digest_spec import get_test_input_sample, get_test_output_sample
78from .io import save_tensor
79from .sample import Sample
80from .tensor import Tensor
82CONDA_CMD = "conda.bat" if platform.system() == "Windows" else "conda"
85class DeprecatedKwargs(TypedDict):
86 absolute_tolerance: NotRequired[AbsoluteTolerance]
87 relative_tolerance: NotRequired[RelativeTolerance]
88 decimal: NotRequired[Optional[int]]
91def enable_determinism(
92 mode: Literal["seed_only", "full"] = "full",
93 weight_formats: Optional[Sequence[SupportedWeightsFormat]] = None,
94):
95 """Seed and configure ML frameworks for maximum reproducibility.
96 May degrade performance. Only recommended for testing reproducibility!
98 Seed any random generators and (if **mode**=="full") request ML frameworks to use
99 deterministic algorithms.
101 Args:
102 mode: determinism mode
103 - 'seed_only' -- only set seeds, or
104 - 'full' determinsm features (might degrade performance or throw exceptions)
105 weight_formats: Limit deep learning importing deep learning frameworks
106 based on weight_formats.
107 E.g. this allows to avoid importing tensorflow when testing with pytorch.
109 Notes:
110 - **mode** == "full" might degrade performance or throw exceptions.
111 - Subsequent inference calls might still differ. Call before each function
112 (sequence) that is expected to be reproducible.
113 - Degraded performance: Use for testing reproducibility only!
114 - Recipes:
115 - [PyTorch](https://pytorch.org/docs/stable/notes/randomness.html)
116 - [Keras](https://keras.io/examples/keras_recipes/reproducibility_recipes/)
117 - [NumPy](https://numpy.org/doc/2.0/reference/random/generated/numpy.random.seed.html)
118 """
119 try:
120 try:
121 import numpy.random
122 except ImportError:
123 pass
124 else:
125 numpy.random.seed(0)
126 except Exception as e:
127 logger.debug(str(e))
129 if (
130 weight_formats is None
131 or "pytorch_state_dict" in weight_formats
132 or "torchscript" in weight_formats
133 ):
134 try:
135 try:
136 import torch
137 except ImportError:
138 pass
139 else:
140 _ = torch.manual_seed(0)
141 torch.use_deterministic_algorithms(mode == "full")
142 except Exception as e:
143 logger.debug(str(e))
145 if (
146 weight_formats is None
147 or "tensorflow_saved_model_bundle" in weight_formats
148 or "keras_hdf5" in weight_formats
149 ):
150 try:
151 os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0"
152 try:
153 import tensorflow as tf
154 except ImportError:
155 pass
156 else:
157 tf.random.set_seed(0)
158 if mode == "full":
159 tf.config.experimental.enable_op_determinism()
160 # TODO: find possibility to switch it off again??
161 except Exception as e:
162 logger.debug(str(e))
164 if weight_formats is None or "keras_hdf5" in weight_formats:
165 try:
166 try:
167 import keras # pyright: ignore[reportMissingTypeStubs]
168 except ImportError:
169 pass
170 else:
171 keras.utils.set_random_seed(0)
172 except Exception as e:
173 logger.debug(str(e))
176def test_model(
177 source: Union[v0_4.ModelDescr, v0_5.ModelDescr, PermissiveFileSource],
178 weight_format: Optional[SupportedWeightsFormat] = None,
179 devices: Optional[List[str]] = None,
180 *,
181 determinism: Literal["seed_only", "full"] = "seed_only",
182 sha256: Optional[Sha256] = None,
183 stop_early: bool = False,
184 working_dir: Optional[Union[os.PathLike[str], str]] = None,
185 **deprecated: Unpack[DeprecatedKwargs],
186) -> ValidationSummary:
187 """Test model inference"""
188 return test_description(
189 source,
190 weight_format=weight_format,
191 devices=devices,
192 determinism=determinism,
193 expected_type="model",
194 sha256=sha256,
195 stop_early=stop_early,
196 working_dir=working_dir,
197 **deprecated,
198 )
201def default_run_command(args: Sequence[str]):
202 logger.info("running '{}'...", " ".join(args))
203 _ = subprocess.check_call(args)
206def test_description(
207 source: Union[ResourceDescr, PermissiveFileSource, BioimageioYamlContent],
208 *,
209 format_version: Union[FormatVersionPlaceholder, str] = "discover",
210 weight_format: Optional[SupportedWeightsFormat] = None,
211 devices: Optional[Sequence[str]] = None,
212 determinism: Literal["seed_only", "full"] = "seed_only",
213 expected_type: Optional[str] = None,
214 sha256: Optional[Sha256] = None,
215 stop_early: bool = False,
216 runtime_env: Union[
217 Literal["currently-active", "as-described"], Path, BioimageioCondaEnv
218 ] = ("currently-active"),
219 run_command: Callable[[Sequence[str]], None] = default_run_command,
220 working_dir: Optional[Union[os.PathLike[str], str]] = None,
221 **deprecated: Unpack[DeprecatedKwargs],
222) -> ValidationSummary:
223 """Test a bioimage.io resource dynamically,
224 for example run prediction of test tensors for models.
226 Args:
227 source: model description source.
228 weight_format: Weight format to test.
229 Default: All weight formats present in **source**.
230 devices: Devices to test with, e.g. 'cpu', 'cuda'.
231 Default (may be weight format dependent): ['cuda'] if available, ['cpu'] otherwise.
232 determinism: Modes to improve reproducibility of test outputs.
233 expected_type: Assert an expected resource description `type`.
234 sha256: Expected SHA256 value of **source**.
235 (Ignored if **source** already is a loaded `ResourceDescr` object.)
236 stop_early: Do not run further subtests after a failed one.
237 runtime_env: (Experimental feature!) The Python environment to run the tests in
238 - `"currently-active"`: Use active Python interpreter.
239 - `"as-described"`: Use `bioimageio.spec.get_conda_env` to generate a conda
240 environment YAML file based on the model weights description.
241 - A `BioimageioCondaEnv` or a path to a conda environment YAML file.
242 Note: The `bioimageio.core` dependency will be added automatically if not present.
243 run_command: (Experimental feature!) Function to execute (conda) terminal commands in a subprocess.
244 The function should raise an exception if the command fails.
245 **run_command** is ignored if **runtime_env** is `"currently-active"`.
246 working_dir: (for debugging) directory to save any temporary files
247 (model packages, conda environments, test summaries).
248 Defaults to a temporary directory.
249 """
250 if runtime_env == "currently-active":
251 rd = load_description_and_test(
252 source,
253 format_version=format_version,
254 weight_format=weight_format,
255 devices=devices,
256 determinism=determinism,
257 expected_type=expected_type,
258 sha256=sha256,
259 stop_early=stop_early,
260 working_dir=working_dir,
261 **deprecated,
262 )
263 return rd.validation_summary
265 if runtime_env == "as-described":
266 conda_env = None
267 elif isinstance(runtime_env, (str, Path)):
268 conda_env = BioimageioCondaEnv.model_validate(read_yaml(Path(runtime_env)))
269 elif isinstance(runtime_env, BioimageioCondaEnv):
270 conda_env = runtime_env
271 else:
272 assert_never(runtime_env)
274 if run_command is not default_run_command:
275 try:
276 run_command(["thiscommandshouldalwaysfail", "please"])
277 except Exception:
278 pass
279 else:
280 raise RuntimeError(
281 "given run_command does not raise an exception for a failing command"
282 )
284 verbose = working_dir is not None
285 if working_dir is None:
286 td_kwargs: Dict[str, Any] = (
287 dict(ignore_cleanup_errors=True) if sys.version_info >= (3, 10) else {}
288 )
289 working_dir_ctxt = TemporaryDirectory(**td_kwargs)
290 else:
291 working_dir_ctxt = nullcontext(working_dir)
293 with working_dir_ctxt as _d:
294 working_dir = Path(_d)
296 if isinstance(source, ResourceDescrBase):
297 descr = source
298 elif isinstance(source, dict):
299 context = get_validation_context().replace(
300 perform_io_checks=True # make sure we perform io checks though
301 )
303 descr = build_description(source, context=context)
304 else:
305 descr = load_description(source, perform_io_checks=True)
307 if isinstance(descr, InvalidDescr):
308 return descr.validation_summary
309 elif isinstance(source, (dict, ResourceDescrBase)):
310 file_source = save_bioimageio_package(
311 descr, output_path=working_dir / "package.zip"
312 )
313 else:
314 file_source = source
316 # elevate status valid-format to passed and start testing
317 descr.validation_summary.status = "passed"
318 try:
319 _test_in_env(
320 file_source,
321 descr=descr,
322 working_dir=working_dir,
323 weight_format=weight_format,
324 conda_env=conda_env,
325 devices=devices,
326 determinism=determinism,
327 expected_type=expected_type,
328 sha256=sha256,
329 stop_early=stop_early,
330 run_command=run_command,
331 verbose=verbose,
332 **deprecated,
333 )
334 except Exception as e:
335 descr.validation_summary.add_detail(
336 ValidationDetail(
337 name="Test in dedicated environment",
338 status="failed",
339 loc=(),
340 errors=[
341 ErrorEntry(
342 loc=(),
343 msg=str(e),
344 type="bioimageio.core",
345 with_traceback=True,
346 )
347 ],
348 )
349 )
351 return descr.validation_summary
354def _test_in_env(
355 source: PermissiveFileSource,
356 *,
357 descr: ResourceDescr,
358 working_dir: Path,
359 weight_format: Optional[SupportedWeightsFormat],
360 conda_env: Optional[BioimageioCondaEnv],
361 devices: Optional[Sequence[str]],
362 determinism: Literal["seed_only", "full"],
363 run_command: Callable[[Sequence[str]], None],
364 stop_early: bool,
365 expected_type: Optional[str],
366 sha256: Optional[Sha256],
367 verbose: bool,
368 **deprecated: Unpack[DeprecatedKwargs],
369):
370 """Test a bioimage.io resource in a given conda environment.
371 Adds details to the existing validation summary of **descr**.
372 """
373 if isinstance(descr, (v0_4.ModelDescr, v0_5.ModelDescr)):
374 if weight_format is None:
375 # run tests for all present weight formats
376 all_present_wfs = [
377 wf for wf in get_args(WeightsFormat) if getattr(descr.weights, wf, None)
378 ]
379 ignore_wfs = [wf for wf in all_present_wfs if wf in ["tensorflow_js"]]
380 logger.info(
381 "Found weight formats {}. Start testing all{}...",
382 all_present_wfs,
383 f" (except: {', '.join(ignore_wfs)}) " if ignore_wfs else "",
384 )
385 for wf in all_present_wfs:
386 _test_in_env(
387 source,
388 descr=descr,
389 working_dir=working_dir / wf,
390 weight_format=wf,
391 devices=devices,
392 determinism=determinism,
393 conda_env=conda_env,
394 run_command=run_command,
395 expected_type=expected_type,
396 sha256=sha256,
397 stop_early=stop_early,
398 verbose=verbose,
399 **deprecated,
400 )
402 return
404 if weight_format == "pytorch_state_dict":
405 wf = descr.weights.pytorch_state_dict
406 elif weight_format == "torchscript":
407 wf = descr.weights.torchscript
408 elif weight_format == "keras_hdf5":
409 wf = descr.weights.keras_hdf5
410 elif weight_format == "onnx":
411 wf = descr.weights.onnx
412 elif weight_format == "tensorflow_saved_model_bundle":
413 wf = descr.weights.tensorflow_saved_model_bundle
414 elif weight_format == "keras_v3":
415 if isinstance(descr, v0_4.ModelDescr):
416 raise ValueError(
417 "Weight format 'keras_v3' is not supported in v0.4 model descriptions. use format version >= 0.5"
418 )
420 wf = descr.weights.keras_v3
421 elif weight_format == "tensorflow_js":
422 raise RuntimeError(
423 "testing 'tensorflow_js' is not supported by bioimageio.core"
424 )
425 else:
426 assert_never(weight_format)
428 assert wf is not None
429 if conda_env is None:
430 conda_env = get_conda_env(entry=wf)
432 test_loc = ("weights", weight_format)
433 else:
434 if conda_env is None:
435 warnings.warn(
436 "No conda environment description given for testing (And no default conda envs available for non-model descriptions)."
437 )
438 return
440 test_loc = ()
442 # remove name as we create a name based on the env description hash value
443 conda_env.name = None
445 dumped_env = conda_env.model_dump(mode="json", exclude_none=True)
446 if not is_yaml_value(dumped_env):
447 raise ValueError(f"Failed to dump conda env to valid YAML {conda_env}")
449 env_io = StringIO()
450 write_yaml(dumped_env, file=env_io)
451 encoded_env = env_io.getvalue().encode()
452 env_name = hashlib.sha256(encoded_env).hexdigest()
454 try:
455 run_command(["where" if platform.system() == "Windows" else "which", CONDA_CMD])
456 except Exception as e:
457 raise RuntimeError("Conda not available") from e
459 try:
460 run_command([CONDA_CMD, "run", "-n", env_name, "python", "--version"])
461 except Exception:
462 working_dir.mkdir(parents=True, exist_ok=True)
463 path = working_dir / "env.yaml"
464 try:
465 _ = path.write_bytes(encoded_env)
466 logger.debug("written conda env to {}", path)
467 run_command(
468 [
469 CONDA_CMD,
470 "env",
471 "create",
472 "--yes",
473 f"--file={path}",
474 f"--name={env_name}",
475 ]
476 + (["--quiet"] if settings.CI else [])
477 )
478 # double check that environment was created successfully
479 run_command([CONDA_CMD, "run", "-n", env_name, "python", "--version"])
480 except Exception as e:
481 descr.validation_summary.add_detail(
482 ValidationDetail(
483 name="Conda environment creation",
484 status="failed",
485 loc=test_loc,
486 recommended_env=conda_env,
487 errors=[
488 ErrorEntry(
489 loc=test_loc,
490 msg=str(e),
491 type="conda",
492 with_traceback=True,
493 )
494 ],
495 )
496 )
497 return
498 else:
499 descr.validation_summary.add_detail(
500 ValidationDetail(
501 name=f"Created conda environment '{env_name}'",
502 status="passed",
503 loc=test_loc,
504 )
505 )
506 else:
507 descr.validation_summary.add_detail(
508 ValidationDetail(
509 name=f"Found existing conda environment '{env_name}'",
510 status="passed",
511 loc=test_loc,
512 )
513 )
515 working_dir.mkdir(parents=True, exist_ok=True)
516 summary_path = working_dir / "summary.json"
517 assert not summary_path.exists(), "Summary file already exists"
518 cmd = []
519 cmd_error = None
520 for summary_path_arg_name in ("summary", "summary-path"):
521 try:
522 run_command(
523 cmd := (
524 [
525 CONDA_CMD,
526 "run",
527 "-n",
528 env_name,
529 "bioimageio",
530 "test",
531 str(source),
532 f"--{summary_path_arg_name}={summary_path.as_posix()}",
533 f"--determinism={determinism}",
534 ]
535 + ([f"--weight-format={weight_format}"] if weight_format else [])
536 + ([f"--expected-type={expected_type}"] if expected_type else [])
537 + (["--stop-early"] if stop_early else [])
538 )
539 )
540 except Exception as e:
541 cmd_error = f"Command '{' '.join(cmd)}' returned with error: {e}."
543 if summary_path.exists():
544 break
545 else:
546 if cmd_error is not None:
547 logger.warning(cmd_error)
549 descr.validation_summary.add_detail(
550 ValidationDetail(
551 name="run 'bioimageio test' command",
552 recommended_env=conda_env,
553 errors=[
554 ErrorEntry(
555 loc=(),
556 type="bioimageio cli",
557 msg=f"test command '{' '.join(cmd)}' did not produce a summary file at {summary_path}",
558 )
559 ],
560 status="failed",
561 )
562 )
563 return
565 # add relevant details from command summary
566 command_summary = ValidationSummary.load_json(summary_path)
567 for detail in command_summary.details:
568 if detail.loc[: len(test_loc)] == test_loc or detail.status == "failed":
569 descr.validation_summary.add_detail(detail)
572@overload
573def load_description_and_test(
574 source: Union[ResourceDescr, PermissiveFileSource, BioimageioYamlContent],
575 *,
576 format_version: Literal["latest"],
577 weight_format: Optional[SupportedWeightsFormat] = None,
578 devices: Optional[Sequence[str]] = None,
579 determinism: Literal["seed_only", "full"] = "seed_only",
580 expected_type: Literal["model"],
581 sha256: Optional[Sha256] = None,
582 stop_early: bool = False,
583 working_dir: Optional[Union[os.PathLike[str], str]] = None,
584 **deprecated: Unpack[DeprecatedKwargs],
585) -> Union[ModelDescr, InvalidDescr]: ...
588@overload
589def load_description_and_test(
590 source: Union[ResourceDescr, PermissiveFileSource, BioimageioYamlContent],
591 *,
592 format_version: Literal["latest"],
593 weight_format: Optional[SupportedWeightsFormat] = None,
594 devices: Optional[Sequence[str]] = None,
595 determinism: Literal["seed_only", "full"] = "seed_only",
596 expected_type: Literal["dataset"],
597 sha256: Optional[Sha256] = None,
598 stop_early: bool = False,
599 working_dir: Optional[Union[os.PathLike[str], str]] = None,
600 **deprecated: Unpack[DeprecatedKwargs],
601) -> Union[DatasetDescr, InvalidDescr]: ...
604@overload
605def load_description_and_test(
606 source: Union[ResourceDescr, PermissiveFileSource, BioimageioYamlContent],
607 *,
608 format_version: Literal["latest"],
609 weight_format: Optional[SupportedWeightsFormat] = None,
610 devices: Optional[Sequence[str]] = None,
611 determinism: Literal["seed_only", "full"] = "seed_only",
612 expected_type: Optional[str] = None,
613 sha256: Optional[Sha256] = None,
614 stop_early: bool = False,
615 working_dir: Optional[Union[os.PathLike[str], str]] = None,
616 **deprecated: Unpack[DeprecatedKwargs],
617) -> Union[LatestResourceDescr, InvalidDescr]: ...
620@overload
621def load_description_and_test(
622 source: Union[ResourceDescr, PermissiveFileSource, BioimageioYamlContent],
623 *,
624 format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
625 weight_format: Optional[SupportedWeightsFormat] = None,
626 devices: Optional[Sequence[str]] = None,
627 determinism: Literal["seed_only", "full"] = "seed_only",
628 expected_type: Literal["model"],
629 sha256: Optional[Sha256] = None,
630 stop_early: bool = False,
631 working_dir: Optional[Union[os.PathLike[str], str]] = None,
632 **deprecated: Unpack[DeprecatedKwargs],
633) -> Union[AnyModelDescr, InvalidDescr]: ...
636@overload
637def load_description_and_test(
638 source: Union[ResourceDescr, PermissiveFileSource, BioimageioYamlContent],
639 *,
640 format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
641 weight_format: Optional[SupportedWeightsFormat] = None,
642 devices: Optional[Sequence[str]] = None,
643 determinism: Literal["seed_only", "full"] = "seed_only",
644 expected_type: Literal["dataset"],
645 sha256: Optional[Sha256] = None,
646 stop_early: bool = False,
647 working_dir: Optional[Union[os.PathLike[str], str]] = None,
648 **deprecated: Unpack[DeprecatedKwargs],
649) -> Union[AnyDatasetDescr, InvalidDescr]: ...
652@overload
653def load_description_and_test(
654 source: Union[ResourceDescr, PermissiveFileSource, BioimageioYamlContent],
655 *,
656 format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
657 weight_format: Optional[SupportedWeightsFormat] = None,
658 devices: Optional[Sequence[str]] = None,
659 determinism: Literal["seed_only", "full"] = "seed_only",
660 expected_type: Optional[str] = None,
661 sha256: Optional[Sha256] = None,
662 stop_early: bool = False,
663 working_dir: Optional[Union[os.PathLike[str], str]] = None,
664 **deprecated: Unpack[DeprecatedKwargs],
665) -> Union[ResourceDescr, InvalidDescr]: ...
668def load_description_and_test(
669 source: Union[ResourceDescr, PermissiveFileSource, BioimageioYamlContent],
670 *,
671 format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
672 weight_format: Optional[SupportedWeightsFormat] = None,
673 devices: Optional[Sequence[str]] = None,
674 determinism: Literal["seed_only", "full"] = "seed_only",
675 expected_type: Optional[str] = None,
676 sha256: Optional[Sha256] = None,
677 stop_early: bool = False,
678 working_dir: Optional[Union[os.PathLike[str], str]] = None,
679 **deprecated: Unpack[DeprecatedKwargs],
680) -> Union[ResourceDescr, InvalidDescr]:
681 """Test a bioimage.io resource dynamically,
682 for example run prediction of test tensors for models.
684 See `test_description` for more details.
686 Returns:
687 A (possibly invalid) resource description object
688 with a populated `.validation_summary` attribute.
689 """
690 if isinstance(source, ResourceDescrBase):
691 root = source.root
692 file_name = source.file_name
693 if (
694 (
695 format_version
696 not in (
697 DISCOVER,
698 source.format_version,
699 ".".join(source.format_version.split(".")[:2]),
700 )
701 )
702 or (c := source.validation_summary.details[0].context) is None
703 or not c.perform_io_checks
704 ):
705 logger.debug(
706 "deserializing source to ensure we validate and test using format {} and perform io checks",
707 format_version,
708 )
709 source = dump_description(source)
710 else:
711 root = Path()
712 file_name = None
714 if isinstance(source, ResourceDescrBase):
715 rd = source
716 elif isinstance(source, dict):
717 # check context for a given root; default to root of source
718 context = get_validation_context(
719 ValidationContext(root=root, file_name=file_name)
720 ).replace(
721 perform_io_checks=True # make sure we perform io checks though
722 )
724 rd = build_description(
725 source,
726 format_version=format_version,
727 context=context,
728 )
729 else:
730 rd = load_description(
731 source, format_version=format_version, sha256=sha256, perform_io_checks=True
732 )
734 rd.validation_summary.env.add(
735 InstalledPackage(name="bioimageio.core", version=__version__)
736 )
738 if expected_type is not None:
739 has_expected_type = _test_expected_resource_type(rd, expected_type)
740 if not has_expected_type:
741 # unexpected type -> invalid format
742 rd.validation_summary.status = "failed"
743 return rd
745 # elevate status valid-format to passed and start testing
746 if rd.validation_summary.status == "valid-format":
747 rd.validation_summary.status = "passed"
749 if isinstance(rd, (v0_4.ModelDescr, v0_5.ModelDescr)):
750 if weight_format is None:
751 weight_formats: List[SupportedWeightsFormat] = [
752 w for w, we in rd.weights if we is not None
753 ] # pyright: ignore[reportAssignmentType]
754 else:
755 weight_formats = [weight_format]
757 enable_determinism(determinism, weight_formats=weight_formats)
758 for w in weight_formats:
759 passed_recreate_test_outputs = _test_recreate_test_outputs(
760 rd,
761 w,
762 devices,
763 stop_early=stop_early,
764 working_dir=working_dir,
765 verbose=working_dir is not None,
766 **deprecated,
767 )
769 if stop_early and not passed_recreate_test_outputs:
770 break
772 if not isinstance(rd, v0_4.ModelDescr):
773 passed_parametrized_inference = _test_parametrized_inference(
774 rd, w, devices, stop_early=stop_early
775 )
776 if stop_early and not passed_parametrized_inference:
777 break
779 # TODO: add execution of jupyter notebooks
780 # TODO: add more tests
782 return rd
785def _get_tolerance(
786 model: Union[v0_4.ModelDescr, v0_5.ModelDescr],
787 wf: SupportedWeightsFormat,
788 m: MemberId,
789 **deprecated: Unpack[DeprecatedKwargs],
790) -> Tuple[RelativeTolerance, AbsoluteTolerance, MismatchedElementsPerMillion]:
791 if isinstance(model, v0_5.ModelDescr):
792 applicable = v0_5.ReproducibilityTolerance()
794 # check legacy test kwargs for weight format specific tolerance
795 if model.config.bioimageio.model_extra is not None:
796 for weights_format, test_kwargs in model.config.bioimageio.model_extra.get(
797 "test_kwargs", {}
798 ).items():
799 if wf == weights_format:
800 applicable = v0_5.ReproducibilityTolerance(
801 relative_tolerance=test_kwargs.get("relative_tolerance", 1e-3),
802 absolute_tolerance=test_kwargs.get("absolute_tolerance", 1e-3),
803 )
804 break
806 # check for weights format and output tensor specific tolerance
807 for a in model.config.bioimageio.reproducibility_tolerance:
808 if (not a.weights_formats or wf in a.weights_formats) and (
809 not a.output_ids or m in a.output_ids
810 ):
811 applicable = a
812 break
814 rtol = applicable.relative_tolerance
815 atol = applicable.absolute_tolerance
816 mismatched_tol = applicable.mismatched_elements_per_million
817 elif (decimal := deprecated.get("decimal")) is not None:
818 warnings.warn(
819 "The argument `decimal` has been deprecated in favour of"
820 + " `relative_tolerance` and `absolute_tolerance`, with different"
821 + " validation logic, using `numpy.testing.assert_allclose, see"
822 + " 'https://numpy.org/doc/stable/reference/generated/"
823 + " numpy.testing.assert_allclose.html'. Passing a value for `decimal`"
824 + " will cause validation to revert to the old behaviour."
825 )
826 atol = 1.5 * 10 ** (-decimal)
827 rtol = 0
828 mismatched_tol = 0
829 else:
830 # use given (deprecated) test kwargs
831 atol = deprecated.get("absolute_tolerance", 1e-3)
832 rtol = deprecated.get("relative_tolerance", 1e-3)
833 mismatched_tol = 0
835 return rtol, atol, mismatched_tol
838def evaluate_mismatched_elements(
839 actual: Tensor, expected: Tensor, rtol: float, atol: float, name: str
840) -> Tuple[float, str, Optional[str]]:
841 try:
842 expected_np = expected.data.to_numpy().astype(np.float32)
843 dims = expected.dims
844 del expected
845 actual_np: NDArray[Any] = actual.data.to_numpy().astype(np.float32)
846 del actual
848 rtol_value = rtol * abs(expected_np)
849 abs_diff = abs(actual_np - expected_np)
850 mismatched = abs_diff > atol + rtol_value
851 mismatched_elements = mismatched.sum().item()
853 mismatched_ppm = mismatched_elements / expected_np.size * 1e6
854 abs_diff[~mismatched] = 0 # ignore non-mismatched elements
856 r_max_idx_flat = (r_diff := (abs_diff / (abs(expected_np) + 1e-6))).argmax()
857 r_max_idx = np.unravel_index(r_max_idx_flat, r_diff.shape)
858 r_max = r_diff[r_max_idx].item()
859 r_actual = actual_np[r_max_idx].item()
860 r_expected = expected_np[r_max_idx].item()
862 # Calculate the max absolute difference with the relative tolerance subtracted
863 abs_diff_wo_rtol: NDArray[np.float32] = abs_diff - rtol_value
864 a_max_idx = np.unravel_index(abs_diff_wo_rtol.argmax(), abs_diff_wo_rtol.shape)
866 a_max = abs_diff[a_max_idx].item()
867 a_actual = actual_np[a_max_idx].item()
868 a_expected = expected_np[a_max_idx].item()
869 except Exception as e:
870 mismatched_ppm = -1
871 msg = ""
872 error_msg = (
873 f"Error while checking if '{name}' disagrees with expected values: {e}"
874 )
875 else:
876 error_msg = None
877 if mismatched_elements:
878 msg = (
879 f"Output '{name}': {mismatched_elements} of "
880 + f"{expected_np.size} elements disagree with expected values ("
881 + (
882 f"{mismatched_ppm / 10_000:.1f}%"
883 if mismatched_ppm >= 1_000
884 else f"{mismatched_ppm:.1f} ppm"
885 )
886 + "). "
887 )
888 else:
889 msg = f"Output `{name}`: all elements agree with expected values. "
891 msg += (
892 f"\nMax relative difference not accounted for by absolute tolerance ({atol:.2e}):\n{r_max:.2e}"
893 + rf" (= \|{r_actual:.2e} - {r_expected:.2e}\|/\|{r_expected:.2e} + 1e-6\|)"
894 + f" at {dict(zip(dims, r_max_idx))} "
895 + f"\nMax absolute difference not accounted for by relative tolerance ({rtol:.2e}):\n{a_max:.2e}"
896 + rf" (= \|{a_actual:.7e} - {a_expected:.7e}\|) at {dict(zip(dims, a_max_idx))}"
897 )
899 return mismatched_ppm, msg, error_msg
902def _test_recreate_test_outputs(
903 model: Union[v0_4.ModelDescr, v0_5.ModelDescr],
904 weight_format: SupportedWeightsFormat,
905 devices: Optional[Sequence[str]],
906 stop_early: bool,
907 *,
908 working_dir: Optional[Union[os.PathLike[str], str]],
909 verbose: bool,
910 **deprecated: Unpack[DeprecatedKwargs],
911) -> bool:
912 test_name = f"Reproduce test outputs from test inputs ({weight_format})"
913 logger.debug("starting '{}'", test_name)
914 error_entries: List[ErrorEntry] = []
915 warning_entries: List[WarningEntry] = []
917 def add_error_entry(msg: str, with_traceback: bool = False):
918 error_entries.append(
919 ErrorEntry(
920 loc=("weights", weight_format),
921 msg=msg,
922 type="bioimageio.core",
923 with_traceback=with_traceback,
924 )
925 )
927 def add_warning_entry(msg: str, severity: WarningSeverity):
928 warning_entries.append(
929 WarningEntry(
930 loc=("weights", weight_format),
931 msg=msg,
932 type="bioimageio.core",
933 severity=severity,
934 )
935 )
937 def save_to_working_dir(name: str, tensor: Tensor) -> List[Path]:
938 saved_paths: List[Path] = []
939 if working_dir is not None and verbose:
940 for p in [
941 Path(working_dir) / f"{name}_{weight_format}{suffix}"
942 for suffix in (".npy", ".tiff")
943 ]:
944 try:
945 save_tensor(p, tensor)
946 except Exception as e:
947 logger.error(
948 "Failed to save tensor {}: {}",
949 p,
950 e,
951 )
952 else:
953 saved_paths.append(p)
955 return saved_paths
957 try:
958 test_input = get_test_input_sample(model)
959 expected = get_test_output_sample(model)
961 with create_prediction_pipeline(
962 bioimageio_model=model, devices=devices, weight_format=weight_format
963 ) as prediction_pipeline:
964 prediction_pipeline.apply_preprocessing(test_input)
965 test_input_preprocessed = deepcopy(test_input)
966 results_not_postprocessed = (
967 prediction_pipeline.predict_sample_without_blocking(
968 test_input,
969 skip_postprocessing=True,
970 skip_preprocessing=True,
971 skip_input_padding=True,
972 skip_output_cropping=True,
973 )
974 )
975 results = deepcopy(results_not_postprocessed)
976 prediction_pipeline.apply_postprocessing(results)
978 if len(results.members) != len(expected.members):
979 add_error_entry(
980 f"Expected {len(expected.members)} outputs, but got {len(results.members)}"
981 )
983 else:
984 intermediate_paths: List[Path] = []
985 for m, t in test_input_preprocessed.members.items():
986 intermediate_paths.extend(
987 save_to_working_dir(f"test_input_preprocessed_{m}", t)
988 )
989 if intermediate_paths:
990 logger.debug("Saved preprocessed test inputs to {}", intermediate_paths)
992 for m, expected in expected.members.items():
993 actual = results.members.get(m)
994 if actual is None:
995 add_error_entry("Output tensors for test case may not be None")
996 if stop_early:
997 break
998 else:
999 continue
1001 if actual.dims != expected.dims:
1002 add_error_entry(
1003 f"Output '{m}' has dims {actual.dims}, but expected {expected.dims}"
1004 )
1005 if stop_early:
1006 break
1007 else:
1008 continue
1010 if actual.tagged_shape != expected.tagged_shape:
1011 add_error_entry(
1012 f"Output '{m}' has shape {actual.tagged_shape}, but expected {expected.tagged_shape}"
1013 )
1014 if stop_early:
1015 break
1016 else:
1017 continue
1019 try:
1020 output_paths = save_to_working_dir(f"actual_output_{m}", actual)
1021 if m in results_not_postprocessed.members:
1022 output_paths.extend(
1023 save_to_working_dir(
1024 f"actual_output_{m}_not_postprocessed",
1025 results_not_postprocessed.members[m],
1026 )
1027 )
1028 except Exception as e:
1029 logger.error(f"Failed to save actual output tensor for '{m}': {e}")
1030 output_paths = None
1032 rtol, atol, mismatched_tol = _get_tolerance(
1033 model, wf=weight_format, m=m, **deprecated
1034 )
1035 mismatched_ppm, msg, error_msg = evaluate_mismatched_elements(
1036 actual, expected, rtol, atol, m
1037 )
1038 if error_msg is not None:
1039 add_error_entry(error_msg)
1040 if stop_early:
1041 break
1043 if output_paths:
1044 msg += f"\n Saved (intermediate) outputs to {output_paths}."
1046 if mismatched_ppm > mismatched_tol:
1047 add_error_entry(msg)
1048 if stop_early:
1049 break
1050 else:
1051 add_warning_entry(
1052 msg, severity=WARNING if mismatched_ppm != 0 else INFO
1053 )
1055 except Exception as e:
1056 if get_validation_context().raise_errors:
1057 raise e
1059 add_error_entry(str(e), with_traceback=True)
1061 model.validation_summary.add_detail(
1062 ValidationDetail(
1063 name=test_name,
1064 loc=("weights", weight_format),
1065 status="failed" if error_entries else "passed",
1066 recommended_env=get_conda_env(entry=dict(model.weights)[weight_format]),
1067 errors=error_entries,
1068 warnings=warning_entries,
1069 )
1070 )
1071 return bool(error_entries)
1074def _test_parametrized_inference(
1075 model: v0_5.ModelDescr,
1076 weight_format: SupportedWeightsFormat,
1077 devices: Optional[Sequence[str]],
1078 *,
1079 stop_early: bool,
1080) -> None:
1081 if not any(
1082 isinstance(a.size, v0_5.ParameterizedSize)
1083 for ipt in model.inputs
1084 for a in ipt.axes
1085 ):
1086 # no parameterized sizes => set n=0
1087 ns: Set[v0_5.ParameterizedSize_N] = {0}
1088 else:
1089 ns = {0, 1, 2}
1091 given_batch_sizes = {
1092 a.size
1093 for ipt in model.inputs
1094 for a in ipt.axes
1095 if isinstance(a, v0_5.BatchAxis)
1096 }
1097 if given_batch_sizes:
1098 batch_sizes = {gbs for gbs in given_batch_sizes if gbs is not None}
1099 if not batch_sizes:
1100 # only arbitrary batch sizes
1101 batch_sizes = {1, 2}
1102 else:
1103 # no batch axis
1104 batch_sizes = {1}
1106 test_cases: Set[Tuple[BatchSize, v0_5.ParameterizedSize_N]] = {
1107 (b, n) for b, n in product(sorted(batch_sizes), sorted(ns))
1108 }
1109 logger.info(
1110 "Testing inference with '{}' for {} different inputs (B, N): {}",
1111 weight_format,
1112 len(test_cases),
1113 test_cases,
1114 )
1116 def generate_test_cases():
1117 tested: Set[Hashable] = set()
1119 def get_ns(n: int):
1120 return {
1121 (t.id, a.id): n
1122 for t in model.inputs
1123 for a in t.axes
1124 if isinstance(a.size, v0_5.ParameterizedSize)
1125 }
1127 for batch_size, n in sorted(test_cases):
1128 input_target_sizes, expected_output_sizes = model.get_axis_sizes(
1129 get_ns(n), batch_size=batch_size
1130 )
1131 hashable_target_size = tuple(
1132 (k, input_target_sizes[k]) for k in sorted(input_target_sizes)
1133 )
1134 if hashable_target_size in tested:
1135 continue
1136 else:
1137 tested.add(hashable_target_size)
1139 resized_test_inputs = Sample(
1140 members={
1141 t.id: (
1142 test_input.members[t.id].resize_to(
1143 {
1144 aid: s
1145 for (tid, aid), s in input_target_sizes.items()
1146 if tid == t.id
1147 },
1148 )
1149 )
1150 for t in model.inputs
1151 },
1152 stat=test_input.stat,
1153 id=test_input.id,
1154 )
1155 expected_output_shapes = {
1156 t.id: {
1157 aid: s
1158 for (tid, aid), s in expected_output_sizes.items()
1159 if tid == t.id
1160 }
1161 for t in model.outputs
1162 }
1163 yield n, batch_size, resized_test_inputs, expected_output_shapes
1165 try:
1166 test_input = get_test_input_sample(model)
1168 with create_prediction_pipeline(
1169 bioimageio_model=model, devices=devices, weight_format=weight_format
1170 ) as prediction_pipeline:
1171 for n, batch_size, inputs, exptected_output_shape in generate_test_cases():
1172 error: Optional[str] = None
1173 try:
1174 result = prediction_pipeline.predict_sample_without_blocking(
1175 inputs, skip_input_padding=True, skip_output_cropping=True
1176 )
1177 except Exception as e:
1178 error = str(e)
1179 else:
1180 if len(result.members) != len(exptected_output_shape):
1181 error = (
1182 f"Expected {len(exptected_output_shape)} outputs,"
1183 + f" but got {len(result.members)}"
1184 )
1186 else:
1187 for m, exp in exptected_output_shape.items():
1188 res = result.members.get(m)
1189 if res is None:
1190 error = "Output tensors may not be None for test case"
1191 break
1193 diff: Dict[AxisId, int] = {}
1194 for a, s in res.sizes.items():
1195 if isinstance((e_aid := exp[AxisId(a)]), int):
1196 if s != e_aid:
1197 diff[AxisId(a)] = s
1198 elif (
1199 s < e_aid.min
1200 or e_aid.max is not None
1201 and s > e_aid.max
1202 ):
1203 diff[AxisId(a)] = s
1204 if diff:
1205 error = (
1206 f"(n={n}) Expected output shape {exp},"
1207 + f" but got {res.sizes} (diff: {diff})"
1208 )
1209 break
1211 model.validation_summary.add_detail(
1212 ValidationDetail(
1213 name=f"Run {weight_format} inference for inputs with"
1214 + f" batch_size: {batch_size} and size parameter n: {n}",
1215 loc=("weights", weight_format),
1216 status="passed" if error is None else "failed",
1217 errors=(
1218 []
1219 if error is None
1220 else [
1221 ErrorEntry(
1222 loc=("weights", weight_format),
1223 msg=error,
1224 type="bioimageio.core",
1225 )
1226 ]
1227 ),
1228 )
1229 )
1230 if stop_early and error is not None:
1231 break
1232 except Exception as e:
1233 if get_validation_context().raise_errors:
1234 raise e
1236 model.validation_summary.add_detail(
1237 ValidationDetail(
1238 name=f"Run {weight_format} inference for parametrized inputs",
1239 status="failed",
1240 loc=("weights", weight_format),
1241 errors=[
1242 ErrorEntry(
1243 loc=("weights", weight_format),
1244 msg=str(e),
1245 type="bioimageio.core",
1246 with_traceback=True,
1247 )
1248 ],
1249 )
1250 )
1253def _test_expected_resource_type(
1254 rd: Union[InvalidDescr, ResourceDescr], expected_type: str
1255):
1256 has_expected_type = rd.type is expected_type
1257 rd.validation_summary.details.append(
1258 ValidationDetail(
1259 name="Has expected resource type",
1260 status="passed" if has_expected_type else "failed",
1261 loc=("type",),
1262 errors=(
1263 []
1264 if has_expected_type
1265 else [
1266 ErrorEntry(
1267 loc=("type",),
1268 type="type",
1269 msg=f"Expected type {expected_type}, found {rd.type}",
1270 )
1271 ]
1272 ),
1273 )
1274 )
1275 return has_expected_type
1278# TODO: Implement `debug_model()`
1279# def debug_model(
1280# model_rdf: Union[RawResourceDescr, ResourceDescr, URI, Path, str],
1281# *,
1282# weight_format: Optional[WeightsFormat] = None,
1283# devices: Optional[List[str]] = None,
1284# ):
1285# """Run the model test and return dict with inputs, results, expected results and intermediates.
1287# Returns dict with tensors "inputs", "inputs_processed", "outputs_raw", "outputs", "expected" and "diff".
1288# """
1289# inputs_raw: Optional = None
1290# inputs_processed: Optional = None
1291# outputs_raw: Optional = None
1292# outputs: Optional = None
1293# expected: Optional = None
1294# diff: Optional = None
1296# model = load_description(
1297# model_rdf, weights_priority_order=None if weight_format is None else [weight_format]
1298# )
1299# if not isinstance(model, Model):
1300# raise ValueError(f"Not a bioimageio.model: {model_rdf}")
1302# prediction_pipeline = create_prediction_pipeline(
1303# bioimageio_model=model, devices=devices, weight_format=weight_format
1304# )
1305# inputs = [
1306# xr.DataArray(load_array(str(in_path)), dims=input_spec.axes)
1307# for in_path, input_spec in zip(model.test_inputs, model.inputs)
1308# ]
1309# input_dict = {input_spec.name: input for input_spec, input in zip(model.inputs, inputs)}
1311# # keep track of the non-processed inputs
1312# inputs_raw = [deepcopy(input) for input in inputs]
1314# computed_measures = {}
1316# prediction_pipeline.apply_preprocessing(input_dict, computed_measures)
1317# inputs_processed = list(input_dict.values())
1318# outputs_raw = prediction_pipeline.predict(*inputs_processed)
1319# output_dict = {output_spec.name: deepcopy(output) for output_spec, output in zip(model.outputs, outputs_raw)}
1320# prediction_pipeline.apply_postprocessing(output_dict, computed_measures)
1321# outputs = list(output_dict.values())
1323# if isinstance(outputs, (np.ndarray, xr.DataArray)):
1324# outputs = [outputs]
1326# expected = [
1327# xr.DataArray(load_array(str(out_path)), dims=output_spec.axes)
1328# for out_path, output_spec in zip(model.test_outputs, model.outputs)
1329# ]
1330# if len(outputs) != len(expected):
1331# error = f"Number of outputs and number of expected outputs disagree: {len(outputs)} != {len(expected)}"
1332# print(error)
1333# else:
1334# diff = []
1335# for res, exp in zip(outputs, expected):
1336# diff.append(res - exp)
1338# return {
1339# "inputs": inputs_raw,
1340# "inputs_processed": inputs_processed,
1341# "outputs_raw": outputs_raw,
1342# "outputs": outputs,
1343# "expected": expected,
1344# "diff": diff,
1345# }