Coverage for src / bioimageio / spec / summary.py: 66%
397 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-03-31 13:09 +0000
« prev ^ index » next coverage.py v7.13.5, created at 2026-03-31 13:09 +0000
1"""Utilities for summarizing and formatting BioImage.IO validation results.
3This module defines data structures to capture validation errors, warnings,
4and summaries for BioImage.IO resource descriptions, along with helpers to
5format these results as plain text, Markdown, or HTML for reporting and
6diagnostics.
7"""
9import html
10import os
11import platform
12import subprocess
13from dataclasses import dataclass
14from datetime import datetime, timezone
15from io import StringIO
16from itertools import chain
17from pathlib import Path
18from tempfile import TemporaryDirectory
19from textwrap import TextWrapper
20from types import MappingProxyType
21from typing import (
22 Any,
23 Callable,
24 Dict,
25 List,
26 Literal,
27 Mapping,
28 NamedTuple,
29 Optional,
30 Sequence,
31 Set,
32 Tuple,
33 Union,
34)
36import annotated_types
37import markdown
38import rich.console
39import rich.markdown
40import rich.traceback
41from loguru import logger
42from pydantic import (
43 BaseModel,
44 Field,
45 field_serializer,
46 field_validator,
47 model_validator,
48)
49from pydantic_core.core_schema import ErrorType
50from typing_extensions import Annotated, Self, assert_never, cast
52from ._internal.io import is_yaml_value
53from ._internal.io_utils import write_yaml
54from ._internal.type_guards import is_dict
55from ._internal.validation_context import ValidationContextSummary
56from ._internal.version_type import Version
57from ._internal.warning_levels import (
58 ALERT,
59 ALERT_NAME,
60 ERROR,
61 ERROR_NAME,
62 INFO,
63 INFO_NAME,
64 WARNING,
65 WARNING_NAME,
66 WarningLevel,
67 WarningSeverity,
68)
69from ._version import VERSION
70from .conda_env import CondaEnv
72CONDA_CMD = "conda.bat" if platform.system() == "Windows" else "conda"
74Loc = Tuple[Union[int, str], ...]
75"""location of error/warning in a nested data structure"""
77WarningSeverityName = Literal["info", "warning", "alert"]
78WarningLevelName = Literal[WarningSeverityName, "error"]
80WARNING_SEVERITY_TO_NAME: Mapping[WarningSeverity, WarningSeverityName] = (
81 MappingProxyType({INFO: INFO_NAME, WARNING: WARNING_NAME, ALERT: ALERT_NAME})
82)
83WARNING_LEVEL_TO_NAME: Mapping[WarningLevel, WarningLevelName] = MappingProxyType(
84 {INFO: INFO_NAME, WARNING: WARNING_NAME, ALERT: ALERT_NAME, ERROR: ERROR_NAME}
85)
86WARNING_NAME_TO_LEVEL: Mapping[WarningLevelName, WarningLevel] = MappingProxyType(
87 {v: k for k, v in WARNING_LEVEL_TO_NAME.items()}
88)
91class ValidationEntry(BaseModel):
92 """Base of `ErrorEntry` and `WarningEntry`"""
94 loc: Loc
95 msg: str
96 type: Union[ErrorType, str]
99class ErrorEntry(ValidationEntry):
100 """An error in a `ValidationDetail`"""
102 with_traceback: bool = False
103 traceback_md: str = ""
104 traceback_html: str = ""
105 # private rich traceback that is not serialized
106 _traceback_rich: Optional[rich.traceback.Traceback] = None
108 @property
109 def traceback_rich(self) -> Optional[rich.traceback.Traceback]:
110 return self._traceback_rich
112 def model_post_init(self, __context: Any):
113 if self.with_traceback and not (self.traceback_md or self.traceback_html):
114 self._traceback_rich = rich.traceback.Traceback()
115 console = rich.console.Console(
116 record=True,
117 file=open(os.devnull, "wt", encoding="utf-8"),
118 color_system="truecolor",
119 width=120,
120 tab_size=4,
121 soft_wrap=True,
122 )
123 console.print(self._traceback_rich)
124 if not self.traceback_md:
125 self.traceback_md = console.export_text(clear=False)
127 if not self.traceback_html:
128 self.traceback_html = console.export_html(clear=False)
131class WarningEntry(ValidationEntry):
132 """A warning in a `ValidationDetail`"""
134 severity: WarningSeverity = WARNING
136 @property
137 def severity_name(self) -> WarningSeverityName:
138 return WARNING_SEVERITY_TO_NAME[self.severity]
141def format_loc(
142 loc: Loc, target: Union[Literal["md", "html", "plain"], rich.console.Console]
143) -> str:
144 """helper to format a location tuple **loc**"""
145 loc_str = ".".join(f"({x})" if x[0].isupper() else x for x in map(str, loc))
147 # additional field validation can make the location information quite convoluted, e.g.
148 # `weights.pytorch_state_dict.dependencies.source.function-after[validate_url_ok(), url['http','https']]` Input should be a valid URL, relative URL without a base
149 # therefore we remove the `.function-after[validate_url_ok(), url['http','https']]` here
150 loc_str, *_ = loc_str.split(".function-after")
151 if loc_str:
152 if target == "md" or isinstance(target, rich.console.Console):
153 start = "`"
154 end = "`"
155 elif target == "html":
156 start = "<code>"
157 end = "</code>"
158 elif target == "plain":
159 start = ""
160 end = ""
161 else:
162 assert_never(target)
164 return f"{start}{loc_str}{end}"
165 else:
166 return ""
169class InstalledPackage(NamedTuple):
170 name: str
171 version: str
172 build: str = ""
173 channel: str = ""
176class ValidationDetail(BaseModel, extra="allow"):
177 """a detail in a validation summary"""
179 name: str
180 status: Literal["passed", "failed"]
181 loc: Loc = ()
182 """location in the RDF that this detail applies to"""
183 errors: List[ErrorEntry] = Field(
184 default_factory=cast(Callable[[], List[ErrorEntry]], list)
185 )
186 warnings: List[WarningEntry] = Field(
187 default_factory=cast(Callable[[], List[WarningEntry]], list)
188 )
190 context: Optional[ValidationContextSummary] = None
192 recommended_env: Optional[CondaEnv] = None
193 """recommended conda environemnt for this validation detail"""
195 saved_conda_compare: Optional[str] = None
196 """output of `conda compare <recommended env>`"""
198 @field_serializer("saved_conda_compare")
199 def _save_conda_compare(self, value: Optional[str]):
200 return self.conda_compare
202 @model_validator(mode="before")
203 def _load_legacy(cls, data: Any):
204 if is_dict(data):
205 field_name = "conda_compare"
206 if (
207 field_name in data
208 and (saved_field_name := f"saved_{field_name}") not in data
209 ):
210 data[saved_field_name] = data.pop(field_name)
212 return data
214 @property
215 def conda_compare(self) -> Optional[str]:
216 if self.recommended_env is None:
217 return None
219 if self.saved_conda_compare is None:
220 dumped_env = self.recommended_env.model_dump(mode="json")
221 if is_yaml_value(dumped_env):
222 with TemporaryDirectory() as d:
223 path = Path(d) / "env.yaml"
224 with path.open("w", encoding="utf-8") as f:
225 write_yaml(dumped_env, f)
227 try:
228 compare_proc = subprocess.run(
229 [CONDA_CMD, "compare", str(path)],
230 stdout=subprocess.PIPE,
231 stderr=subprocess.STDOUT,
232 shell=False,
233 text=True,
234 )
235 except Exception as e:
236 self.saved_conda_compare = f"Failed to run `conda compare`: {e}"
237 else:
238 self.saved_conda_compare = (
239 compare_proc.stdout
240 or f"`conda compare` exited with {compare_proc.returncode}"
241 )
242 else:
243 self.saved_conda_compare = (
244 "Failed to dump recommended env to valid yaml"
245 )
247 return self.saved_conda_compare
249 @property
250 def status_icon(self) -> str:
251 if self.status == "passed":
252 return "✔️"
253 else:
254 return "❌"
257class ValidationSummary(BaseModel, extra="allow"):
258 """Summarizes output of all bioimageio validations and tests
259 for one specific `ResourceDescr` instance."""
261 name: str
262 """Name of the validation"""
264 source_name: str
265 """Source of the validated bioimageio description"""
267 id: Optional[str] = None
268 """ID of the validated resource"""
270 version: Optional[Version] = None
271 """Version of the validated resource"""
273 type: str
274 """Type of the validated resource"""
276 format_version: str
277 """Format version of the validated resource"""
279 status: Literal["passed", "valid-format", "failed"]
280 """Overall status of the bioimageio validation"""
282 metadata_completeness: Annotated[float, annotated_types.Interval(ge=0, le=1)] = 0.0
283 """Estimate of completeness of the metadata in the resource description.
285 Note: This completeness estimate may change with subsequent releases
286 and should be considered bioimageio.spec version specific.
287 """
289 details: List[ValidationDetail]
290 """List of validation details"""
291 env: Set[InstalledPackage] = Field(
292 default_factory=lambda: {
293 InstalledPackage(
294 name="bioimageio.spec",
295 version=VERSION,
296 )
297 }
298 )
299 """List of selected, relevant package versions"""
301 saved_conda_list: Optional[str] = None
303 @field_serializer("saved_conda_list")
304 def _save_conda_list(self, value: Optional[str]):
305 return self.conda_list
307 @property
308 def conda_list(self) -> str:
309 if self.saved_conda_list is None:
310 try:
311 p = subprocess.run(
312 [CONDA_CMD, "list"],
313 stdout=subprocess.PIPE,
314 stderr=subprocess.STDOUT,
315 shell=False,
316 text=True,
317 )
318 except Exception as e:
319 self.saved_conda_list = f"Failed to run `conda list`: {e}"
320 else:
321 self.saved_conda_list = (
322 p.stdout or f"`conda list` exited with {p.returncode}"
323 )
325 return self.saved_conda_list
327 @property
328 def status_icon(self) -> str:
329 if self.status == "passed":
330 return "✔️"
331 elif self.status == "valid-format":
332 return "🟡"
333 else:
334 return "❌"
336 @property
337 def errors(self) -> List[ErrorEntry]:
338 return list(chain.from_iterable(d.errors for d in self.details))
340 @property
341 def warnings(self) -> List[WarningEntry]:
342 return list(chain.from_iterable(d.warnings for d in self.details))
344 def format(
345 self,
346 *,
347 width: Optional[int] = None,
348 include_conda_list: bool = False,
349 ) -> str:
350 """Format summary as Markdown string (alias to `format_md`)"""
351 return self.format_md(width=width, include_conda_list=include_conda_list)
353 def format_md(
354 self,
355 *,
356 width: Optional[int] = None,
357 include_conda_list: bool = False,
358 ) -> str:
359 """Format summary as Markdown string"""
360 return self._format(
361 width=width, target="md", include_conda_list=include_conda_list
362 )
364 def format_html(
365 self,
366 *,
367 width: Optional[int] = None,
368 include_conda_list: bool = False,
369 ) -> str:
370 md_with_html = self._format(
371 target="html", width=width, include_conda_list=include_conda_list
372 )
373 return markdown.markdown(
374 md_with_html, extensions=["tables", "fenced_code", "nl2br"]
375 )
377 def display(
378 self,
379 *,
380 width: Optional[int] = None,
381 include_conda_list: bool = False,
382 tab_size: int = 4,
383 soft_wrap: bool = True,
384 ) -> None:
385 try: # render as HTML in Jupyter notebook
386 from IPython.core.getipython import get_ipython
387 from IPython.display import (
388 display_html, # pyright: ignore[reportUnknownVariableType]
389 )
390 except ImportError:
391 pass
392 else:
393 if get_ipython() is not None:
394 _ = display_html(
395 self.format_html(
396 width=width, include_conda_list=include_conda_list
397 ),
398 raw=True,
399 )
400 return
402 # render with rich
403 _ = self._format(
404 target=rich.console.Console(
405 width=width,
406 tab_size=tab_size,
407 soft_wrap=soft_wrap,
408 ),
409 width=width,
410 include_conda_list=include_conda_list,
411 )
413 def add_detail(self, detail: ValidationDetail, update_status: bool = True) -> None:
414 if update_status:
415 if self.status == "valid-format" and detail.status == "passed":
416 # once status is 'valid-format' we can only improve to 'passed'
417 self.status = "passed"
418 elif self.status == "passed" and detail.status == "failed":
419 # once status is 'passed' it can only degrade to 'valid-format'
420 self.status = "valid-format"
421 # once format is 'failed' it cannot improve
423 self.details.append(detail)
425 def log(
426 self,
427 to: Union[Literal["display"], Path, Sequence[Union[Literal["display"], Path]]],
428 ) -> List[Path]:
429 """Convenience method to display the validation summary in the terminal and/or
430 save it to disk. See `save` for details."""
431 if to == "display":
432 display = True
433 save_to = []
434 elif isinstance(to, Path):
435 display = False
436 save_to = [to]
437 else:
438 display = "display" in to
439 save_to = [p for p in to if p != "display"]
441 if display:
442 self.display()
444 return self.save(save_to)
446 def save(
447 self, path: Union[Path, Sequence[Path]] = Path("{id}_summary_{now}")
448 ) -> List[Path]:
449 """Save the validation/test summary in JSON, Markdown or HTML format.
451 Returns:
452 List of file paths the summary was saved to.
454 Notes:
455 - Format is chosen based on the suffix: `.json`, `.md`, `.html`.
456 - If **path** has no suffix it is assumed to be a direcotry to which a
457 `summary.json`, `summary.md` and `summary.html` are saved to.
458 """
459 if isinstance(path, (str, Path)):
460 path = [Path(path)]
462 # folder to file paths
463 file_paths: List[Path] = []
464 for p in path:
465 if p.suffix:
466 file_paths.append(p)
467 else:
468 file_paths.extend(
469 [
470 p / "summary.json",
471 p / "summary.md",
472 p / "summary.html",
473 ]
474 )
476 now = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
477 for p in file_paths:
478 p = Path(str(p).format(id=self.id or "bioimageio", now=now))
479 if p.suffix == ".json":
480 self.save_json(p)
481 elif p.suffix == ".md":
482 self.save_markdown(p)
483 elif p.suffix == ".html":
484 self.save_html(p)
485 else:
486 raise ValueError(f"Unknown summary path suffix '{p.suffix}'")
488 return file_paths
490 def save_json(
491 self, path: Path = Path("summary.json"), *, indent: Optional[int] = 2
492 ) -> None:
493 """Save validation/test summary as JSON file."""
494 json_str = self.model_dump_json(indent=indent)
495 path.parent.mkdir(exist_ok=True, parents=True)
496 _ = path.write_text(json_str, encoding="utf-8")
497 logger.info("Saved summary to {}", path.absolute())
499 def save_markdown(self, path: Path = Path("summary.md")) -> None:
500 """Save rendered validation/test summary as Markdown file."""
501 formatted = self.format_md()
502 path.parent.mkdir(exist_ok=True, parents=True)
503 _ = path.write_text(formatted, encoding="utf-8")
504 logger.info("Saved Markdown formatted summary to {}", path.absolute())
506 def save_html(self, path: Path = Path("summary.html")) -> None:
507 """Save rendered validation/test summary as HTML file."""
508 path.parent.mkdir(exist_ok=True, parents=True)
510 html = self.format_html()
511 _ = path.write_text(html, encoding="utf-8")
512 logger.info("Saved HTML formatted summary to {}", path.absolute())
514 @classmethod
515 def load_json(cls, path: Path) -> Self:
516 """Load validation/test summary from a suitable JSON file"""
517 json_str = Path(path).read_text(encoding="utf-8")
518 return cls.model_validate_json(json_str)
520 @field_validator("env", mode="before")
521 def _convert_old_env(cls, value: List[Union[List[str], Dict[str, str]]]):
522 """convert old style dict values of `env` for backwards compatibility"""
523 if isinstance(value, list):
524 return [
525 (
526 (v["name"], v["version"], v.get("build", ""), v.get("channel", ""))
527 if isinstance(v, dict) and "name" in v and "version" in v
528 else v
529 )
530 for v in value
531 ]
532 else:
533 return value
535 def _format(
536 self,
537 *,
538 target: Union[rich.console.Console, Literal["html", "md"]],
539 width: Optional[int],
540 include_conda_list: bool,
541 ) -> str:
542 return _format_summary(
543 self,
544 target=target,
545 width=width or 100,
546 include_conda_list=include_conda_list,
547 )
550def _format_summary(
551 summary: ValidationSummary,
552 *,
553 hide_tracebacks: bool = False, # TODO: remove?
554 hide_source: bool = False, # TODO: remove?
555 hide_env: bool = False, # TODO: remove?
556 target: Union[rich.console.Console, Literal["html", "md"]] = "md",
557 include_conda_list: bool,
558 width: int,
559) -> str:
560 parts: List[str] = []
561 format_table = _format_html_table if target == "html" else _format_md_table
562 details_below: Dict[str, Union[str, Tuple[str, rich.traceback.Traceback]]] = {}
563 left_out_details: int = 0
564 left_out_details_header = "Left out details"
566 def add_part(part: str):
567 parts.append(part)
568 if isinstance(target, rich.console.Console):
569 target.print(rich.markdown.Markdown(part))
571 def add_section(header: str):
572 if target == "md" or isinstance(target, rich.console.Console):
573 add_part(f"\n### {header}\n")
574 elif target == "html":
575 parts.append(f'<h3 id="{header_to_tag(header)}">{header}</h3>')
576 else:
577 assert_never(target)
579 def header_to_tag(header: str):
580 return (
581 header.replace("`", "")
582 .replace("(", "")
583 .replace(")", "")
584 .replace(" ", "-")
585 .lower()
586 )
588 def add_as_details_below(
589 title: str, text: Union[str, Tuple[str, rich.traceback.Traceback]]
590 ):
591 """returns a header and its tag to link to details below"""
593 def make_link(header: str):
594 tag = header_to_tag(header)
595 if target == "md":
596 return f"[{header}](#{tag})"
597 elif target == "html":
598 return f'<a href="#{tag}">{header}</a>'
599 elif isinstance(target, rich.console.Console):
600 return f"{header} below"
601 else:
602 assert_never(target)
604 for n in range(1, 4):
605 header = f"{title} {n}"
606 if header in details_below:
607 if details_below[header] == text:
608 return make_link(header)
609 else:
610 details_below[header] = text
611 return make_link(header)
613 nonlocal left_out_details
614 left_out_details += 1
615 return make_link(left_out_details_header)
617 @dataclass
618 class CodeCell:
619 text: str
621 @dataclass
622 class CodeRef:
623 text: str
625 def format_code(
626 code: str,
627 lang: str = "",
628 title: str = "Details",
629 cell_line_limit: int = 15,
630 cell_width_limit: int = 120,
631 ) -> Union[CodeRef, CodeCell]:
632 if not code.strip():
633 return CodeCell("")
635 if target == "html":
636 html_lang = f' lang="{lang}"' if lang else ""
637 code = f"<pre{html_lang}>{code}</pre>"
638 put_below = (
639 code.count("\n") > cell_line_limit
640 or max(map(len, code.split("\n"))) > cell_width_limit
641 )
642 else:
643 put_below = True
644 code = f"\n```{lang}\n{code}\n```\n"
646 if put_below:
647 link = add_as_details_below(title, code)
648 return CodeRef(f"See {link}.")
649 else:
650 return CodeCell(code)
652 def format_traceback(entry: ErrorEntry):
653 if isinstance(target, rich.console.Console):
654 if entry.traceback_rich is None:
655 return format_code(entry.traceback_md, title="Traceback")
656 else:
657 link = add_as_details_below(
658 "Traceback", (entry.traceback_md, entry.traceback_rich)
659 )
660 return CodeRef(f"See {link}.")
662 if target == "md":
663 return format_code(entry.traceback_md, title="Traceback")
664 elif target == "html":
665 return format_code(entry.traceback_html, title="Traceback")
666 else:
667 assert_never(target)
669 def format_text(text: str):
670 if target == "html":
671 return [f"<pre>{text}</pre>"]
672 else:
673 return text.split("\n")
675 def get_info_table():
676 info_rows = [
677 [summary.status_icon, summary.name.strip(".").strip()],
678 ["status", summary.status],
679 ]
680 if not hide_source:
681 info_rows.append(["source", html.escape(summary.source_name)])
683 if summary.id is not None:
684 info_rows.append(["id", summary.id])
686 if summary.version is not None:
687 info_rows.append(["version", str(summary.version)])
689 info_rows.append(["applied format", f"{summary.type} {summary.format_version}"])
690 if not hide_env:
691 info_rows.extend([[e.name, e.version] for e in sorted(summary.env)])
693 if include_conda_list:
694 info_rows.append(
695 ["conda list", format_code(summary.conda_list, title="Conda List").text]
696 )
697 return format_table(info_rows)
699 def get_details_table():
700 details = [["", "Location", "Details"]]
702 def append_detail(
703 status: str, loc: Loc, text: str, code: Union[CodeRef, CodeCell, None]
704 ):
705 text_lines = format_text(text)
706 status_lines = [""] * len(text_lines)
707 loc_lines = [""] * len(text_lines)
708 status_lines[0] = status
709 loc_lines[0] = format_loc(loc, target)
710 for s_line, loc_line, text_line in zip(status_lines, loc_lines, text_lines):
711 details.append([s_line, loc_line, text_line])
713 if code is not None:
714 details.append(["", "", code.text])
716 for d in summary.details:
717 details.append([d.status_icon, format_loc(d.loc, target), d.name])
719 for entry in d.errors:
720 append_detail(
721 "❌",
722 entry.loc,
723 entry.msg,
724 None if hide_tracebacks else format_traceback(entry),
725 )
727 for entry in d.warnings:
728 append_detail(
729 "⚠" if entry.severity > INFO else "ℹ", entry.loc, entry.msg, None
730 )
732 if d.recommended_env is not None:
733 rec_env = StringIO()
734 json_env = d.recommended_env.model_dump(
735 mode="json", exclude_defaults=True
736 )
737 assert is_yaml_value(json_env)
738 write_yaml(json_env, rec_env)
739 append_detail(
740 "",
741 d.loc,
742 f"recommended conda environment ({d.name})",
743 format_code(
744 rec_env.getvalue(),
745 lang="yaml",
746 title="Recommended Conda Environment",
747 ),
748 )
750 if d.conda_compare:
751 wrapped_conda_compare = "\n".join(
752 TextWrapper(width=width - 4).wrap(d.conda_compare)
753 )
754 append_detail(
755 "",
756 d.loc,
757 f"conda compare ({d.name})",
758 format_code(
759 wrapped_conda_compare,
760 title="Conda Environment Comparison",
761 ),
762 )
764 return format_table(details)
766 add_part(get_info_table())
767 add_part(get_details_table())
769 for header, text in details_below.items():
770 add_section(header)
771 if isinstance(text, tuple):
772 assert isinstance(target, rich.console.Console)
773 text, rich_obj = text
774 target.print(rich_obj)
775 parts.append(f"{text}\n")
776 else:
777 add_part(f"{text}\n")
779 if left_out_details:
780 parts.append(
781 f"\n{left_out_details_header}\nLeft out {left_out_details} more details for brevity.\n"
782 )
784 return "".join(parts)
787def _format_md_table(rows: List[List[str]]) -> str:
788 """format `rows` as markdown table"""
789 n_cols = len(rows[0])
790 assert all(len(row) == n_cols for row in rows)
791 col_widths = [max(max(len(row[i]) for row in rows), 3) for i in range(n_cols)]
793 # fix new lines in table cell
794 rows = [[line.replace("\n", "<br>") for line in r] for r in rows]
796 lines = [" | ".join(rows[0][i].center(col_widths[i]) for i in range(n_cols))]
797 lines.append(" | ".join("---".center(col_widths[i]) for i in range(n_cols)))
798 lines.extend(
799 [
800 " | ".join(row[i].ljust(col_widths[i]) for i in range(n_cols))
801 for row in rows[1:]
802 ]
803 )
804 return "\n| " + " |\n| ".join(lines) + " |\n"
807def _format_html_table(rows: List[List[str]]) -> str:
808 """format `rows` as HTML table"""
810 def get_line(cells: List[str], cell_tag: Literal["th", "td"] = "td"):
811 return (
812 [" <tr>"]
813 + [
814 f' <{cell_tag} style="text-align:{"center" if cell_tag == "th" else "left"}">{c}</{cell_tag}>'
815 for c in cells
816 ]
817 + [" </tr>"]
818 )
820 table = ["<table>"] + get_line(rows[0], cell_tag="th")
821 for r in rows[1:]:
822 table.extend(get_line(r))
824 table.append("</table>")
826 return "\n".join(table)