Coverage for src / bioimageio / spec / summary.py: 65%
397 statements
« prev ^ index » next coverage.py v7.13.4, created at 2026-02-23 10:51 +0000
« prev ^ index » next coverage.py v7.13.4, created at 2026-02-23 10:51 +0000
1"""Utilities for summarizing and formatting BioImage.IO validation results.
3This module defines data structures to capture validation errors, warnings,
4and summaries for BioImage.IO resource descriptions, along with helpers to
5format these results as plain text, Markdown, or HTML for reporting and
6diagnostics.
7"""
9import html
10import os
11import platform
12import subprocess
13from dataclasses import dataclass
14from datetime import datetime, timezone
15from io import StringIO
16from itertools import chain
17from pathlib import Path
18from tempfile import TemporaryDirectory
19from textwrap import TextWrapper
20from types import MappingProxyType
21from typing import (
22 Any,
23 Dict,
24 List,
25 Literal,
26 Mapping,
27 NamedTuple,
28 Optional,
29 Sequence,
30 Set,
31 Tuple,
32 Union,
33)
35import annotated_types
36import markdown
37import rich.console
38import rich.markdown
39import rich.traceback
40from loguru import logger
41from pydantic import (
42 BaseModel,
43 Field,
44 field_serializer,
45 field_validator,
46 model_validator,
47)
48from pydantic_core.core_schema import ErrorType
49from typing_extensions import Annotated, Self, assert_never
51from ._internal.io import is_yaml_value
52from ._internal.io_utils import write_yaml
53from ._internal.type_guards import is_dict
54from ._internal.validation_context import ValidationContextSummary
55from ._internal.version_type import Version
56from ._internal.warning_levels import (
57 ALERT,
58 ALERT_NAME,
59 ERROR,
60 ERROR_NAME,
61 INFO,
62 INFO_NAME,
63 WARNING,
64 WARNING_NAME,
65 WarningLevel,
66 WarningSeverity,
67)
68from ._version import VERSION
69from .conda_env import CondaEnv
71CONDA_CMD = "conda.bat" if platform.system() == "Windows" else "conda"
73Loc = Tuple[Union[int, str], ...]
74"""location of error/warning in a nested data structure"""
76WarningSeverityName = Literal["info", "warning", "alert"]
77WarningLevelName = Literal[WarningSeverityName, "error"]
79WARNING_SEVERITY_TO_NAME: Mapping[WarningSeverity, WarningSeverityName] = (
80 MappingProxyType({INFO: INFO_NAME, WARNING: WARNING_NAME, ALERT: ALERT_NAME})
81)
82WARNING_LEVEL_TO_NAME: Mapping[WarningLevel, WarningLevelName] = MappingProxyType(
83 {INFO: INFO_NAME, WARNING: WARNING_NAME, ALERT: ALERT_NAME, ERROR: ERROR_NAME}
84)
85WARNING_NAME_TO_LEVEL: Mapping[WarningLevelName, WarningLevel] = MappingProxyType(
86 {v: k for k, v in WARNING_LEVEL_TO_NAME.items()}
87)
90class ValidationEntry(BaseModel):
91 """Base of `ErrorEntry` and `WarningEntry`"""
93 loc: Loc
94 msg: str
95 type: Union[ErrorType, str]
98class ErrorEntry(ValidationEntry):
99 """An error in a `ValidationDetail`"""
101 with_traceback: bool = False
102 traceback_md: str = ""
103 traceback_html: str = ""
104 # private rich traceback that is not serialized
105 _traceback_rich: Optional[rich.traceback.Traceback] = None
107 @property
108 def traceback_rich(self) -> Optional[rich.traceback.Traceback]:
109 return self._traceback_rich
111 def model_post_init(self, __context: Any):
112 if self.with_traceback and not (self.traceback_md or self.traceback_html):
113 self._traceback_rich = rich.traceback.Traceback()
114 console = rich.console.Console(
115 record=True,
116 file=open(os.devnull, "wt", encoding="utf-8"),
117 color_system="truecolor",
118 width=120,
119 tab_size=4,
120 soft_wrap=True,
121 )
122 console.print(self._traceback_rich)
123 if not self.traceback_md:
124 self.traceback_md = console.export_text(clear=False)
126 if not self.traceback_html:
127 self.traceback_html = console.export_html(clear=False)
130class WarningEntry(ValidationEntry):
131 """A warning in a `ValidationDetail`"""
133 severity: WarningSeverity = WARNING
135 @property
136 def severity_name(self) -> WarningSeverityName:
137 return WARNING_SEVERITY_TO_NAME[self.severity]
140def format_loc(
141 loc: Loc, target: Union[Literal["md", "html", "plain"], rich.console.Console]
142) -> str:
143 """helper to format a location tuple **loc**"""
144 loc_str = ".".join(f"({x})" if x[0].isupper() else x for x in map(str, loc))
146 # additional field validation can make the location information quite convoluted, e.g.
147 # `weights.pytorch_state_dict.dependencies.source.function-after[validate_url_ok(), url['http','https']]` Input should be a valid URL, relative URL without a base
148 # therefore we remove the `.function-after[validate_url_ok(), url['http','https']]` here
149 loc_str, *_ = loc_str.split(".function-after")
150 if loc_str:
151 if target == "md" or isinstance(target, rich.console.Console):
152 start = "`"
153 end = "`"
154 elif target == "html":
155 start = "<code>"
156 end = "</code>"
157 elif target == "plain":
158 start = ""
159 end = ""
160 else:
161 assert_never(target)
163 return f"{start}{loc_str}{end}"
164 else:
165 return ""
168class InstalledPackage(NamedTuple):
169 name: str
170 version: str
171 build: str = ""
172 channel: str = ""
175class ValidationDetail(BaseModel, extra="allow"):
176 """a detail in a validation summary"""
178 name: str
179 status: Literal["passed", "failed"]
180 loc: Loc = ()
181 """location in the RDF that this detail applies to"""
182 errors: List[ErrorEntry] = Field( # pyright: ignore[reportUnknownVariableType]
183 default_factory=list
184 )
185 warnings: List[WarningEntry] = Field( # pyright: ignore[reportUnknownVariableType]
186 default_factory=list
187 )
188 context: Optional[ValidationContextSummary] = None
190 recommended_env: Optional[CondaEnv] = None
191 """recommended conda environemnt for this validation detail"""
193 saved_conda_compare: Optional[str] = None
194 """output of `conda compare <recommended env>`"""
196 @field_serializer("saved_conda_compare")
197 def _save_conda_compare(self, value: Optional[str]):
198 return self.conda_compare
200 @model_validator(mode="before")
201 def _load_legacy(cls, data: Any):
202 if is_dict(data):
203 field_name = "conda_compare"
204 if (
205 field_name in data
206 and (saved_field_name := f"saved_{field_name}") not in data
207 ):
208 data[saved_field_name] = data.pop(field_name)
210 return data
212 @property
213 def conda_compare(self) -> Optional[str]:
214 if self.recommended_env is None:
215 return None
217 if self.saved_conda_compare is None:
218 dumped_env = self.recommended_env.model_dump(mode="json")
219 if is_yaml_value(dumped_env):
220 with TemporaryDirectory() as d:
221 path = Path(d) / "env.yaml"
222 with path.open("w", encoding="utf-8") as f:
223 write_yaml(dumped_env, f)
225 try:
226 compare_proc = subprocess.run(
227 [CONDA_CMD, "compare", str(path)],
228 stdout=subprocess.PIPE,
229 stderr=subprocess.STDOUT,
230 shell=False,
231 text=True,
232 )
233 except Exception as e:
234 self.saved_conda_compare = f"Failed to run `conda compare`: {e}"
235 else:
236 self.saved_conda_compare = (
237 compare_proc.stdout
238 or f"`conda compare` exited with {compare_proc.returncode}"
239 )
240 else:
241 self.saved_conda_compare = (
242 "Failed to dump recommended env to valid yaml"
243 )
245 return self.saved_conda_compare
247 @property
248 def status_icon(self) -> str:
249 if self.status == "passed":
250 return "✔️"
251 else:
252 return "❌"
255class ValidationSummary(BaseModel, extra="allow"):
256 """Summarizes output of all bioimageio validations and tests
257 for one specific `ResourceDescr` instance."""
259 name: str
260 """Name of the validation"""
262 source_name: str
263 """Source of the validated bioimageio description"""
265 id: Optional[str] = None
266 """ID of the validated resource"""
268 version: Optional[Version] = None
269 """Version of the validated resource"""
271 type: str
272 """Type of the validated resource"""
274 format_version: str
275 """Format version of the validated resource"""
277 status: Literal["passed", "valid-format", "failed"]
278 """Overall status of the bioimageio validation"""
280 metadata_completeness: Annotated[float, annotated_types.Interval(ge=0, le=1)] = 0.0
281 """Estimate of completeness of the metadata in the resource description.
283 Note: This completeness estimate may change with subsequent releases
284 and should be considered bioimageio.spec version specific.
285 """
287 details: List[ValidationDetail]
288 """List of validation details"""
289 env: Set[InstalledPackage] = Field(
290 default_factory=lambda: {
291 InstalledPackage(
292 name="bioimageio.spec",
293 version=VERSION,
294 )
295 }
296 )
297 """List of selected, relevant package versions"""
299 saved_conda_list: Optional[str] = None
301 @field_serializer("saved_conda_list")
302 def _save_conda_list(self, value: Optional[str]):
303 return self.conda_list
305 @property
306 def conda_list(self) -> str:
307 if self.saved_conda_list is None:
308 try:
309 p = subprocess.run(
310 [CONDA_CMD, "list"],
311 stdout=subprocess.PIPE,
312 stderr=subprocess.STDOUT,
313 shell=False,
314 text=True,
315 )
316 except Exception as e:
317 self.saved_conda_list = f"Failed to run `conda list`: {e}"
318 else:
319 self.saved_conda_list = (
320 p.stdout or f"`conda list` exited with {p.returncode}"
321 )
323 return self.saved_conda_list
325 @property
326 def status_icon(self) -> str:
327 if self.status == "passed":
328 return "✔️"
329 elif self.status == "valid-format":
330 return "🟡"
331 else:
332 return "❌"
334 @property
335 def errors(self) -> List[ErrorEntry]:
336 return list(chain.from_iterable(d.errors for d in self.details))
338 @property
339 def warnings(self) -> List[WarningEntry]:
340 return list(chain.from_iterable(d.warnings for d in self.details))
342 def format(
343 self,
344 *,
345 width: Optional[int] = None,
346 include_conda_list: bool = False,
347 ) -> str:
348 """Format summary as Markdown string (alias to `format_md`)"""
349 return self.format_md(width=width, include_conda_list=include_conda_list)
351 def format_md(
352 self,
353 *,
354 width: Optional[int] = None,
355 include_conda_list: bool = False,
356 ) -> str:
357 """Format summary as Markdown string"""
358 return self._format(
359 width=width, target="md", include_conda_list=include_conda_list
360 )
362 def format_html(
363 self,
364 *,
365 width: Optional[int] = None,
366 include_conda_list: bool = False,
367 ) -> str:
368 md_with_html = self._format(
369 target="html", width=width, include_conda_list=include_conda_list
370 )
371 return markdown.markdown(
372 md_with_html, extensions=["tables", "fenced_code", "nl2br"]
373 )
375 def display(
376 self,
377 *,
378 width: Optional[int] = None,
379 include_conda_list: bool = False,
380 tab_size: int = 4,
381 soft_wrap: bool = True,
382 ) -> None:
383 try: # render as HTML in Jupyter notebook
384 from IPython.core.getipython import get_ipython
385 from IPython.display import (
386 display_html, # pyright: ignore[reportUnknownVariableType]
387 )
388 except ImportError:
389 pass
390 else:
391 if get_ipython() is not None:
392 _ = display_html(
393 self.format_html(
394 width=width, include_conda_list=include_conda_list
395 ),
396 raw=True,
397 )
398 return
400 # render with rich
401 _ = self._format(
402 target=rich.console.Console(
403 width=width,
404 tab_size=tab_size,
405 soft_wrap=soft_wrap,
406 ),
407 width=width,
408 include_conda_list=include_conda_list,
409 )
411 def add_detail(self, detail: ValidationDetail, update_status: bool = True) -> None:
412 if update_status:
413 if self.status == "valid-format" and detail.status == "passed":
414 # once status is 'valid-format' we can only improve to 'passed'
415 self.status = "passed"
416 elif self.status == "passed" and detail.status == "failed":
417 # once status is 'passed' it can only degrade to 'valid-format'
418 self.status = "valid-format"
419 # once format is 'failed' it cannot improve
421 self.details.append(detail)
423 def log(
424 self,
425 to: Union[Literal["display"], Path, Sequence[Union[Literal["display"], Path]]],
426 ) -> List[Path]:
427 """Convenience method to display the validation summary in the terminal and/or
428 save it to disk. See `save` for details."""
429 if to == "display":
430 display = True
431 save_to = []
432 elif isinstance(to, Path):
433 display = False
434 save_to = [to]
435 else:
436 display = "display" in to
437 save_to = [p for p in to if p != "display"]
439 if display:
440 self.display()
442 return self.save(save_to)
444 def save(
445 self, path: Union[Path, Sequence[Path]] = Path("{id}_summary_{now}")
446 ) -> List[Path]:
447 """Save the validation/test summary in JSON, Markdown or HTML format.
449 Returns:
450 List of file paths the summary was saved to.
452 Notes:
453 - Format is chosen based on the suffix: `.json`, `.md`, `.html`.
454 - If **path** has no suffix it is assumed to be a direcotry to which a
455 `summary.json`, `summary.md` and `summary.html` are saved to.
456 """
457 if isinstance(path, (str, Path)):
458 path = [Path(path)]
460 # folder to file paths
461 file_paths: List[Path] = []
462 for p in path:
463 if p.suffix:
464 file_paths.append(p)
465 else:
466 file_paths.extend(
467 [
468 p / "summary.json",
469 p / "summary.md",
470 p / "summary.html",
471 ]
472 )
474 now = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
475 for p in file_paths:
476 p = Path(str(p).format(id=self.id or "bioimageio", now=now))
477 if p.suffix == ".json":
478 self.save_json(p)
479 elif p.suffix == ".md":
480 self.save_markdown(p)
481 elif p.suffix == ".html":
482 self.save_html(p)
483 else:
484 raise ValueError(f"Unknown summary path suffix '{p.suffix}'")
486 return file_paths
488 def save_json(
489 self, path: Path = Path("summary.json"), *, indent: Optional[int] = 2
490 ) -> None:
491 """Save validation/test summary as JSON file."""
492 json_str = self.model_dump_json(indent=indent)
493 path.parent.mkdir(exist_ok=True, parents=True)
494 _ = path.write_text(json_str, encoding="utf-8")
495 logger.info("Saved summary to {}", path.absolute())
497 def save_markdown(self, path: Path = Path("summary.md")) -> None:
498 """Save rendered validation/test summary as Markdown file."""
499 formatted = self.format_md()
500 path.parent.mkdir(exist_ok=True, parents=True)
501 _ = path.write_text(formatted, encoding="utf-8")
502 logger.info("Saved Markdown formatted summary to {}", path.absolute())
504 def save_html(self, path: Path = Path("summary.html")) -> None:
505 """Save rendered validation/test summary as HTML file."""
506 path.parent.mkdir(exist_ok=True, parents=True)
508 html = self.format_html()
509 _ = path.write_text(html, encoding="utf-8")
510 logger.info("Saved HTML formatted summary to {}", path.absolute())
512 @classmethod
513 def load_json(cls, path: Path) -> Self:
514 """Load validation/test summary from a suitable JSON file"""
515 json_str = Path(path).read_text(encoding="utf-8")
516 return cls.model_validate_json(json_str)
518 @field_validator("env", mode="before")
519 def _convert_old_env(cls, value: List[Union[List[str], Dict[str, str]]]):
520 """convert old style dict values of `env` for backwards compatibility"""
521 if isinstance(value, list):
522 return [
523 (
524 (v["name"], v["version"], v.get("build", ""), v.get("channel", ""))
525 if isinstance(v, dict) and "name" in v and "version" in v
526 else v
527 )
528 for v in value
529 ]
530 else:
531 return value
533 def _format(
534 self,
535 *,
536 target: Union[rich.console.Console, Literal["html", "md"]],
537 width: Optional[int],
538 include_conda_list: bool,
539 ) -> str:
540 return _format_summary(
541 self,
542 target=target,
543 width=width or 100,
544 include_conda_list=include_conda_list,
545 )
548def _format_summary(
549 summary: ValidationSummary,
550 *,
551 hide_tracebacks: bool = False, # TODO: remove?
552 hide_source: bool = False, # TODO: remove?
553 hide_env: bool = False, # TODO: remove?
554 target: Union[rich.console.Console, Literal["html", "md"]] = "md",
555 include_conda_list: bool,
556 width: int,
557) -> str:
558 parts: List[str] = []
559 format_table = _format_html_table if target == "html" else _format_md_table
560 details_below: Dict[str, Union[str, Tuple[str, rich.traceback.Traceback]]] = {}
561 left_out_details: int = 0
562 left_out_details_header = "Left out details"
564 def add_part(part: str):
565 parts.append(part)
566 if isinstance(target, rich.console.Console):
567 target.print(rich.markdown.Markdown(part))
569 def add_section(header: str):
570 if target == "md" or isinstance(target, rich.console.Console):
571 add_part(f"\n### {header}\n")
572 elif target == "html":
573 parts.append(f'<h3 id="{header_to_tag(header)}">{header}</h3>')
574 else:
575 assert_never(target)
577 def header_to_tag(header: str):
578 return (
579 header.replace("`", "")
580 .replace("(", "")
581 .replace(")", "")
582 .replace(" ", "-")
583 .lower()
584 )
586 def add_as_details_below(
587 title: str, text: Union[str, Tuple[str, rich.traceback.Traceback]]
588 ):
589 """returns a header and its tag to link to details below"""
591 def make_link(header: str):
592 tag = header_to_tag(header)
593 if target == "md":
594 return f"[{header}](#{tag})"
595 elif target == "html":
596 return f'<a href="#{tag}">{header}</a>'
597 elif isinstance(target, rich.console.Console):
598 return f"{header} below"
599 else:
600 assert_never(target)
602 for n in range(1, 4):
603 header = f"{title} {n}"
604 if header in details_below:
605 if details_below[header] == text:
606 return make_link(header)
607 else:
608 details_below[header] = text
609 return make_link(header)
611 nonlocal left_out_details
612 left_out_details += 1
613 return make_link(left_out_details_header)
615 @dataclass
616 class CodeCell:
617 text: str
619 @dataclass
620 class CodeRef:
621 text: str
623 def format_code(
624 code: str,
625 lang: str = "",
626 title: str = "Details",
627 cell_line_limit: int = 15,
628 cell_width_limit: int = 120,
629 ) -> Union[CodeRef, CodeCell]:
630 if not code.strip():
631 return CodeCell("")
633 if target == "html":
634 html_lang = f' lang="{lang}"' if lang else ""
635 code = f"<pre{html_lang}>{code}</pre>"
636 put_below = (
637 code.count("\n") > cell_line_limit
638 or max(map(len, code.split("\n"))) > cell_width_limit
639 )
640 else:
641 put_below = True
642 code = f"\n```{lang}\n{code}\n```\n"
644 if put_below:
645 link = add_as_details_below(title, code)
646 return CodeRef(f"See {link}.")
647 else:
648 return CodeCell(code)
650 def format_traceback(entry: ErrorEntry):
651 if isinstance(target, rich.console.Console):
652 if entry.traceback_rich is None:
653 return format_code(entry.traceback_md, title="Traceback")
654 else:
655 link = add_as_details_below(
656 "Traceback", (entry.traceback_md, entry.traceback_rich)
657 )
658 return CodeRef(f"See {link}.")
660 if target == "md":
661 return format_code(entry.traceback_md, title="Traceback")
662 elif target == "html":
663 return format_code(entry.traceback_html, title="Traceback")
664 else:
665 assert_never(target)
667 def format_text(text: str):
668 if target == "html":
669 return [f"<pre>{text}</pre>"]
670 else:
671 return text.split("\n")
673 def get_info_table():
674 info_rows = [
675 [summary.status_icon, summary.name.strip(".").strip()],
676 ["status", summary.status],
677 ]
678 if not hide_source:
679 info_rows.append(["source", html.escape(summary.source_name)])
681 if summary.id is not None:
682 info_rows.append(["id", summary.id])
684 if summary.version is not None:
685 info_rows.append(["version", str(summary.version)])
687 info_rows.append(["applied format", f"{summary.type} {summary.format_version}"])
688 if not hide_env:
689 info_rows.extend([[e.name, e.version] for e in sorted(summary.env)])
691 if include_conda_list:
692 info_rows.append(
693 ["conda list", format_code(summary.conda_list, title="Conda List").text]
694 )
695 return format_table(info_rows)
697 def get_details_table():
698 details = [["", "Location", "Details"]]
700 def append_detail(
701 status: str, loc: Loc, text: str, code: Union[CodeRef, CodeCell, None]
702 ):
703 text_lines = format_text(text)
704 status_lines = [""] * len(text_lines)
705 loc_lines = [""] * len(text_lines)
706 status_lines[0] = status
707 loc_lines[0] = format_loc(loc, target)
708 for s_line, loc_line, text_line in zip(status_lines, loc_lines, text_lines):
709 details.append([s_line, loc_line, text_line])
711 if code is not None:
712 details.append(["", "", code.text])
714 for d in summary.details:
715 details.append([d.status_icon, format_loc(d.loc, target), d.name])
717 for entry in d.errors:
718 append_detail(
719 "❌",
720 entry.loc,
721 entry.msg,
722 None if hide_tracebacks else format_traceback(entry),
723 )
725 for entry in d.warnings:
726 append_detail("⚠", entry.loc, entry.msg, None)
728 if d.recommended_env is not None:
729 rec_env = StringIO()
730 json_env = d.recommended_env.model_dump(
731 mode="json", exclude_defaults=True
732 )
733 assert is_yaml_value(json_env)
734 write_yaml(json_env, rec_env)
735 append_detail(
736 "",
737 d.loc,
738 f"recommended conda environment ({d.name})",
739 format_code(
740 rec_env.getvalue(),
741 lang="yaml",
742 title="Recommended Conda Environment",
743 ),
744 )
746 if d.conda_compare:
747 wrapped_conda_compare = "\n".join(
748 TextWrapper(width=width - 4).wrap(d.conda_compare)
749 )
750 append_detail(
751 "",
752 d.loc,
753 f"conda compare ({d.name})",
754 format_code(
755 wrapped_conda_compare,
756 title="Conda Environment Comparison",
757 ),
758 )
760 return format_table(details)
762 add_part(get_info_table())
763 add_part(get_details_table())
765 for header, text in details_below.items():
766 add_section(header)
767 if isinstance(text, tuple):
768 assert isinstance(target, rich.console.Console)
769 text, rich_obj = text
770 target.print(rich_obj)
771 parts.append(f"{text}\n")
772 else:
773 add_part(f"{text}\n")
775 if left_out_details:
776 parts.append(
777 f"\n{left_out_details_header}\nLeft out {left_out_details} more details for brevity.\n"
778 )
780 return "".join(parts)
783def _format_md_table(rows: List[List[str]]) -> str:
784 """format `rows` as markdown table"""
785 n_cols = len(rows[0])
786 assert all(len(row) == n_cols for row in rows)
787 col_widths = [max(max(len(row[i]) for row in rows), 3) for i in range(n_cols)]
789 # fix new lines in table cell
790 rows = [[line.replace("\n", "<br>") for line in r] for r in rows]
792 lines = [" | ".join(rows[0][i].center(col_widths[i]) for i in range(n_cols))]
793 lines.append(" | ".join("---".center(col_widths[i]) for i in range(n_cols)))
794 lines.extend(
795 [
796 " | ".join(row[i].ljust(col_widths[i]) for i in range(n_cols))
797 for row in rows[1:]
798 ]
799 )
800 return "\n| " + " |\n| ".join(lines) + " |\n"
803def _format_html_table(rows: List[List[str]]) -> str:
804 """format `rows` as HTML table"""
806 def get_line(cells: List[str], cell_tag: Literal["th", "td"] = "td"):
807 return (
808 [" <tr>"]
809 + [f" <{cell_tag}>{c}</{cell_tag}>" for c in cells]
810 + [" </tr>"]
811 )
813 table = ["<table>"] + get_line(rows[0], cell_tag="th")
814 for r in rows[1:]:
815 table.extend(get_line(r))
817 table.append("</table>")
819 return "\n".join(table)