Coverage for src/bioimageio/spec/summary.py: 66%
396 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-18 09:17 +0000
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-18 09:17 +0000
1"""Utilities for summarizing and formatting BioImage.IO validation results.
3This module defines data structures to capture validation errors, warnings,
4and summaries for BioImage.IO resource descriptions, along with helpers to
5format these results as plain text, Markdown, or HTML for reporting and
6diagnostics.
7"""
9from __future__ import annotations
11import html
12import platform
13import subprocess
14from dataclasses import dataclass
15from datetime import datetime, timezone
16from io import StringIO
17from itertools import chain
18from pathlib import Path
19from tempfile import TemporaryDirectory
20from textwrap import TextWrapper
21from types import MappingProxyType
22from typing import (
23 Any,
24 Callable,
25 List,
26 Literal,
27 Mapping,
28 NamedTuple,
29 Sequence,
30 Tuple,
31 Union,
32 cast,
33)
35import annotated_types
36import markdown
37import rich.console
38import rich.markdown
39import rich.traceback
40from loguru import logger
41from pydantic import (
42 BaseModel,
43 Field,
44 field_serializer,
45 field_validator,
46 model_validator,
47)
48from pydantic_core.core_schema import ErrorType
49from typing_extensions import Annotated, Self, assert_never
51from ._internal.io import is_yaml_value
52from ._internal.io_utils import write_yaml
53from ._internal.type_guards import is_dict
54from ._internal.validation_context import ValidationContextSummary
55from ._internal.version_type import Version
56from ._internal.warning_levels import (
57 ALERT,
58 ALERT_NAME,
59 ERROR,
60 ERROR_NAME,
61 INFO,
62 INFO_NAME,
63 WARNING,
64 WARNING_NAME,
65 WarningLevel,
66 WarningSeverity,
67)
68from ._version import VERSION
69from .conda_env import CondaEnv
71CONDA_CMD = "conda.bat" if platform.system() == "Windows" else "conda"
73Loc = Tuple[Union[int, str], ...]
74"""location of error/warning in a nested data structure"""
76WarningSeverityName = Literal["info", "warning", "alert"]
77WarningLevelName = Literal[WarningSeverityName, "error"]
79WARNING_SEVERITY_TO_NAME: Mapping[WarningSeverity, WarningSeverityName] = (
80 MappingProxyType({INFO: INFO_NAME, WARNING: WARNING_NAME, ALERT: ALERT_NAME})
81)
82WARNING_LEVEL_TO_NAME: Mapping[WarningLevel, WarningLevelName] = MappingProxyType(
83 {INFO: INFO_NAME, WARNING: WARNING_NAME, ALERT: ALERT_NAME, ERROR: ERROR_NAME}
84)
85WARNING_NAME_TO_LEVEL: Mapping[WarningLevelName, WarningLevel] = MappingProxyType(
86 {v: k for k, v in WARNING_LEVEL_TO_NAME.items()}
87)
90class ValidationEntry(BaseModel):
91 """Base of `ErrorEntry` and `WarningEntry`"""
93 loc: Loc
94 msg: str
95 type: ErrorType | str
98class ErrorEntry(ValidationEntry):
99 """An error in a `ValidationDetail`"""
101 with_traceback: bool = False
102 traceback_md: str = ""
103 traceback_html: str = ""
104 # private rich traceback that is not serialized
105 _traceback_rich: rich.traceback.Traceback | None = None
107 @property
108 def traceback_rich(self) -> rich.traceback.Traceback | None:
109 return self._traceback_rich
111 def model_post_init(self, __context: Any, /):
112 if self.with_traceback and not (self.traceback_md or self.traceback_html):
113 self._traceback_rich = rich.traceback.Traceback()
114 console = rich.console.Console(
115 record=True,
116 file=StringIO(),
117 color_system="truecolor",
118 width=120,
119 tab_size=4,
120 soft_wrap=True,
121 )
122 console.print(self._traceback_rich)
123 if not self.traceback_md:
124 self.traceback_md = console.export_text(clear=False)
126 if not self.traceback_html:
127 self.traceback_html = console.export_html(clear=False)
130class WarningEntry(ValidationEntry):
131 """A warning in a `ValidationDetail`"""
133 severity: WarningSeverity = WARNING
135 @property
136 def severity_name(self) -> WarningSeverityName:
137 return WARNING_SEVERITY_TO_NAME[self.severity]
140def format_loc(
141 loc: Loc, target: Literal["md", "html", "plain"] | rich.console.Console
142) -> str:
143 """helper to format a location tuple **loc**"""
144 loc_str = ".".join(f"({x})" if x[0].isupper() else x for x in map(str, loc))
146 # additional field validation can make the location information quite convoluted, e.g.
147 # `weights.pytorch_state_dict.dependencies.source.function-after[validate_url_ok(), url['http','https']]` Input should be a valid URL, relative URL without a base
148 # therefore we remove the `.function-after[validate_url_ok(), url['http','https']]` here
149 loc_str, *_ = loc_str.split(".function-after")
150 if loc_str:
151 if target == "md" or isinstance(target, rich.console.Console):
152 start = "`"
153 end = "`"
154 elif target == "html":
155 start = "<code>"
156 end = "</code>"
157 elif target == "plain":
158 start = ""
159 end = ""
160 else:
161 assert_never(target)
163 return f"{start}{loc_str}{end}"
164 else:
165 return ""
168class InstalledPackage(NamedTuple):
169 name: str
170 version: str
171 build: str = ""
172 channel: str = ""
175class ValidationDetail(BaseModel, extra="allow"):
176 """a detail in a validation summary"""
178 name: str
179 status: Literal["passed", "failed"]
180 loc: Loc = ()
181 """location in the RDF that this detail applies to"""
182 errors: list[ErrorEntry] = Field(
183 default_factory=cast(Callable[[], List[ErrorEntry]], list)
184 )
185 warnings: list[WarningEntry] = Field(
186 default_factory=cast(Callable[[], List[WarningEntry]], list)
187 )
189 context: ValidationContextSummary | None = None
191 recommended_env: CondaEnv | None = None
192 """recommended conda environemnt for this validation detail"""
194 saved_conda_compare: str | None = None
195 """output of `conda compare <recommended env>`"""
197 @field_serializer("saved_conda_compare")
198 def _save_conda_compare(self, value: str | None):
199 return self.conda_compare
201 @model_validator(mode="before")
202 def _load_legacy(cls, data: Any):
203 if is_dict(data):
204 field_name = "conda_compare"
205 if (
206 field_name in data
207 and (saved_field_name := f"saved_{field_name}") not in data
208 ):
209 data[saved_field_name] = data.pop(field_name)
211 return data
213 @property
214 def conda_compare(self) -> str | None:
215 if self.recommended_env is None:
216 return None
218 if self.saved_conda_compare is None:
219 dumped_env = self.recommended_env.model_dump(mode="json")
220 if is_yaml_value(dumped_env):
221 with TemporaryDirectory() as d:
222 path = Path(d) / "env.yaml"
223 with path.open("w", encoding="utf-8") as f:
224 write_yaml(dumped_env, f)
226 try:
227 compare_proc = subprocess.run(
228 [CONDA_CMD, "compare", str(path)],
229 stdout=subprocess.PIPE,
230 stderr=subprocess.STDOUT,
231 shell=False,
232 text=True,
233 check=False,
234 )
235 except Exception as e:
236 self.saved_conda_compare = f"Failed to run `conda compare`: {e}"
237 else:
238 self.saved_conda_compare = (
239 compare_proc.stdout
240 or f"`conda compare` exited with {compare_proc.returncode}"
241 )
242 else:
243 self.saved_conda_compare = (
244 "Failed to dump recommended env to valid yaml"
245 )
247 return self.saved_conda_compare
249 @property
250 def status_icon(self) -> str:
251 if self.status == "passed":
252 return "✔️"
253 else:
254 return "❌"
257class ValidationSummary(BaseModel, extra="allow"):
258 """Summarizes output of all bioimageio validations and tests
259 for one specific `ResourceDescr` instance."""
261 name: str
262 """Name of the validation"""
264 source_name: str
265 """Source of the validated bioimageio description"""
267 id: str | None = None
268 """ID of the validated resource"""
270 version: Version | None = None
271 """Version of the validated resource"""
273 type: str
274 """Type of the validated resource"""
276 format_version: str
277 """Format version of the validated resource"""
279 status: Literal["passed", "valid-format", "failed"]
280 """Overall status of the bioimageio validation"""
282 metadata_completeness: Annotated[float, annotated_types.Interval(ge=0, le=1)] = 0.0
283 """Estimate of completeness of the metadata in the resource description.
285 Note: This completeness estimate may change with subsequent releases
286 and should be considered bioimageio.spec version specific.
287 """
289 details: list[ValidationDetail]
290 """List of validation details"""
291 env: set[InstalledPackage] = Field(
292 default_factory=lambda: {
293 InstalledPackage(
294 name="bioimageio.spec",
295 version=VERSION,
296 )
297 }
298 )
299 """List of selected, relevant package versions"""
301 saved_conda_list: str | None = None
303 @field_serializer("saved_conda_list")
304 def _save_conda_list(self, value: str | None):
305 return self.conda_list
307 @property
308 def conda_list(self) -> str:
309 if self.saved_conda_list is None:
310 try:
311 p = subprocess.run(
312 [CONDA_CMD, "list"],
313 stdout=subprocess.PIPE,
314 stderr=subprocess.STDOUT,
315 shell=False,
316 text=True,
317 check=False,
318 )
319 except Exception as e:
320 self.saved_conda_list = f"Failed to run `conda list`: {e}"
321 else:
322 self.saved_conda_list = (
323 p.stdout or f"`conda list` exited with {p.returncode}"
324 )
326 return self.saved_conda_list
328 @property
329 def status_icon(self) -> str:
330 if self.status == "passed":
331 return "✔️"
332 elif self.status == "valid-format":
333 return "🟡"
334 else:
335 return "❌"
337 @property
338 def errors(self) -> list[ErrorEntry]:
339 return list(chain.from_iterable(d.errors for d in self.details))
341 @property
342 def warnings(self) -> list[WarningEntry]:
343 return list(chain.from_iterable(d.warnings for d in self.details))
345 def format(
346 self,
347 *,
348 width: int | None = None,
349 include_conda_list: bool = False,
350 ) -> str:
351 """Format summary as Markdown string (alias to `format_md`)"""
352 return self.format_md(width=width, include_conda_list=include_conda_list)
354 def format_md(
355 self,
356 *,
357 width: int | None = None,
358 include_conda_list: bool = False,
359 ) -> str:
360 """Format summary as Markdown string"""
361 return self._format(
362 width=width, target="md", include_conda_list=include_conda_list
363 )
365 def format_html(
366 self,
367 *,
368 width: int | None = None,
369 include_conda_list: bool = False,
370 ) -> str:
371 md_with_html = self._format(
372 target="html", width=width, include_conda_list=include_conda_list
373 )
374 return markdown.markdown(
375 md_with_html, extensions=["tables", "fenced_code", "nl2br"]
376 )
378 def display(
379 self,
380 *,
381 width: int | None = None,
382 include_conda_list: bool = False,
383 tab_size: int = 4,
384 soft_wrap: bool = True,
385 ) -> None:
386 try: # render as HTML in Jupyter notebook
387 from IPython.core.getipython import get_ipython
388 from IPython.display import (
389 display_html, # pyright: ignore[reportUnknownVariableType]
390 )
391 except ImportError:
392 pass
393 else:
394 if get_ipython() is not None:
395 _ = display_html(
396 self.format_html(
397 width=width, include_conda_list=include_conda_list
398 ),
399 raw=True,
400 )
401 return
403 # render with rich
404 _ = self._format(
405 target=rich.console.Console(
406 width=width,
407 tab_size=tab_size,
408 soft_wrap=soft_wrap,
409 ),
410 width=width,
411 include_conda_list=include_conda_list,
412 )
414 def add_detail(
415 self,
416 detail: ValidationDetail,
417 update_status: bool = True,
418 ) -> None:
419 """add a validation detail to the summary and, if `update_status` is True,
420 possibly downgrade the overall status from "passed" to "valid-format"
421 """
422 if detail in self.details:
423 return
425 # An overall status 'passed' can degrade to 'valid-format'
426 # The status can not be upgraded here as we do not know if it has been downgraded before or not yet tested.
427 # Once status is 'failed' it cannot change anymore.
428 if update_status and self.status == "passed" and detail.status == "failed":
429 # "passed" -> "valid-format"
430 self.status = "valid-format"
432 self.details.append(detail)
434 def log(
435 self,
436 to: Literal["display"] | Path | Sequence[Literal["display"] | Path],
437 ) -> list[Path]:
438 """Convenience method to display the validation summary in the terminal and/or
439 save it to disk. See `save` for details."""
440 if to == "display":
441 display = True
442 save_to = []
443 elif isinstance(to, Path):
444 display = False
445 save_to = [to]
446 else:
447 display = "display" in to
448 save_to = [p for p in to if p != "display"]
450 if display:
451 self.display()
453 return self.save(save_to)
455 def save(
456 self, path: Path | Sequence[Path] = Path("{id}_summary_{now}")
457 ) -> list[Path]:
458 """Save the validation/test summary in JSON, Markdown or HTML format.
460 Returns:
461 List of file paths the summary was saved to.
463 Notes:
464 - Format is chosen based on the suffix: `.json`, `.md`, `.html`.
465 - If **path** has no suffix it is assumed to be a direcotry to which a
466 `summary.json`, `summary.md` and `summary.html` are saved to.
467 """
468 if isinstance(path, (str, Path)):
469 path = [Path(path)]
471 # folder to file paths
472 file_paths: list[Path] = []
473 for p in path:
474 if p.suffix:
475 file_paths.append(p)
476 else:
477 file_paths.extend(
478 [
479 p / "summary.json",
480 p / "summary.md",
481 p / "summary.html",
482 ]
483 )
485 now = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
486 for p in file_paths:
487 p = Path(str(p).format(id=self.id or "bioimageio", now=now))
488 if p.suffix == ".json":
489 self.save_json(p)
490 elif p.suffix == ".md":
491 self.save_markdown(p)
492 elif p.suffix == ".html":
493 self.save_html(p)
494 else:
495 raise ValueError(f"Unknown summary path suffix '{p.suffix}'")
497 return file_paths
499 def save_json(
500 self, path: Path = Path("summary.json"), *, indent: int | None = 2
501 ) -> None:
502 """Save validation/test summary as JSON file."""
503 json_str = self.model_dump_json(indent=indent)
504 path.parent.mkdir(exist_ok=True, parents=True)
505 _ = path.write_text(json_str, encoding="utf-8")
506 logger.info("Saved summary to {}", path.absolute())
508 def save_markdown(self, path: Path = Path("summary.md")) -> None:
509 """Save rendered validation/test summary as Markdown file."""
510 formatted = self.format_md()
511 path.parent.mkdir(exist_ok=True, parents=True)
512 _ = path.write_text(formatted, encoding="utf-8")
513 logger.info("Saved Markdown formatted summary to {}", path.absolute())
515 def save_html(self, path: Path = Path("summary.html")) -> None:
516 """Save rendered validation/test summary as HTML file."""
517 path.parent.mkdir(exist_ok=True, parents=True)
519 html = self.format_html()
520 _ = path.write_text(html, encoding="utf-8")
521 logger.info("Saved HTML formatted summary to {}", path.absolute())
523 @classmethod
524 def load_json(cls, path: Path) -> Self:
525 """Load validation/test summary from a suitable JSON file"""
526 json_str = Path(path).read_text(encoding="utf-8")
527 return cls.model_validate_json(json_str)
529 @field_validator("env", mode="before")
530 def _convert_old_env(cls, value: list[list[str] | dict[str, str]]):
531 """convert old style dict values of `env` for backwards compatibility"""
532 if isinstance(value, list):
533 return [
534 (
535 (v["name"], v["version"], v.get("build", ""), v.get("channel", ""))
536 if isinstance(v, dict) and "name" in v and "version" in v
537 else v
538 )
539 for v in value
540 ]
541 else:
542 return value
544 def _format(
545 self,
546 *,
547 target: rich.console.Console | Literal["html", "md"],
548 width: int | None,
549 include_conda_list: bool,
550 ) -> str:
551 return _format_summary(
552 self,
553 target=target,
554 width=width or 100,
555 include_conda_list=include_conda_list,
556 )
559def _format_summary(
560 summary: ValidationSummary,
561 *,
562 hide_tracebacks: bool = False, # TODO: remove?
563 hide_source: bool = False, # TODO: remove?
564 hide_env: bool = False, # TODO: remove?
565 target: rich.console.Console | Literal["html", "md"] = "md",
566 include_conda_list: bool,
567 width: int,
568) -> str:
569 parts: list[str] = []
570 format_table = _format_html_table if target == "html" else _format_md_table
571 details_below: dict[str, str | tuple[str, rich.traceback.Traceback]] = {}
572 left_out_details: int = 0
573 left_out_details_header = "Left out details"
575 def add_part(part: str):
576 parts.append(part)
577 if isinstance(target, rich.console.Console):
578 target.print(rich.markdown.Markdown(part))
580 def add_section(header: str):
581 if target == "md" or isinstance(target, rich.console.Console):
582 add_part(f"\n### {header}\n")
583 elif target == "html":
584 parts.append(f'<h3 id="{header_to_tag(header)}">{header}</h3>')
585 else:
586 assert_never(target)
588 def header_to_tag(header: str):
589 return (
590 header.replace("`", "")
591 .replace("(", "")
592 .replace(")", "")
593 .replace(" ", "-")
594 .lower()
595 )
597 def add_as_details_below(
598 title: str, text: str | tuple[str, rich.traceback.Traceback]
599 ):
600 """returns a header and its tag to link to details below"""
602 def make_link(header: str):
603 tag = header_to_tag(header)
604 if target == "md":
605 return f"[{header}](#{tag})"
606 elif target == "html":
607 return f'<a href="#{tag}">{header}</a>'
608 elif isinstance(target, rich.console.Console):
609 return f"{header} below"
610 else:
611 assert_never(target)
613 for n in range(1, 4):
614 header = f"{title} {n}"
615 if header in details_below:
616 if details_below[header] == text:
617 return make_link(header)
618 else:
619 details_below[header] = text
620 return make_link(header)
622 nonlocal left_out_details
623 left_out_details += 1
624 return make_link(left_out_details_header)
626 @dataclass
627 class CodeCell:
628 text: str
630 @dataclass
631 class CodeRef:
632 text: str
634 def format_code(
635 code: str,
636 lang: str = "",
637 title: str = "Details",
638 cell_line_limit: int = 15,
639 cell_width_limit: int = 120,
640 ) -> CodeRef | CodeCell:
641 if not code.strip():
642 return CodeCell("")
644 if target == "html":
645 html_lang = f' lang="{lang}"' if lang else ""
646 code = f"<pre{html_lang}>{code}</pre>"
647 put_below = (
648 code.count("\n") > cell_line_limit
649 or max(map(len, code.split("\n"))) > cell_width_limit
650 )
651 else:
652 put_below = True
653 code = f"\n```{lang}\n{code}\n```\n"
655 if put_below:
656 link = add_as_details_below(title, code)
657 return CodeRef(f"See {link}.")
658 else:
659 return CodeCell(code)
661 def format_traceback(entry: ErrorEntry):
662 if isinstance(target, rich.console.Console):
663 if entry.traceback_rich is None:
664 return format_code(entry.traceback_md, title="Traceback")
665 else:
666 link = add_as_details_below(
667 "Traceback", (entry.traceback_md, entry.traceback_rich)
668 )
669 return CodeRef(f"See {link}.")
671 if target == "md":
672 return format_code(entry.traceback_md, title="Traceback")
673 elif target == "html":
674 return format_code(entry.traceback_html, title="Traceback")
675 else:
676 assert_never(target)
678 def format_text(text: str):
679 if target == "html":
680 return [f"<pre>{text}</pre>"]
681 else:
682 return text.split("\n")
684 def get_info_table():
685 info_rows = [
686 [summary.status_icon, summary.name.strip(".").strip()],
687 ["status", summary.status],
688 ]
689 if not hide_source:
690 info_rows.append(["source", html.escape(summary.source_name)])
692 if summary.id is not None:
693 info_rows.append(["id", summary.id])
695 if summary.version is not None:
696 info_rows.append(["version", str(summary.version)])
698 info_rows.append(["applied format", f"{summary.type} {summary.format_version}"])
699 if not hide_env:
700 info_rows.extend([[e.name, e.version] for e in sorted(summary.env)])
702 if include_conda_list:
703 info_rows.append(
704 ["conda list", format_code(summary.conda_list, title="Conda List").text]
705 )
706 return format_table(info_rows)
708 def get_details_table():
709 details = [["", "Location", "Details"]]
711 def append_detail(
712 status: str, loc: Loc, text: str, code: CodeRef | CodeCell | None
713 ):
714 text_lines = format_text(text)
715 status_lines = [""] * len(text_lines)
716 loc_lines = [""] * len(text_lines)
717 status_lines[0] = status
718 loc_lines[0] = format_loc(loc, target)
719 for s_line, loc_line, text_line in zip(status_lines, loc_lines, text_lines):
720 details.append([s_line, loc_line, text_line])
722 if code is not None:
723 details.append(["", "", code.text])
725 for d in summary.details:
726 details.append([d.status_icon, format_loc(d.loc, target), d.name])
728 for entry in d.errors:
729 append_detail(
730 "❌",
731 entry.loc,
732 entry.msg,
733 None if hide_tracebacks else format_traceback(entry),
734 )
736 for entry in d.warnings:
737 append_detail(
738 "⚠" if entry.severity > INFO else "ℹ", entry.loc, entry.msg, None
739 )
741 if d.recommended_env is not None:
742 rec_env = StringIO()
743 json_env = d.recommended_env.model_dump(
744 mode="json", exclude_defaults=True
745 )
746 assert is_yaml_value(json_env)
747 write_yaml(json_env, rec_env)
748 append_detail(
749 "",
750 d.loc,
751 f"recommended conda environment ({d.name})",
752 format_code(
753 rec_env.getvalue(),
754 lang="yaml",
755 title="Recommended Conda Environment",
756 ),
757 )
759 if d.conda_compare:
760 wrapped_conda_compare = "\n".join(
761 TextWrapper(width=width - 4).wrap(d.conda_compare)
762 )
763 append_detail(
764 "",
765 d.loc,
766 f"conda compare ({d.name})",
767 format_code(
768 wrapped_conda_compare,
769 title="Conda Environment Comparison",
770 ),
771 )
773 return format_table(details)
775 add_part(get_info_table())
776 add_part(get_details_table())
778 for header, text in details_below.items():
779 add_section(header)
780 if isinstance(text, tuple):
781 assert isinstance(target, rich.console.Console)
782 text, rich_obj = text
783 target.print(rich_obj)
784 parts.append(f"{text}\n")
785 else:
786 add_part(f"{text}\n")
788 if left_out_details:
789 parts.append(
790 f"\n{left_out_details_header}\nLeft out {left_out_details} more details for brevity.\n"
791 )
793 return "".join(parts)
796def _format_md_table(rows: list[list[str]]) -> str:
797 """format `rows` as markdown table"""
798 n_cols = len(rows[0])
799 assert all(len(row) == n_cols for row in rows)
800 col_widths = [max(max(len(row[i]) for row in rows), 3) for i in range(n_cols)]
802 # fix new lines in table cell
803 rows = [[line.replace("\n", "<br>") for line in r] for r in rows]
805 lines = [" | ".join(rows[0][i].center(col_widths[i]) for i in range(n_cols))]
806 lines.append(" | ".join("---".center(col_widths[i]) for i in range(n_cols)))
807 lines.extend(
808 [
809 " | ".join(row[i].ljust(col_widths[i]) for i in range(n_cols))
810 for row in rows[1:]
811 ]
812 )
813 return "\n| " + " |\n| ".join(lines) + " |\n"
816def _format_html_table(rows: list[list[str]]) -> str:
817 """format `rows` as HTML table"""
819 def get_line(cells: list[str], cell_tag: Literal["th", "td"] = "td"):
820 return (
821 [" <tr>"]
822 + [
823 f' <{cell_tag} style="text-align:{"center" if cell_tag == "th" else "left"}">{c}</{cell_tag}>'
824 for c in cells
825 ]
826 + [" </tr>"]
827 )
829 table = ["<table>"] + get_line(rows[0], cell_tag="th")
830 for r in rows[1:]:
831 table.extend(get_line(r))
833 table.append("</table>")
835 return "\n".join(table)