Coverage for bioimageio/core/cli.py: 83%

1"""bioimageio CLI 

2 

3Note: Some docstrings use a hair space ' ' 

4 to place the added '(default: ...)' on a new line. 

5""" 

6 

7import json 

8import shutil 

9import subprocess 

10import sys 

11from abc import ABC 

12from argparse import RawTextHelpFormatter 

13from difflib import SequenceMatcher 

14from functools import cached_property 

15from io import StringIO 

16from pathlib import Path 

17from pprint import pformat, pprint 

18from typing import ( 

19 Any, 

20 Dict, 

21 Iterable, 

22 List, 

23 Literal, 

24 Mapping, 

25 Optional, 

26 Sequence, 

27 Set, 

28 Tuple, 

29 Type, 

30 Union, 

31) 

32 

33import rich.markdown 

34from loguru import logger 

35from pydantic import AliasChoices, BaseModel, Field, model_validator 

36from pydantic_settings import ( 

37 BaseSettings, 

38 CliPositionalArg, 

39 CliSettingsSource, 

40 CliSubCommand, 

41 JsonConfigSettingsSource, 

42 PydanticBaseSettingsSource, 

43 SettingsConfigDict, 

44 YamlConfigSettingsSource, 

45) 

46from tqdm import tqdm 

47from typing_extensions import assert_never 

48 

49from bioimageio.spec import ( 

50 AnyModelDescr, 

51 InvalidDescr, 

52 ResourceDescr, 

53 load_description, 

54 save_bioimageio_yaml_only, 

55 settings, 

56 update_format, 

57 update_hashes, 

58) 

59from bioimageio.spec._internal.io import is_yaml_value 

60from bioimageio.spec._internal.io_basics import ZipPath 

61from bioimageio.spec._internal.io_utils import open_bioimageio_yaml 

62from bioimageio.spec._internal.types import NotEmpty 

63from bioimageio.spec.dataset import DatasetDescr 

64from bioimageio.spec.model import ModelDescr, v0_4, v0_5 

65from bioimageio.spec.notebook import NotebookDescr 

66from bioimageio.spec.utils import download, ensure_description_is_model, write_yaml 

67 

68from .commands import WeightFormatArgAll, WeightFormatArgAny, package, test 

69from .common import MemberId, SampleId, SupportedWeightsFormat 

70from .digest_spec import get_member_ids, load_sample_for_model 

71from .io import load_dataset_stat, save_dataset_stat, save_sample 

72from .prediction import create_prediction_pipeline 

73from .proc_setup import ( 

74 DatasetMeasure, 

75 Measure, 

76 MeasureValue, 

77 StatsCalculator, 

78 get_required_dataset_measures, 

79) 

80from .sample import Sample 

81from .stat_measures import Stat 

82from .utils import VERSION, compare 

83from .weight_converters._add_weights import add_weights 

84 

85WEIGHT_FORMAT_ALIASES = AliasChoices( 

86 "weight-format", 

87 "weights-format", 

88 "weight_format", 

89 "weights_format", 

90) 

91 

92 

93class CmdBase(BaseModel, use_attribute_docstrings=True, cli_implicit_flags=True): 

94 pass 

95 

96 

97class ArgMixin(BaseModel, use_attribute_docstrings=True, cli_implicit_flags=True): 

98 pass 

99 

100 

101class WithSummaryLogging(ArgMixin): 

102 summary: Union[ 

103 Literal["display"], Path, Sequence[Union[Literal["display"], Path]] 

104 ] = Field( 

105 "display", 

106 examples=[ 

107 "display", 

108 Path("summary.md"), 

109 Path("bioimageio_summaries/"), 

110 ["display", Path("summary.md")], 

111 ], 

112 ) 

113 """Display the validation summary or save it as JSON, Markdown or HTML. 

114 The format is chosen based on the suffix: `.json`, `.md`, `.html`. 

115 If a folder is given (path w/o suffix) the summary is saved in all formats. 

116 Choose/add `"display"` to render the validation summary to the terminal. 

117 """ 

118 

119 def log(self, descr: Union[ResourceDescr, InvalidDescr]): 

120 _ = descr.validation_summary.log(self.summary) 

121 

122 

123class WithSource(ArgMixin): 

124 source: CliPositionalArg[str] 

125 """Url/path to a (folder with a) bioimageio.yaml/rdf.yaml file 

126 or a bioimage.io resource identifier, e.g. 'affable-shark'""" 

127 

128 @cached_property 

129 def descr(self): 

130 return load_description(self.source) 

131 

132 @property 

133 def descr_id(self) -> str: 

134 """a more user-friendly description id 

135 (replacing legacy ids with their nicknames) 

136 """ 

137 if isinstance(self.descr, InvalidDescr): 

138 return str(getattr(self.descr, "id", getattr(self.descr, "name"))) 

139 

140 nickname = None 

141 if ( 

142 isinstance(self.descr.config, v0_5.Config) 

143 and (bio_config := self.descr.config.bioimageio) 

144 and bio_config.model_extra is not None 

145 ): 

146 nickname = bio_config.model_extra.get("nickname") 

147 

148 return str(nickname or self.descr.id or self.descr.name) 

149 

150 

151class ValidateFormatCmd(CmdBase, WithSource, WithSummaryLogging): 

152 """Validate the meta data format of a bioimageio resource.""" 

153 

154 perform_io_checks: bool = Field( 

155 settings.perform_io_checks, alias="perform-io-checks" 

156 ) 

157 """Wether or not to perform validations that requires downloading remote files. 

158 Note: Default value is set by `BIOIMAGEIO_PERFORM_IO_CHECKS` environment variable. 

159 """ 

160 

161 @cached_property 

162 def descr(self): 

163 return load_description(self.source, perform_io_checks=self.perform_io_checks) 

164 

165 def run(self): 

166 self.log(self.descr) 

167 sys.exit( 

168 0 

169 if self.descr.validation_summary.status in ("valid-format", "passed") 

170 else 1 

171 ) 

172 

173 

174class TestCmd(CmdBase, WithSource, WithSummaryLogging): 

175 """Test a bioimageio resource (beyond meta data formatting).""" 

176 

177 weight_format: WeightFormatArgAll = Field( 

178 "all", 

179 alias="weight-format", 

180 validation_alias=WEIGHT_FORMAT_ALIASES, 

181 ) 

182 """The weight format to limit testing to. 

183 

184 (only relevant for model resources)""" 

185 

186 devices: Optional[Union[str, Sequence[str]]] = None 

187 """Device(s) to use for testing""" 

188 

189 runtime_env: Union[Literal["currently-active", "as-described"], Path] = Field( 

190 "currently-active", alias="runtime-env" 

191 ) 

192 """The python environment to run the tests in 

193 - `"currently-active"`: use active Python interpreter 

194 - `"as-described"`: generate a conda environment YAML file based on the model 

195 weights description. 

196 - A path to a conda environment YAML. 

197 Note: The `bioimageio.core` dependency will be added automatically if not present. 

198 """ 

199 

200 determinism: Literal["seed_only", "full"] = "seed_only" 

201 """Modes to improve reproducibility of test outputs.""" 

202 

203 stop_early: bool = Field( 

204 False, alias="stop-early", validation_alias=AliasChoices("stop-early", "x") 

205 ) 

206 """Do not run further subtests after a failed one.""" 

207 

208 def run(self): 

209 sys.exit( 

210 test( 

211 self.descr, 

212 weight_format=self.weight_format, 

213 devices=self.devices, 

214 summary=self.summary, 

215 runtime_env=self.runtime_env, 

216 determinism=self.determinism, 

217 ) 

218 ) 

219 

220 

221class PackageCmd(CmdBase, WithSource, WithSummaryLogging): 

222 """Save a resource's metadata with its associated files.""" 

223 

224 path: CliPositionalArg[Path] 

225 """The path to write the (zipped) package to. 

226 If it does not have a `.zip` suffix 

227 this command will save the package as an unzipped folder instead.""" 

228 

229 weight_format: WeightFormatArgAll = Field( 

230 "all", 

231 alias="weight-format", 

232 validation_alias=WEIGHT_FORMAT_ALIASES, 

233 ) 

234 """The weight format to include in the package (for model descriptions only).""" 

235 

236 def run(self): 

237 if isinstance(self.descr, InvalidDescr): 

238 self.log(self.descr) 

239 raise ValueError(f"Invalid {self.descr.type} description.") 

240 

241 sys.exit( 

242 package( 

243 self.descr, 

244 self.path, 

245 weight_format=self.weight_format, 

246 ) 

247 ) 

248 

249 

250def _get_stat( 

251 model_descr: AnyModelDescr, 

252 dataset: Iterable[Sample], 

253 dataset_length: int, 

254 stats_path: Path, 

255) -> Mapping[DatasetMeasure, MeasureValue]: 

256 req_dataset_meas, _ = get_required_dataset_measures(model_descr) 

257 if not req_dataset_meas: 

258 return {} 

259 

260 req_dataset_meas, _ = get_required_dataset_measures(model_descr) 

261 

262 if stats_path.exists(): 

263 logger.info("loading precomputed dataset measures from {}", stats_path) 

264 stat = load_dataset_stat(stats_path) 

265 for m in req_dataset_meas: 

266 if m not in stat: 

267 raise ValueError(f"Missing {m} in {stats_path}") 

268 

269 return stat 

270 

271 stats_calc = StatsCalculator(req_dataset_meas) 

272 

273 for sample in tqdm( 

274 dataset, total=dataset_length, desc="precomputing dataset stats", unit="sample" 

275 ): 

276 stats_calc.update(sample) 

277 

278 stat = stats_calc.finalize() 

279 save_dataset_stat(stat, stats_path) 

280 

281 return stat 

282 

283 

284class UpdateCmdBase(CmdBase, WithSource, ABC): 

285 output: Union[Literal["display", "stdout"], Path] = "display" 

286 """Output updated bioimageio.yaml to the terminal or write to a file. 

287 Notes: 

288 - `"display"`: Render to the terminal with syntax highlighting. 

289 - `"stdout"`: Write to sys.stdout without syntax highligthing. 

290 (More convenient for copying the updated bioimageio.yaml from the terminal.) 

291 """ 

292 

293 diff: Union[bool, Path] = Field(True, alias="diff") 

294 """Output a diff of original and updated bioimageio.yaml. 

295 If a given path has an `.html` extension, a standalone HTML file is written, 

296 otherwise the diff is saved in unified diff format (pure text). 

297 """ 

298 

299 exclude_unset: bool = Field(True, alias="exclude-unset") 

300 """Exclude fields that have not explicitly be set.""" 

301 

302 exclude_defaults: bool = Field(False, alias="exclude-defaults") 

303 """Exclude fields that have the default value (even if set explicitly).""" 

304 

305 @cached_property 

306 def updated(self) -> Union[ResourceDescr, InvalidDescr]: 

307 raise NotImplementedError 

308 

309 def run(self): 

310 original_yaml = open_bioimageio_yaml(self.source).unparsed_content 

311 assert isinstance(original_yaml, str) 

312 stream = StringIO() 

313 

314 save_bioimageio_yaml_only( 

315 self.updated, 

316 stream, 

317 exclude_unset=self.exclude_unset, 

318 exclude_defaults=self.exclude_defaults, 

319 ) 

320 updated_yaml = stream.getvalue() 

321 

322 diff = compare( 

323 original_yaml.split("\n"), 

324 updated_yaml.split("\n"), 

325 diff_format=( 

326 "html" 

327 if isinstance(self.diff, Path) and self.diff.suffix == ".html" 

328 else "unified" 

329 ), 

330 ) 

331 

332 if isinstance(self.diff, Path): 

333 _ = self.diff.write_text(diff, encoding="utf-8") 

334 elif self.diff: 

335 console = rich.console.Console() 

336 diff_md = f"## Diff\n\n````````diff\n{diff}\n````````" 

337 console.print(rich.markdown.Markdown(diff_md)) 

338 

339 if isinstance(self.output, Path): 

340 _ = self.output.write_text(updated_yaml, encoding="utf-8") 

341 logger.info(f"written updated description to {self.output}") 

342 elif self.output == "display": 

343 updated_md = f"## Updated bioimageio.yaml\n\n```yaml\n{updated_yaml}\n```" 

344 rich.console.Console().print(rich.markdown.Markdown(updated_md)) 

345 elif self.output == "stdout": 

346 print(updated_yaml) 

347 else: 

348 assert_never(self.output) 

349 

350 if isinstance(self.updated, InvalidDescr): 

351 logger.warning("Update resulted in invalid description") 

352 _ = self.updated.validation_summary.display() 

353 

354 

355class UpdateFormatCmd(UpdateCmdBase): 

356 """Update the metadata format to the latest format version.""" 

357 

358 exclude_defaults: bool = Field(True, alias="exclude-defaults") 

359 """Exclude fields that have the default value (even if set explicitly). 

360 

361 Note: 

362 The update process sets most unset fields explicitly with their default value. 

363 """ 

364 

365 perform_io_checks: bool = Field( 

366 settings.perform_io_checks, alias="perform-io-checks" 

367 ) 

368 """Wether or not to attempt validation that may require file download. 

369 If `True` file hash values are added if not present.""" 

370 

371 @cached_property 

372 def updated(self): 

373 return update_format( 

374 self.source, 

375 exclude_defaults=self.exclude_defaults, 

376 perform_io_checks=self.perform_io_checks, 

377 ) 

378 

379 

380class UpdateHashesCmd(UpdateCmdBase): 

381 """Create a bioimageio.yaml description with updated file hashes.""" 

382 

383 @cached_property 

384 def updated(self): 

385 return update_hashes(self.source) 

386 

387 

388class PredictCmd(CmdBase, WithSource): 

389 """Run inference on your data with a bioimage.io model.""" 

390 

391 inputs: NotEmpty[Sequence[Union[str, NotEmpty[Tuple[str, ...]]]]] = ( 

392 "{input_id}/001.tif", 

393 ) 

394 """Model input sample paths (for each input tensor) 

395 

396 The input paths are expected to have shape... 

397 - (n_samples,) or (n_samples,1) for models expecting a single input tensor 

398 - (n_samples,) containing the substring '{input_id}', or 

399 - (n_samples, n_model_inputs) to provide each input tensor path explicitly. 

400 

401 All substrings that are replaced by metadata from the model description: 

402 - '{model_id}' 

403 - '{input_id}' 

404 

405 Example inputs to process sample 'a' and 'b' 

406 for a model expecting a 'raw' and a 'mask' input tensor: 

407 --inputs="[[\\"a_raw.tif\\",\\"a_mask.tif\\"],[\\"b_raw.tif\\",\\"b_mask.tif\\"]]" 

408 (Note that JSON double quotes need to be escaped.) 

409 

410 Alternatively a `bioimageio-cli.yaml` (or `bioimageio-cli.json`) file 

411 may provide the arguments, e.g.: 

412 ```yaml 

413 inputs: 

414 - [a_raw.tif, a_mask.tif] 

415 - [b_raw.tif, b_mask.tif] 

416 ``` 

417 

418 `.npy` and any file extension supported by imageio are supported. 

419 Aavailable formats are listed at 

420 https://imageio.readthedocs.io/en/stable/formats/index.html#all-formats. 

421 Some formats have additional dependencies. 

422 

423   

424 """ 

425 

426 outputs: Union[str, NotEmpty[Tuple[str, ...]]] = ( 

427 "outputs_{model_id}/{output_id}/{sample_id}.tif" 

428 ) 

429 """Model output path pattern (per output tensor) 

430 

431 All substrings that are replaced: 

432 - '{model_id}' (from model description) 

433 - '{output_id}' (from model description) 

434 - '{sample_id}' (extracted from input paths) 

435 

436   

437 """ 

438 

439 overwrite: bool = False 

440 """allow overwriting existing output files""" 

441 

442 blockwise: bool = False 

443 """process inputs blockwise""" 

444 

445 stats: Path = Path("dataset_statistics.json") 

446 """path to dataset statistics 

447 (will be written if it does not exist, 

448 but the model requires statistical dataset measures) 

449  """ 

450 

451 preview: bool = False 

452 """preview which files would be processed 

453 and what outputs would be generated.""" 

454 

455 weight_format: WeightFormatArgAny = Field( 

456 "any", 

457 alias="weight-format", 

458 validation_alias=WEIGHT_FORMAT_ALIASES, 

459 ) 

460 """The weight format to use.""" 

461 

462 example: bool = False 

463 """generate and run an example 

464 

465 1. downloads example model inputs 

466 2. creates a `{model_id}_example` folder 

467 3. writes input arguments to `{model_id}_example/bioimageio-cli.yaml` 

468 4. executes a preview dry-run 

469 5. executes prediction with example input 

470 

471   

472 """ 

473 

474 def _example(self): 

475 model_descr = ensure_description_is_model(self.descr) 

476 input_ids = get_member_ids(model_descr.inputs) 

477 example_inputs = ( 

478 model_descr.sample_inputs 

479 if isinstance(model_descr, v0_4.ModelDescr) 

480 else [ipt.sample_tensor or ipt.test_tensor for ipt in model_descr.inputs] 

481 ) 

482 if not example_inputs: 

483 raise ValueError(f"{self.descr_id} does not specify any example inputs.") 

484 

485 inputs001: List[str] = [] 

486 example_path = Path(f"{self.descr_id}_example") 

487 example_path.mkdir(exist_ok=True) 

488 

489 for t, src in zip(input_ids, example_inputs): 

490 local = download(src).path 

491 dst = Path(f"{example_path}/{t}/001{''.join(local.suffixes)}") 

492 dst.parent.mkdir(parents=True, exist_ok=True) 

493 inputs001.append(dst.as_posix()) 

494 if isinstance(local, Path): 

495 shutil.copy(local, dst) 

496 elif isinstance(local, ZipPath): 

497 _ = local.root.extract(local.at, path=dst) 

498 else: 

499 assert_never(local) 

500 

501 inputs = [tuple(inputs001)] 

502 output_pattern = f"{example_path}/outputs/{{output_id}}/{{sample_id}}.tif" 

503 

504 bioimageio_cli_path = example_path / YAML_FILE 

505 stats_file = "dataset_statistics.json" 

506 stats = (example_path / stats_file).as_posix() 

507 cli_example_args = dict( 

508 inputs=inputs, 

509 outputs=output_pattern, 

510 stats=stats_file, 

511 blockwise=self.blockwise, 

512 ) 

513 assert is_yaml_value(cli_example_args) 

514 write_yaml( 

515 cli_example_args, 

516 bioimageio_cli_path, 

517 ) 

518 

519 yaml_file_content = None 

520 

521 # escaped double quotes 

522 inputs_json = json.dumps(inputs) 

523 inputs_escaped = inputs_json.replace('"', r"\"") 

524 source_escaped = self.source.replace('"', r"\"") 

525 

526 def get_example_command(preview: bool, escape: bool = False): 

527 q: str = '"' if escape else "" 

528 

529 return [ 

530 "bioimageio", 

531 "predict", 

532 # --no-preview not supported for py=3.8 

533 *(["--preview"] if preview else []), 

534 "--overwrite", 

535 *(["--blockwise"] if self.blockwise else []), 

536 f"--stats={q}{stats}{q}", 

537 f"--inputs={q}{inputs_escaped if escape else inputs_json}{q}", 

538 f"--outputs={q}{output_pattern}{q}", 

539 f"{q}{source_escaped if escape else self.source}{q}", 

540 ] 

541 

542 if Path(YAML_FILE).exists(): 

543 logger.info( 

544 "temporarily removing '{}' to execute example prediction", YAML_FILE 

545 ) 

546 yaml_file_content = Path(YAML_FILE).read_bytes() 

547 Path(YAML_FILE).unlink() 

548 

549 try: 

550 _ = subprocess.run(get_example_command(True), check=True) 

551 _ = subprocess.run(get_example_command(False), check=True) 

552 finally: 

553 if yaml_file_content is not None: 

554 _ = Path(YAML_FILE).write_bytes(yaml_file_content) 

555 logger.debug("restored '{}'", YAML_FILE) 

556 

557 print( 

558 "🎉 Sucessfully ran example prediction!\n" 

559 + "To predict the example input using the CLI example config file" 

560 + f" {example_path/YAML_FILE}, execute `bioimageio predict` from {example_path}:\n" 

561 + f"$ cd {str(example_path)}\n" 

562 + f'$ bioimageio predict "{source_escaped}"\n\n' 

563 + "Alternatively run the following command" 

564 + " in the current workind directory, not the example folder:\n$ " 

565 + " ".join(get_example_command(False, escape=True)) 

566 + f"\n(note that a local '{JSON_FILE}' or '{YAML_FILE}' may interfere with this)" 

567 ) 

568 

569 def run(self): 

570 if self.example: 

571 return self._example() 

572 

573 model_descr = ensure_description_is_model(self.descr) 

574 

575 input_ids = get_member_ids(model_descr.inputs) 

576 output_ids = get_member_ids(model_descr.outputs) 

577 

578 minimum_input_ids = tuple( 

579 str(ipt.id) if isinstance(ipt, v0_5.InputTensorDescr) else str(ipt.name) 

580 for ipt in model_descr.inputs 

581 if not isinstance(ipt, v0_5.InputTensorDescr) or not ipt.optional 

582 ) 

583 maximum_input_ids = tuple( 

584 str(ipt.id) if isinstance(ipt, v0_5.InputTensorDescr) else str(ipt.name) 

585 for ipt in model_descr.inputs 

586 ) 

587 

588 def expand_inputs(i: int, ipt: Union[str, Tuple[str, ...]]) -> Tuple[str, ...]: 

589 if isinstance(ipt, str): 

590 ipts = tuple( 

591 ipt.format(model_id=self.descr_id, input_id=t) for t in input_ids 

592 ) 

593 else: 

594 ipts = tuple( 

595 p.format(model_id=self.descr_id, input_id=t) 

596 for t, p in zip(input_ids, ipt) 

597 ) 

598 

599 if len(set(ipts)) < len(ipts): 

600 if len(minimum_input_ids) == len(maximum_input_ids): 

601 n = len(minimum_input_ids) 

602 else: 

603 n = f"{len(minimum_input_ids)}-{len(maximum_input_ids)}" 

604 

605 raise ValueError( 

606 f"[input sample #{i}] Include '{{input_id}}' in path pattern or explicitly specify {n} distinct input paths (got {ipt})" 

607 ) 

608 

609 if len(ipts) < len(minimum_input_ids): 

610 raise ValueError( 

611 f"[input sample #{i}] Expected at least {len(minimum_input_ids)} inputs {minimum_input_ids}, got {ipts}" 

612 ) 

613 

614 if len(ipts) > len(maximum_input_ids): 

615 raise ValueError( 

616 f"Expected at most {len(maximum_input_ids)} inputs {maximum_input_ids}, got {ipts}" 

617 ) 

618 

619 return ipts 

620 

621 inputs = [expand_inputs(i, ipt) for i, ipt in enumerate(self.inputs, start=1)] 

622 

623 sample_paths_in = [ 

624 {t: Path(p) for t, p in zip(input_ids, ipts)} for ipts in inputs 

625 ] 

626 

627 sample_ids = _get_sample_ids(sample_paths_in) 

628 

629 def expand_outputs(): 

630 if isinstance(self.outputs, str): 

631 outputs = [ 

632 tuple( 

633 Path( 

634 self.outputs.format( 

635 model_id=self.descr_id, output_id=t, sample_id=s 

636 ) 

637 ) 

638 for t in output_ids 

639 ) 

640 for s in sample_ids 

641 ] 

642 else: 

643 outputs = [ 

644 tuple( 

645 Path(p.format(model_id=self.descr_id, output_id=t, sample_id=s)) 

646 for t, p in zip(output_ids, self.outputs) 

647 ) 

648 for s in sample_ids 

649 ] 

650 

651 for i, out in enumerate(outputs, start=1): 

652 if len(set(out)) < len(out): 

653 raise ValueError( 

654 f"[output sample #{i}] Include '{{output_id}}' in path pattern or explicitly specify {len(output_ids)} distinct output paths (got {out})" 

655 ) 

656 

657 if len(out) != len(output_ids): 

658 raise ValueError( 

659 f"[output sample #{i}] Expected {len(output_ids)} outputs {output_ids}, got {out}" 

660 ) 

661 

662 return outputs 

663 

664 outputs = expand_outputs() 

665 

666 sample_paths_out = [ 

667 {MemberId(t): Path(p) for t, p in zip(output_ids, out)} for out in outputs 

668 ] 

669 

670 if not self.overwrite: 

671 for sample_paths in sample_paths_out: 

672 for p in sample_paths.values(): 

673 if p.exists(): 

674 raise FileExistsError( 

675 f"{p} already exists. use --overwrite to (re-)write outputs anyway." 

676 ) 

677 if self.preview: 

678 print("🛈 bioimageio prediction preview structure:") 

679 pprint( 

680 { 

681 "{sample_id}": dict( 

682 inputs={"{input_id}": "<input path>"}, 

683 outputs={"{output_id}": "<output path>"}, 

684 ) 

685 } 

686 ) 

687 print("🔎 bioimageio prediction preview output:") 

688 pprint( 

689 { 

690 s: dict( 

691 inputs={t: p.as_posix() for t, p in sp_in.items()}, 

692 outputs={t: p.as_posix() for t, p in sp_out.items()}, 

693 ) 

694 for s, sp_in, sp_out in zip( 

695 sample_ids, sample_paths_in, sample_paths_out 

696 ) 

697 } 

698 ) 

699 return 

700 

701 def input_dataset(stat: Stat): 

702 for s, sp_in in zip(sample_ids, sample_paths_in): 

703 yield load_sample_for_model( 

704 model=model_descr, 

705 paths=sp_in, 

706 stat=stat, 

707 sample_id=s, 

708 ) 

709 

710 stat: Dict[Measure, MeasureValue] = dict( 

711 _get_stat( 

712 model_descr, input_dataset({}), len(sample_ids), self.stats 

713 ).items() 

714 ) 

715 

716 pp = create_prediction_pipeline( 

717 model_descr, 

718 weight_format=None if self.weight_format == "any" else self.weight_format, 

719 ) 

720 predict_method = ( 

721 pp.predict_sample_with_blocking 

722 if self.blockwise 

723 else pp.predict_sample_without_blocking 

724 ) 

725 

726 for sample_in, sp_out in tqdm( 

727 zip(input_dataset(dict(stat)), sample_paths_out), 

728 total=len(inputs), 

729 desc=f"predict with {self.descr_id}", 

730 unit="sample", 

731 ): 

732 sample_out = predict_method(sample_in) 

733 save_sample(sp_out, sample_out) 

734 

735 

736class AddWeightsCmd(CmdBase, WithSource, WithSummaryLogging): 

737 output: CliPositionalArg[Path] 

738 """The path to write the updated model package to.""" 

739 

740 source_format: Optional[SupportedWeightsFormat] = Field(None, alias="source-format") 

741 """Exclusively use these weights to convert to other formats.""" 

742 

743 target_format: Optional[SupportedWeightsFormat] = Field(None, alias="target-format") 

744 """Exclusively add this weight format.""" 

745 

746 verbose: bool = False 

747 """Log more (error) output.""" 

748 

749 def run(self): 

750 model_descr = ensure_description_is_model(self.descr) 

751 if isinstance(model_descr, v0_4.ModelDescr): 

752 raise TypeError( 

753 f"model format {model_descr.format_version} not supported." 

754 + " Please update the model first." 

755 ) 

756 updated_model_descr = add_weights( 

757 model_descr, 

758 output_path=self.output, 

759 source_format=self.source_format, 

760 target_format=self.target_format, 

761 verbose=self.verbose, 

762 ) 

763 if updated_model_descr is None: 

764 return 

765 

766 self.log(updated_model_descr) 

767 

768 

769JSON_FILE = "bioimageio-cli.json" 

770YAML_FILE = "bioimageio-cli.yaml" 

771 

772 

773class Bioimageio( 

774 BaseSettings, 

775 cli_implicit_flags=True, 

776 cli_parse_args=True, 

777 cli_prog_name="bioimageio", 

778 cli_use_class_docs_for_groups=True, 

779 use_attribute_docstrings=True, 

780): 

781 """bioimageio - CLI for bioimage.io resources 🦒""" 

782 

783 model_config = SettingsConfigDict( 

784 json_file=JSON_FILE, 

785 yaml_file=YAML_FILE, 

786 ) 

787 

788 validate_format: CliSubCommand[ValidateFormatCmd] = Field(alias="validate-format") 

789 "Check a resource's metadata format" 

790 

791 test: CliSubCommand[TestCmd] 

792 "Test a bioimageio resource (beyond meta data formatting)" 

793 

794 package: CliSubCommand[PackageCmd] 

795 "Package a resource" 

796 

797 predict: CliSubCommand[PredictCmd] 

798 "Predict with a model resource" 

799 

800 update_format: CliSubCommand[UpdateFormatCmd] = Field(alias="update-format") 

801 """Update the metadata format""" 

802 

803 update_hashes: CliSubCommand[UpdateHashesCmd] = Field(alias="update-hashes") 

804 """Create a bioimageio.yaml description with updated file hashes.""" 

805 

806 add_weights: CliSubCommand[AddWeightsCmd] = Field(alias="add-weights") 

807 """Add additional weights to the model descriptions converted from available 

808 formats to improve deployability.""" 

809 

810 @classmethod 

811 def settings_customise_sources( 

812 cls, 

813 settings_cls: Type[BaseSettings], 

814 init_settings: PydanticBaseSettingsSource, 

815 env_settings: PydanticBaseSettingsSource, 

816 dotenv_settings: PydanticBaseSettingsSource, 

817 file_secret_settings: PydanticBaseSettingsSource, 

818 ) -> Tuple[PydanticBaseSettingsSource, ...]: 

819 cli: CliSettingsSource[BaseSettings] = CliSettingsSource( 

820 settings_cls, 

821 cli_parse_args=True, 

822 formatter_class=RawTextHelpFormatter, 

823 ) 

824 sys_args = pformat(sys.argv) 

825 logger.info("starting CLI with arguments:\n{}", sys_args) 

826 return ( 

827 cli, 

828 init_settings, 

829 YamlConfigSettingsSource(settings_cls), 

830 JsonConfigSettingsSource(settings_cls), 

831 ) 

832 

833 @model_validator(mode="before") 

834 @classmethod 

835 def _log(cls, data: Any): 

836 logger.info( 

837 "loaded CLI input:\n{}", 

838 pformat({k: v for k, v in data.items() if v is not None}), 

839 ) 

840 return data 

841 

842 def run(self): 

843 logger.info( 

844 "executing CLI command:\n{}", 

845 pformat({k: v for k, v in self.model_dump().items() if v is not None}), 

846 ) 

847 cmd = ( 

848 self.add_weights 

849 or self.package 

850 or self.predict 

851 or self.test 

852 or self.update_format 

853 or self.update_hashes 

854 or self.validate_format 

855 ) 

856 assert cmd is not None 

857 cmd.run() 

858 

859 

860assert isinstance(Bioimageio.__doc__, str) 

861Bioimageio.__doc__ += f""" 

862 

863library versions: 

864 bioimageio.core {VERSION} 

865 bioimageio.spec {VERSION} 

866 

867spec format versions: 

868 model RDF {ModelDescr.implemented_format_version} 

869 dataset RDF {DatasetDescr.implemented_format_version} 

870 notebook RDF {NotebookDescr.implemented_format_version} 

871 

872""" 

873 

874 

875def _get_sample_ids( 

876 input_paths: Sequence[Mapping[MemberId, Path]], 

877) -> Sequence[SampleId]: 

878 """Get sample ids for given input paths, based on the common path per sample. 

879 

880 Falls back to sample01, samle02, etc...""" 

881 

882 matcher = SequenceMatcher() 

883 

884 def get_common_seq(seqs: Sequence[Sequence[str]]) -> Sequence[str]: 

885 """extract a common sequence from multiple sequences 

886 (order sensitive; strips whitespace and slashes) 

887 """ 

888 common = seqs[0] 

889 

890 for seq in seqs[1:]: 

891 if not seq: 

892 continue 

893 matcher.set_seqs(common, seq) 

894 i, _, size = matcher.find_longest_match()