Coverage for src/bioimageio/core/io.py: 72%

692 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 09:46 +0000

1from __future__ import annotations 

2 

3import atexit 

4import json 

5import os 

6import warnings 

7import zipfile 

8from collections.abc import Callable, Iterable, Mapping, Sequence 

9from concurrent.futures import ThreadPoolExecutor 

10from contextlib import nullcontext 

11from io import BytesIO, TextIOWrapper 

12from itertools import chain 

13from pathlib import Path, PosixPath, PurePath, PurePosixPath 

14from shutil import copyfileobj 

15from typing import TYPE_CHECKING, Any, Literal, TypeAlias, TypedDict, cast 

16 

17import dask.array as da 

18import dask.diagnostics 

19import numpy as np 

20import pydantic 

21import xarray as xr 

22from exceptiongroup import ExceptionGroup 

23from imageio.v3 import ( 

24 imread, # pyright: ignore[reportUnknownVariableType] 

25 imwrite, # type: ignore 

26) 

27from loguru import logger 

28from pydantic import BaseModel, RootModel, TypeAdapter 

29from typing_extensions import TypeAliasType as _TypeAliasType 

30from typing_extensions import assert_never 

31 

32from bioimageio.spec._internal.io import ( 

33 RelativeDirectory, 

34 ZarrSource, 

35 ZarrUrl, 

36 get_reader, 

37 interprete_file_source, 

38) 

39from bioimageio.spec.common import ( 

40 BytesReader, 

41 FileDescr, 

42 FileSource, 

43 FtpUrl, 

44 HttpUrl, 

45 PermissiveFileSource, 

46 RelativeFilePath, 

47 RootHttpUrl, 

48 ZipPath, 

49) 

50from bioimageio.spec.utils import load_array, save_array 

51 

52from ._utils.color import hex_to_rgb 

53from ._utils.io_zarr import open_zarr_multiscale_array 

54from ._utils.tiff_metadata import ( 

55 ImageJMetadata, 

56 OmeChannelMetadata, 

57 OmeMetadata, 

58 create_ome_metadata_from_xml_string, 

59) 

60from ._utils.type_guards import is_sequence, is_tuple 

61from ._utils.type_hints import cast_and_assert 

62from .axis import ( 

63 BATCH_AXIS_ID, 

64 AxisId, 

65 AxisLike, 

66 PerAxis, 

67 single_letter_dims_if_possible, 

68) 

69from .common import PerMember 

70from .sample import Sample 

71from .stat_measures import DatasetMeasure, MeasureValue, SampleMeasure, Stat 

72from .tensor import Tensor 

73 

74if TYPE_CHECKING: 

75 JsonValue: TypeAlias = ( 

76 bool | int | float | str | None | list["JsonValue"] | dict[str, "JsonValue"] 

77 ) # note: order relevant for deserializing 

78 

79else: 

80 # for pydantic validation we need to use `TypeAliasType`, 

81 # see https://docs.pydantic.dev/latest/concepts/types/#named-recursive-types 

82 # however this results in a partially unknown type with the current pyright 1.1.388 

83 JsonValue: TypeAlias = _TypeAliasType( 

84 "JsonValue", 

85 bool | int | float | str | None | list["JsonValue"] | dict[str, "JsonValue"], 

86 ) 

87 

88JsonValueReadOnly: TypeAlias = ( 

89 bool 

90 | int 

91 | float 

92 | str 

93 | None 

94 | Sequence["JsonValueReadOnly"] 

95 | Mapping[str, "JsonValueReadOnly"] 

96) 

97 

98IO_Lib: TypeAlias = Literal[ 

99 "bioio", "imageio", "clearscale", "numpy", "tifffile", "h5py" 

100] 

101 

102 

103def load_tensor( 

104 source: FileDescr | PermissiveFileSource | ZipPath, 

105 /, 

106 axes: Sequence[AxisLike] | None = None, 

107 io_lib: IO_Lib | None = None, 

108) -> Tensor: 

109 """Load an image tensor 

110 

111 Args: 

112 source: The source of the image data. 

113 axes: The (desired) axes of the image data. 

114 io_lib: library to use for loading the image. 

115 Defaults to trying various libraries depending on file extension. 

116 

117 Returns: 

118 The loaded tensor. 

119 """ 

120 tensor, _unstacked_dim_order = load_tensor_and_return_unstacked_dim_order( 

121 source, axes=axes, io_lib=io_lib 

122 ) 

123 return tensor 

124 

125 

126def load_tensor_and_return_unstacked_dim_order( 

127 source: FileDescr | PermissiveFileSource | ZipPath, 

128 /, 

129 axes: Sequence[AxisLike] | None = None, 

130 io_lib: IO_Lib | None = None, 

131) -> tuple[Tensor, Sequence[AxisId] | None]: 

132 """Load an image tensor 

133 

134 Args: 

135 source: The source of the image data. 

136 axes: The (desired) axes of the image data. 

137 io_lib: library to use for loading the image. 

138 Defaults to trying various libraries depending on file extension. 

139 

140 Returns: 

141 A tuple of the loaded tensor and the unstacked dimension order if the initially loaded 

142 tensor had extra dimensions that were stacked to a multi-index batch dimension. 

143 """ 

144 file_source, extensions, internal_path = _interprete_tensor_source(source) 

145 if io_lib is None: 

146 try_io_libs = _select_io_libs_based_on_extension(extensions) 

147 

148 exceptions: list[Exception] = [] 

149 for auto_io_lib in try_io_libs: 149 ↛ 160line 149 didn't jump to line 160 because the loop on line 149 didn't complete

150 try: 

151 return load_tensor_and_return_unstacked_dim_order( 

152 source, axes=axes, io_lib=auto_io_lib 

153 ) 

154 except Exception as e: 

155 exceptions.append(e) 

156 logger.opt(exception=e).warning( 

157 "Failed to load tensor with io_lib={}", auto_io_lib 

158 ) 

159 

160 raise ExceptionGroup( 

161 "Failed to load tensor with any available io_lib", exceptions 

162 ) 

163 

164 elif io_lib == "bioio": 

165 t = _load_tensor_bioio( 

166 file_source, None if internal_path is None else str(internal_path) 

167 ) 

168 elif io_lib == "clearscale": 

169 t = _load_tensor_clearscale(source) 

170 elif io_lib == "imageio": 

171 reader = get_reader(source) 

172 array = imread( 

173 reader.read(), 

174 extension=extensions[-1] if extensions else None, # pyright: ignore[reportArgumentType] 

175 ) 

176 try: 

177 t = Tensor.from_array(array, dims=axes) 

178 except Exception: 

179 if axes is None: 

180 raise 

181 

182 t = Tensor.from_array(array, dims=None) 

183 else: 

184 axes = None 

185 elif io_lib == "numpy": 

186 array = load_array(source) 

187 try: 

188 t = Tensor.from_array(array, dims=axes) 

189 except Exception: 

190 if axes is None: 

191 raise 

192 

193 t = Tensor.from_array(array, dims=None) 

194 else: 

195 axes = None 

196 

197 elif io_lib == "tifffile": 

198 t = _load_tensor_tifffile(source) 

199 elif io_lib == "h5py": 

200 t = _load_tensor_h5py(file_source, internal_path, axes=axes) 

201 else: 

202 assert_never(io_lib) 

203 

204 if axes is None: 

205 unstacked_dim_order = None 

206 else: 

207 t, unstacked_dim_order = t.transpose_and_return_unstacked_dim_order( 

208 axes, extra_dims="squeeze_or_stack", missing_dims="unstack_or_expand" 

209 ) 

210 

211 logger.info("Loaded tensor {} with {}", t.tagged_shape, io_lib) 

212 return t, unstacked_dim_order 

213 

214 

215def _select_io_libs_based_on_extension( 

216 extensions: list[str] | None, 

217) -> tuple[IO_Lib, ...]: 

218 if not extensions: 218 ↛ 219line 218 didn't jump to line 219 because the condition on line 218 was never true

219 return ("clearscale", "bioio", "imageio") 

220 

221 if extensions[-2:] in ([".ome", ".tif"], [".ome", ".tiff"]): 221 ↛ 222line 221 didn't jump to line 222 because the condition on line 221 was never true

222 return ("tifffile", "bioio") 

223 elif extensions[-1] == ".npy": 

224 return ("numpy",) 

225 elif extensions[-1] == ".zarr": 

226 return ("clearscale", "bioio") 

227 elif extensions[-1] in (".tif", ".tiff"): 

228 return ("tifffile", "imageio") 

229 elif extensions[-1] in (".h4", ".h5", ".hdf", ".hdf4", ".hdf5", ".he5"): 

230 return ("h5py", "bioio") 

231 else: 

232 return ("bioio", "imageio") 

233 

234 

235def _load_tensor_h5py( 

236 source: ZarrSource | FileSource | ZipPath, 

237 internal_path: PurePath | None, 

238 *, 

239 axes: Sequence[AxisLike] | None = None, 

240) -> Tensor: 

241 import h5py # pyright: ignore[reportMissingTypeStubs] 

242 

243 if isinstance(source, RelativeDirectory): 243 ↛ 244line 243 didn't jump to line 244 because the condition on line 243 was never true

244 source = source.absolute() 

245 

246 file_source = ( 

247 source if isinstance(source, Path) else BytesIO(get_reader(source).read()) 

248 ) 

249 f = h5py.File(file_source, "r") 

250 # we need to keep the file open for lazy loading, so we close it on exit 

251 _ = atexit.register(f.close) 

252 if internal_path is not None: 

253 key = internal_path.as_posix() 

254 elif len(f.keys()) == 0: 

255 raise ValueError(f"HDF5 file {source} is empty") 

256 else: 

257 if "data" in f: 257 ↛ 260line 257 didn't jump to line 260 because the condition on line 257 was always true

258 key = "data" 

259 else: 

260 key = str(min(f.keys())) # pyright: ignore[reportUnknownArgumentType] 

261 

262 if len(f.keys()) > 1: 

263 warnings.warn( 

264 f"choosing to load dataset '{key}' from HDF5 file with multiple datasets: {list(f.keys())}" 

265 ) 

266 

267 dataset = f[key] 

268 if not isinstance(dataset, h5py.Dataset): 

269 raise TypeError(f"HDF5 path {internal_path} does not point to a dataset") 

270 

271 labels = cast(Any, dataset.attrs.get("DIMENSION_LABELS")) 

272 if isinstance(labels, np.ndarray): 

273 labels = labels.tolist() 

274 if not is_sequence(labels): 

275 if axes is None or dataset.ndim == len(axes): 275 ↛ 279line 275 didn't jump to line 279 because the condition on line 275 was always true

276 dims = axes 

277 dims_out = None 

278 else: 

279 dims = None # guess dims 

280 dims_out = axes 

281 else: 

282 dims = tuple(AxisId(str(a)) for a in labels) 

283 dims_out = axes 

284 

285 tensor = Tensor.from_array( 

286 cast_and_assert(da.Array, da.from_array(dataset)), dims=dims 

287 ) 

288 for attr, setter in ( 

289 ("axis_units", tensor.set_physical_scale_unit), 

290 ("axis_scale", tensor.set_physical_scale), 

291 ): 

292 values = cast(Any, dataset.attrs.get(attr)) 

293 if isinstance(values, np.ndarray): 

294 values = values.tolist() 

295 if is_sequence(values): 

296 if len(values) != tensor.ndim: 

297 raise ValueError(f"HDF5 attribute {attr} must have one value per axis") 

298 setter(dict(zip(tensor.dims, values))) 

299 

300 if dims_out is not None: 300 ↛ 301line 300 didn't jump to line 301 because the condition on line 300 was never true

301 tensor = tensor.transpose( 

302 dims_out, extra_dims="squeeze_or_stack", missing_dims="expand" 

303 ) 

304 

305 return tensor 

306 

307 

308def _load_tensor_bioio( 

309 source: ZarrSource | FileSource | ZipPath, 

310 internal_path: str | None, 

311) -> Tensor: 

312 """Load an image tensor using bioio""" 

313 

314 import bioio 

315 

316 logger.debug("Loading tensor from {} with bioio", source) 

317 

318 if isinstance(source, FileDescr): 318 ↛ 319line 318 didn't jump to line 319 because the condition on line 318 was never true

319 src = source.source 

320 else: 

321 src = source 

322 

323 if isinstance(src, (RelativeFilePath, RelativeDirectory)): 323 ↛ 324line 323 didn't jump to line 324 because the condition on line 323 was never true

324 src = src.absolute() 

325 

326 del source 

327 

328 try: 

329 img = bioio.BioImage(src) 

330 except Exception: 

331 # try to open zarr groups along the internal path for local zarr storage 

332 if not isinstance(src, Path) or internal_path is None: 332 ↛ 333line 332 didn't jump to line 333 because the condition on line 332 was never true

333 raise 

334 

335 img = None 

336 for gs in chain([src / internal_path], (src / internal_path).parents): 336 ↛ 353line 336 didn't jump to line 353 because the loop on line 336 didn't complete

337 if gs == src: 337 ↛ 338line 337 didn't jump to line 338 because the condition on line 337 was never true

338 break 

339 try: 

340 import bioio_ome_zarr 

341 

342 img = bioio.BioImage(gs, reader=bioio_ome_zarr.Reader) 

343 except Exception: 

344 logger.info(f"Failed to load tensor with bioio from {gs}") 

345 else: 

346 if gs == src / internal_path: 

347 internal_path = None 

348 else: 

349 internal_path = (src / internal_path).relative_to(gs).as_posix() 

350 

351 break 

352 

353 if img is None: 353 ↛ 354line 353 didn't jump to line 354 because the condition on line 353 was never true

354 raise 

355 

356 if internal_path is not None: 

357 internal_path_found = False 

358 try: 

359 import bioio_ome_zarr 

360 

361 except ImportError: 

362 pass 

363 else: 

364 if isinstance(img.reader, bioio_ome_zarr.Reader): 364 ↛ 376line 364 didn't jump to line 376 because the condition on line 364 was always true

365 multiscale = img.reader._multiscales_metadata[img.current_scene_index] # pyright: ignore 

366 datasets = multiscale.get("datasets", []) # pyright: ignore 

367 data_paths = [datasets[i].get("path") for i in img.resolution_levels] # pyright: ignore 

368 if internal_path not in data_paths: 368 ↛ 369line 368 didn't jump to line 369 because the condition on line 368 was never true

369 raise ValueError( 

370 f"Resolution level {internal_path} not found in multiscales datasets: {data_paths}" 

371 ) 

372 resolution_level = data_paths.index(internal_path) 

373 img.set_resolution_level(resolution_level) 

374 internal_path_found = True 

375 else: 

376 raise NotImplementedError( 

377 f"Reading from internal path {internal_path} is not implemented for bioio reader {type(img.reader)}" # pyright: ignore[reportUnknownArgumentType] 

378 ) 

379 

380 if not internal_path_found: 380 ↛ 381line 380 didn't jump to line 381 because the condition on line 380 was never true

381 raise ValueError( 

382 f"Internal path {internal_path} not found with bioio in {src}" 

383 ) 

384 

385 dataarray_bioio = cast_and_assert(xr.DataArray, img.xarray_dask_data.squeeze()) 

386 

387 # map bioio TCZYX/YXS axes to AxisId objects 

388 # (e.g. "T" -> AxisId("time"), "C" -> AxisId("channel"), "S" -> AxisId("channel")) 

389 axes_map = { 

390 d: AxisId("channel") if d == "S" else AxisId(d) for d in dataarray_bioio.dims 

391 } 

392 dataarray = cast_and_assert(xr.DataArray, dataarray_bioio.rename(axes_map)) 

393 

394 t = Tensor.from_xarray(dataarray) 

395 

396 # add physical scale 

397 scale_bioio: dict[str, float | None] = { 

398 "T": img.scale.T, 

399 "C": img.scale.C, 

400 "Z": img.scale.Z, 

401 "Y": img.scale.Y, 

402 "X": img.scale.X, 

403 } 

404 if (raw_c_unit := img.dimension_properties.C.unit) is None or str( # pyright: ignore[reportUnknownVariableType] 404 ↛ 409line 404 didn't jump to line 409 because the condition on line 404 was always true

405 raw_c_unit # pyright: ignore[reportUnknownArgumentType] 

406 ) == "dimensionless": 

407 c_unit = None 

408 else: 

409 c_unit = str(raw_c_unit) # pyright: ignore[reportUnknownArgumentType] 

410 unit_bioio: dict[str, str | None] = { 

411 "T": img.dimension_properties.T.unit, 

412 "C": c_unit, 

413 "Z": img.dimension_properties.Z.unit, 

414 "Y": img.dimension_properties.Y.unit, 

415 "X": img.dimension_properties.X.unit, 

416 } 

417 scale = { 

418 axes_map[k]: v 

419 for k, v in scale_bioio.items() 

420 if k in axes_map and v is not None 

421 } 

422 unit = { 

423 axes_map[k]: str(v) 

424 for k, v in unit_bioio.items() 

425 if k in axes_map and v is not None 

426 } 

427 try: 

428 t.set_physical_scale(scale) 

429 except Exception as e: 

430 logger.opt(exception=e).warning( 

431 "Failed to interprete physical scale: {}", scale 

432 ) 

433 try: 

434 t.set_physical_scale_unit(unit) 

435 except Exception as e: 

436 logger.opt(exception=e).warning( 

437 "Failed to interprete physical scale unit: {}", unit 

438 ) 

439 return t 

440 

441 

442def _load_tensor_clearscale(source: FileDescr | PermissiveFileSource) -> Tensor: 

443 """load a single image as numpy array""" 

444 import dask.array as da 

445 

446 if isinstance(source, FileDescr): 446 ↛ 447line 446 didn't jump to line 447 because the condition on line 446 was never true

447 source = source.source.absolute() 

448 

449 if isinstance(source, RelativeFilePath): 449 ↛ 450line 449 didn't jump to line 450 because the condition on line 449 was never true

450 source = source.absolute() 

451 

452 if isinstance(source, ZipPath): 452 ↛ 453line 452 didn't jump to line 453 because the condition on line 452 was never true

453 raise NotImplementedError( 

454 "clearscale: Loading from zip files is not implemented." 

455 ) 

456 

457 logger.debug("loading tensor from {} with clearscale+zarr-python", source) 

458 array, scale = open_zarr_multiscale_array( 

459 source.as_posix() if isinstance(source, Path) else str(source) 

460 ) 

461 image = da.from_zarr(array) 

462 axes = tuple(str(a) for a in scale.shape) 

463 tensor = Tensor.from_xarray(xr.DataArray(image, dims=axes)) 

464 try: 

465 tensor.set_physical_scale( 

466 {AxisId(str(a)): value for a, value in scale.pixel_size.items()} 

467 ) 

468 except Exception as e: 

469 logger.opt(exception=e).warning( 

470 "Failed to interprete clearscale scale.pixel_size: {}", scale.pixel_size 

471 ) 

472 try: 

473 tensor.set_physical_scale_unit( 

474 {AxisId(str(a)): value for a, value in scale.unit.items()} 

475 ) 

476 except Exception as e: 

477 logger.opt(exception=e).warning( 

478 "Failed to interprete clearscale scale.unit: {}", scale.unit 

479 ) 

480 return tensor 

481 

482 

483def _load_tensor_tifffile(source: FileDescr | PermissiveFileSource) -> Tensor: 

484 import tifffile 

485 

486 logger.debug("loading tensor from {} with tifffile", source) 

487 if isinstance(source, FileDescr): 487 ↛ 488line 487 didn't jump to line 488 because the condition on line 487 was never true

488 source = source.source.absolute() 

489 

490 if isinstance(source, RelativeFilePath): 490 ↛ 491line 490 didn't jump to line 491 because the condition on line 490 was never true

491 source = source.absolute() 

492 

493 if isinstance(source, ZipPath): 493 ↛ 494line 493 didn't jump to line 494 because the condition on line 493 was never true

494 ctxt = source.open("rb") 

495 name = source.name 

496 suffixes = source.suffixes 

497 else: 

498 if isinstance(source, (RootHttpUrl, pydantic.AnyUrl)): 498 ↛ 499line 498 didn't jump to line 499 because the condition on line 498 was never true

499 source = str(source) 

500 

501 if isinstance(source, Path): 501 ↛ 504line 501 didn't jump to line 504 because the condition on line 501 was always true

502 suffixes = source.suffixes 

503 else: 

504 suffixes = PurePosixPath(source).suffixes 

505 

506 ctxt = nullcontext(source) 

507 name = None 

508 

509 with ctxt as f: 

510 assert not isinstance(f, TextIOWrapper) 

511 

512 tif = tifffile.TiffFile(f, name=name) 

513 if not hasattr(tif, "asxarray"): 

514 raise ImportError( 

515 "tifffile version >= 2026.5.2 is required for loading tensors with tifffile" 

516 ) 

517 with tif: 

518 xr_data = tif.asxarray() 

519 page = tif.pages[0] 

520 if isinstance(page, tifffile.TiffFrame): 520 ↛ 521line 520 didn't jump to line 521 because the condition on line 520 was never true

521 page = None if page.pages is None else page.pages[0] 

522 if isinstance(page, tifffile.TiffFrame): 

523 page = None 

524 

525 x_res = None 

526 y_res = None 

527 unit = None 

528 if page is not None: 528 ↛ 554line 528 didn't jump to line 554 because the condition on line 528 was always true

529 if "XResolution" in page.tags: 529 ↛ 540line 529 didn't jump to line 540 because the condition on line 529 was always true

530 try: 

531 x_res = page.tags["XResolution"].value 

532 x_res = x_res[0] / x_res[1] if is_tuple(x_res) else x_res 

533 x_res = None if x_res is None else float(x_res) 

534 except Exception as e: 

535 logger.opt(exception=e).warning( 

536 "Failed to interprete tiff XResolution tag: {}", x_res 

537 ) 

538 x_res = None 

539 

540 if "YResolution" in page.tags: 540 ↛ 551line 540 didn't jump to line 551 because the condition on line 540 was always true

541 try: 

542 y_res = page.tags["YResolution"].value 

543 y_res = y_res[0] / y_res[1] if is_tuple(y_res) else y_res 

544 y_res = None if y_res is None else float(y_res) 

545 except Exception as e: 

546 logger.opt(exception=e).warning( 

547 "Failed to interprete tiff YResolution tag: {}", y_res 

548 ) 

549 y_res = None 

550 

551 unit_tag = page.tags.get("ResolutionUnit") 

552 unit = unit_tag.value if unit_tag else None 

553 

554 ij_metadata = ImageJMetadata(**(tif.imagej_metadata or {})) 

555 ome_metadata = None 

556 if ( 556 ↛ 560line 556 didn't jump to line 560 because the condition on line 556 was never true

557 suffixes[-2:] in ([".ome", ".tif"], [".ome", ".tiff"]) 

558 or not ij_metadata 

559 ) and tif.ome_metadata is not None: 

560 try: 

561 ome_metadata = create_ome_metadata_from_xml_string(tif.ome_metadata) 

562 except Exception as e: 

563 logger.opt(exception=e).warning( 

564 "Failed to parse OME-XML metadata from tif.ome_metadata" 

565 ) 

566 

567 if ome_metadata is None: 567 ↛ 570line 567 didn't jump to line 570

568 ome_metadata = OmeMetadata() 

569 

570 t = Tensor.from_xarray(xr_data) 

571 

572 # set physical scale 

573 x_scale = ome_metadata.PhysicalSizeX or ( 

574 1 / x_res if x_res is not None and x_res != 0 else None 

575 ) 

576 y_scale = ome_metadata.PhysicalSizeY or ( 

577 1 / y_res if y_res is not None and y_res != 0 else x_scale 

578 ) 

579 z_scale = ome_metadata.PhysicalSizeZ or ij_metadata.spacing or x_scale 

580 t_scale = ( 

581 ome_metadata.TimeIncrement or ij_metadata.finterval or None 

582 if ij_metadata.fps is None 

583 else 1 / ij_metadata.fps 

584 ) 

585 scale = { 

586 AxisId("x"): x_scale, 

587 AxisId("y"): y_scale, 

588 AxisId("z"): z_scale, 

589 AxisId("time"): t_scale, 

590 } 

591 try: 

592 t.set_physical_scale({k: v for k, v in scale.items() if k in t.dims}) 

593 except Exception as e: 

594 logger.opt(exception=e).warning( 

595 "Failed to interprete physical scale from tif metadata: {}", scale 

596 ) 

597 

598 # set physical scale unit 

599 x_unit = ome_metadata.PhysicalSizeXUnit or ij_metadata.unit 

600 if x_unit is None and unit is not None: 600 ↛ 601line 600 didn't jump to line 601 because the condition on line 600 was never true

601 try: 

602 x_unit = tifffile.RESUNIT(unit).name.lower() 

603 except Exception as e: 

604 logger.opt(exception=e).warning( 

605 "Failed to interprete tiff ResolutionUnit tag: {}", unit 

606 ) 

607 x_unit = None 

608 

609 y_unit = ome_metadata.PhysicalSizeYUnit or ij_metadata.yunit or x_unit 

610 z_unit = ome_metadata.PhysicalSizeZUnit or ij_metadata.zunit or x_unit 

611 t_unit = ome_metadata.TimeIncrementUnit or "s" 

612 

613 scale_unit = { 

614 AxisId("x"): x_unit, 

615 AxisId("y"): y_unit, 

616 AxisId("z"): z_unit, 

617 AxisId("time"): t_unit, 

618 } 

619 try: 

620 t.set_physical_scale_unit({k: v for k, v in scale_unit.items() if k in t.dims}) 

621 except Exception as e: 

622 logger.opt(exception=e).warning( 

623 "Failed to interprete physical scale unit from tif metadata: {}", scale_unit 

624 ) 

625 

626 # set channel names 

627 if ome_metadata.Channel is not None: 627 ↛ 628line 627 didn't jump to line 628 because the condition on line 627 was never true

628 t.channel_names = ome_metadata.Channel.Name 

629 

630 return t 

631 

632 

633Suffix = str 

634 

635 

636def save_tensor( 

637 path: Path | str, 

638 tensor: Tensor, 

639 *, 

640 io_lib: IO_Lib | None = None, 

641 chunk_shape: PerAxis[int] | None = None, 

642 shard_shape: PerAxis[int] | None = None, 

643 progressbar: bool = not os.getenv("GITHUB_ACTIONS"), 

644) -> None: 

645 _ = _save_tensor_impl( 

646 path, 

647 tensor, 

648 io_lib=io_lib, 

649 chunk_shape=chunk_shape, 

650 shard_shape=shard_shape, 

651 progressbar=progressbar, 

652 ) 

653 

654 

655def _save_tensor_impl( 

656 path: Path | str, 

657 tensor: Tensor, 

658 *, 

659 io_lib: IO_Lib | None, 

660 chunk_shape: PerAxis[int] | None, 

661 shard_shape: PerAxis[int] | None, 

662 progressbar: bool, 

663) -> IO_Lib: 

664 try: 

665 output_path, extensions, internal_path = _interprete_tensor_source(path) 

666 except Exception: 

667 output_path, extensions, internal_path = _interprete_tensor_target(path) 

668 

669 if ( 

670 extensions is None 

671 or ".npy" not in extensions 

672 and tensor.tagged_shape.get(BATCH_AXIS_ID, 2) == 1 

673 ): 

674 tensor = tensor.squeeze(BATCH_AXIS_ID) 

675 

676 if isinstance(output_path, RootHttpUrl): 676 ↛ 677line 676 didn't jump to line 677 because the condition on line 676 was never true

677 raise NotImplementedError( 

678 "Saving tensors to HTTP URLs is not implemented. Please save to a local file path." 

679 ) 

680 

681 if io_lib is None: 

682 try_io_libs = _select_io_libs_based_on_extension(extensions) 

683 exceptions: list[Exception] = [] 

684 for try_io_lib in try_io_libs: 684 ↛ 700line 684 didn't jump to line 700 because the loop on line 684 didn't complete

685 try: 

686 return _save_tensor_impl( 

687 path, 

688 tensor, 

689 io_lib=try_io_lib, 

690 chunk_shape=chunk_shape, 

691 shard_shape=shard_shape, 

692 progressbar=progressbar, 

693 ) 

694 except Exception as e: 

695 exceptions.append(e) 

696 logger.opt(exception=e).warning( 

697 "Failed to save tensor with io_lib={}", try_io_lib 

698 ) 

699 

700 raise ExceptionGroup( 

701 "Failed to save tensor with any available io_lib", exceptions 

702 ) 

703 

704 if progressbar: 704 ↛ 705line 704 didn't jump to line 705 because the condition on line 704 was never true

705 pbar_ctxt = dask.diagnostics.ProgressBar() # pyright: ignore[reportPrivateImportUsage] 

706 else: 

707 pbar_ctxt = nullcontext() 

708 

709 with pbar_ctxt: 

710 if io_lib == "bioio": 

711 _save_tensor_bioio( 

712 output_path, 

713 tensor, 

714 internal_path=internal_path, 

715 chunk_shape=chunk_shape, 

716 shard_shape=shard_shape, 

717 ) 

718 elif io_lib == "clearscale": 

719 _save_tensor_clearscale( 

720 output_path, 

721 tensor, 

722 internal_path=internal_path, 

723 chunk_shape=chunk_shape, 

724 shard_shape=shard_shape, 

725 ) 

726 elif io_lib == "imageio": 

727 _save_tensor_imageio(output_path, tensor, internal_path=internal_path) 

728 elif io_lib == "numpy": 

729 if internal_path: 729 ↛ 730line 729 didn't jump to line 730 because the condition on line 729 was never true

730 raise NotImplementedError( 

731 "Saving to an internal path is not implemented for numpy." 

732 ) 

733 _save_tensor_numpy(output_path, tensor) 

734 elif io_lib == "tifffile": 

735 if internal_path is not None: 735 ↛ 736line 735 didn't jump to line 736 because the condition on line 735 was never true

736 raise NotImplementedError( 

737 "Saving to an internal path is not implemented for tifffile." 

738 ) 

739 

740 _save_tensor_tifffile(output_path, extensions, tensor) 

741 elif io_lib == "h5py": 

742 _save_tensor_h5py( 

743 output_path, 

744 tensor, 

745 internal_path=internal_path, 

746 chunk_shape=chunk_shape, 

747 ) 

748 else: 

749 assert_never(io_lib) 

750 

751 return io_lib 

752 

753 

754def _save_tensor_h5py( 

755 path: Path | ZipPath | FtpUrl, 

756 tensor: Tensor, 

757 internal_path: PurePath | None, 

758 chunk_shape: PerAxis[int] | None = None, 

759) -> None: 

760 import h5py # pyright: ignore[reportMissingTypeStubs] 

761 

762 if not isinstance(path, Path): 762 ↛ 763line 762 didn't jump to line 763 because the condition on line 762 was never true

763 raise NotImplementedError("Saving with h5py requires a local file path.") 

764 

765 name = internal_path.as_posix() if internal_path is not None else "data" 

766 path.parent.mkdir(parents=True, exist_ok=True) 

767 with h5py.File(path, "a") as f: 

768 if name in f: 

769 if not isinstance(f[name], h5py.Dataset): 769 ↛ 770line 769 didn't jump to line 770 because the condition on line 769 was never true

770 raise ValueError(f"HDF5 path {name} does not point to a dataset") 

771 del f[name] 

772 

773 dataset = f.create_dataset( 

774 name, 

775 dtype=tensor.dtype, 

776 shape=tensor.shape, 

777 chunks=tuple(chunk_shape[a] for a in tensor.dims) 

778 if chunk_shape is not None 

779 else True, 

780 compression="lzf", 

781 ) 

782 dataset.attrs["DIMENSION_LABELS"] = [str(a) for a in tensor.dims] 

783 units = tensor.get_physical_scale_unit() 

784 if any(units.values()): 784 ↛ 786line 784 didn't jump to line 786 because the condition on line 784 was always true

785 dataset.attrs["axis_units"] = [units.get(a, "") for a in tensor.dims] 

786 scale = tensor.get_physical_scale() 

787 if scale: 787 ↛ 790line 787 didn't jump to line 790 because the condition on line 787 was always true

788 dataset.attrs["axis_scale"] = [scale.get(a, 1.0) for a in tensor.dims] 

789 

790 if all(d[0].lower() in "tczxy" for d in tensor.dims): 790 ↛ 806line 790 didn't jump to line 806 because the condition on line 790 was always true

791 # add vigra axis tags 

792 dataset.attrs["axistags"] = json.dumps( 

793 { 

794 "axes": [ 

795 { 

796 "key": (a := str(d[0]).lower()), 

797 "typeFlags": 1 if a == "c" else 2 if a in "zxy" else 8, 

798 "resolution": scale.get(d, 0), 

799 "description": units.get(d, ""), 

800 } 

801 for d in tensor.dims 

802 ] 

803 } 

804 ) 

805 

806 data = cast_and_assert(da.Array, tensor.as_dask().data.data) 

807 da.store(data, dataset, compute=True, lock=True) 

808 

809 

810def _save_tensor_bioio( 

811 path: Path | ZipPath | FtpUrl, 

812 tensor: Tensor, 

813 internal_path: PurePath | None, 

814 chunk_shape: PerAxis[int] | None = None, 

815 shard_shape: PerAxis[int] | None = None, 

816) -> None: 

817 """Save an image tensor using bioio""" 

818 

819 import bioio_base.types 

820 

821 logger.info( 

822 "Saving tensor to {}{} with bioio", 

823 path, 

824 f"/{internal_path}" if internal_path else "", 

825 ) 

826 bioio_axis_map = { 

827 "batch": "S", 

828 "channel": "C", 

829 "index": "S", 

830 "time": "T", 

831 } 

832 scale = tensor.get_physical_scale() 

833 

834 if path.suffix.lower() == ".zarr": 

835 if internal_path and str(internal_path) != "0": 

836 raise NotImplementedError( 

837 "Saving to an internal path that is not '0' is not implemented for zarr files with io_lib='bioio'." 

838 ) 

839 

840 import bioio_ome_zarr.writers 

841 

842 try: 

843 import zarr 

844 except ImportError: 

845 zarr_format = 3 

846 else: 

847 zarr_format = zarr.config.get("default_zarr_format") 

848 if zarr_format not in (2, 3): 848 ↛ 849line 848 didn't jump to line 849 because the condition on line 848 was never true

849 raise NotImplementedError( 

850 f"DEFAULT_ZARR_FORMAT={zarr_format} is not supported. (expected 2 or 3)" 

851 ) 

852 

853 axes_names = single_letter_dims_if_possible(tensor.dims) 

854 axes_types = [ 

855 {"time": "time", "t": "time", "channel": "channel", "c": "channel"}.get( 

856 str(a).lower(), "space" 

857 ) 

858 for a in tensor.dims 

859 ] 

860 

861 channel_names = tensor.channel_names 

862 channel_colors = tensor.channel_colors 

863 

864 channels = [ 

865 bioio_ome_zarr.writers.Channel( 

866 label=f"channel{i}" if channel_names is None else channel_names[i], 

867 color="#FF0000" if channel_colors is None else channel_colors[i], 

868 ) 

869 for i in range(tensor.tagged_shape.get(AxisId("channel"), 0)) 

870 ] 

871 

872 unit = tensor.get_physical_scale_unit() 

873 writer = bioio_ome_zarr.writers.OMEZarrWriter( 

874 store=path, 

875 level_shapes=[[tensor.tagged_shape[d] for d in tensor.dims]], 

876 dtype=tensor.dtype, 

877 zarr_format=zarr_format, 

878 image_name=tensor.name or "Image", 

879 axes_names=list(axes_names), 

880 axes_types=axes_types, 

881 axes_units=[unit.get(a) for a in tensor.dims], 

882 physical_pixel_size=[scale.get(a, 1.0) for a in tensor.dims], 

883 channels=channels, 

884 chunk_shape=[chunk_shape[a] for a in tensor.dims] 

885 if chunk_shape is not None 

886 else None, 

887 shard_shape=[shard_shape[a] for a in tensor.dims] 

888 if shard_shape is not None 

889 else None, 

890 ) 

891 logger.info("OME-Zarr metadata: {}", writer.preview_metadata()) 

892 writer.write_full_volume(tensor.data.data) 

893 return 

894 elif path.suffix.lower() in (".tif", ".tiff"): 

895 from bioio_ome_tiff.writers import OmeTiffWriter 

896 

897 save = OmeTiffWriter.save # pyright: ignore[reportUnknownVariableType] 

898 bioio_axis_letters = "TCZYX" 

899 elif path.suffix.lower() in (".gif", ".mp4", ".mkv"): 899 ↛ 900line 899 didn't jump to line 900 because the condition on line 899 was never true

900 from bioio_imageio.writers import TimeseriesWriter 

901 

902 tensor = tensor.squeeze() 

903 save = TimeseriesWriter.save # pyright: ignore[reportUnknownVariableType] 

904 bioio_axis_map["channel"] = "S" 

905 bioio_axis_letters = "YXS" 

906 elif path.suffix.lower() in ( 906 ↛ 927line 906 didn't jump to line 927 because the condition on line 906 was always true

907 ".png", 

908 ".bmp", 

909 ".jpg", 

910 ".mov", 

911 ".avi", 

912 ".mpg", 

913 ".mpeg", 

914 ".mp4", 

915 ".mkv", 

916 ".wmv", 

917 ".ogg", 

918 ): 

919 from bioio_imageio.writers import TwoDWriter 

920 

921 tensor = tensor.squeeze() 

922 save = TwoDWriter.save # pyright: ignore[reportUnknownVariableType] 

923 # channel dim denoted as 'S' 

924 bioio_axis_map["channel"] = "S" 

925 bioio_axis_letters = "YXS" 

926 else: 

927 raise RuntimeError( 

928 f"Failed to identify a suitable writer for {path} with suffix={path.suffix}." 

929 ) 

930 

931 bioio_data = tensor.data 

932 dim_map = { 

933 d: bioio_axis_map.get(str(d).lower(), str(d)[:1].upper()) for d in tensor.dims 

934 } 

935 

936 bioio_data = cast_and_assert(xr.DataArray, bioio_data.rename(dim_map)) 

937 

938 if not all(str(d) in bioio_axis_letters for d in bioio_data.dims): 938 ↛ 939line 938 didn't jump to line 939 because the condition on line 938 was never true

939 logger.warning( 

940 "Attempting to save tensor with dimensions {} not in '{}' using bioio.", 

941 bioio_data.dims, 

942 bioio_axis_letters, 

943 ) 

944 

945 dim_order = "".join(str(d) for d in bioio_data.dims) 

946 

947 image_name = None if internal_path is None else str(internal_path) 

948 physical_scale = tensor.get_physical_scale() 

949 _ = save( 

950 bioio_data.data, 

951 path, 

952 dim_order=dim_order, 

953 image_name=image_name, 

954 channel_names=tensor.channel_names, 

955 channel_colors=None 

956 if (channel_colors := tensor.channel_colors) is None 

957 else [list(hex_to_rgb(c)) for c in channel_colors], 

958 physical_pixel_sizes=bioio_base.types.PhysicalPixelSizes( 

959 physical_scale.get(AxisId("z")), 

960 physical_scale.get(AxisId("y")), 

961 physical_scale.get(AxisId("x")), 

962 ), 

963 ) 

964 

965 

966def _save_tensor_clearscale( 

967 path: Path | ZipPath | FtpUrl, 

968 tensor: Tensor, 

969 internal_path: PurePath | None, 

970 chunk_shape: PerAxis[int] | None = None, 

971 shard_shape: PerAxis[int] | None = None, 

972) -> None: 

973 import clearscale 

974 import dask.array 

975 import zarr 

976 

977 # we parallelize at a higher level, so here we write single-threaded. 

978 _ = zarr.config.set({"threading.max_workers": 1}) 

979 

980 single_letter_dims = single_letter_dims_if_possible(tensor.dims) 

981 

982 # prefer canonical order for standard volumes 

983 canonical_axes = "tczyx" 

984 if all(d in canonical_axes for d in single_letter_dims): 984 ↛ 1003line 984 didn't jump to line 1003 because the condition on line 984 was always true

985 canonical_single_letter_dims = tuple( 

986 a for a in canonical_axes if a in single_letter_dims 

987 ) 

988 canonical_order = tuple( 

989 tensor.dims[single_letter_dims.index(a)] 

990 for a in canonical_single_letter_dims 

991 ) 

992 if tensor.dims != canonical_order: 992 ↛ 993line 992 didn't jump to line 993 because the condition on line 992 was never true

993 logger.debug( 

994 "Transposing tensor from {} to canonical order {} for saving with clearscale+zarr-python", 

995 tensor.dims, 

996 canonical_order, 

997 ) 

998 tensor = tensor.transpose( 

999 canonical_order, missing_dims="raise", extra_dims="raise" 

1000 ) 

1001 single_letter_dims = canonical_single_letter_dims 

1002 

1003 tensor_shape_tuple = tuple(tensor.tagged_shape[d] for d in tensor.dims) 

1004 

1005 zarr_format = zarr.config.get("default_zarr_format") 

1006 scale_key = "s0" if internal_path is None else str(internal_path) 

1007 logger.info( 

1008 "Saving tensor {} to '{}' of {} with clearscale+zarr-python using zarr_format={} and dims={}", 

1009 tensor.tagged_shape, 

1010 scale_key, 

1011 path, 

1012 zarr_format, 

1013 single_letter_dims, 

1014 ) 

1015 

1016 dim_map = dict(zip(tensor.dims, single_letter_dims)) 

1017 multiscale: Any = clearscale.Multiscale( 

1018 { 

1019 scale_key: clearscale.Scale( 

1020 shape=clearscale.Shape( 

1021 **dict(zip(dim_map.values(), tensor_shape_tuple)) 

1022 ), 

1023 pixel_size={ 

1024 dim_map[k]: v for k, v in tensor.get_physical_scale().items() 

1025 } 

1026 or None, 

1027 unit={ 

1028 dim_map[k]: v for k, v in tensor.get_physical_scale_unit().items() 

1029 } 

1030 or None, 

1031 ) 

1032 } 

1033 ) 

1034 if zarr_format == 2: 

1035 zarr_attrs: dict[str, JsonValueReadOnly] = { 

1036 "multiscales": [ 

1037 multiscale.to_ome_zarr(version="0.4") 

1038 ], # TODO: add axis semantics 

1039 } 

1040 elif zarr_format == 3: 1040 ↛ 1050line 1040 didn't jump to line 1050 because the condition on line 1040 was always true

1041 zarr_attrs: dict[str, JsonValueReadOnly] = { 

1042 "ome": { 

1043 "version": "0.5", 

1044 "multiscales": [ 

1045 multiscale.to_ome_zarr(version="0.5") 

1046 ], # TODO: add axis semantics 

1047 } 

1048 } 

1049 else: 

1050 raise NotImplementedError(f"Zarr format {zarr_format} is not supported.") 

1051 

1052 zarr_group = zarr.open_group( 

1053 str(path), 

1054 mode="w", 

1055 zarr_format=zarr_format, 

1056 attributes=zarr_attrs, 

1057 ) 

1058 

1059 create_array_kwargs = _CreateArrayKwargs( 

1060 name=scale_key, 

1061 overwrite=True, 

1062 dimension_names=None if zarr_format == 2 else tuple(dim_map.values()), 

1063 chunks="auto" 

1064 if chunk_shape is None 

1065 else tuple(chunk_shape[d] for d in tensor.dims), 

1066 shards=None 

1067 if zarr_format == 2 

1068 else "auto" 

1069 if shard_shape is None 

1070 else tuple(shard_shape[d] for d in tensor.dims), 

1071 ) 

1072 zarr_array = zarr_group.require_array( 

1073 **create_array_kwargs, 

1074 shape=[tensor.tagged_shape[d] for d in tensor.dims], 

1075 dtype=tensor.dtype, 

1076 ) 

1077 xr_array = tensor.data 

1078 if dm := {k: v for k, v in dim_map.items() if k != v}: 

1079 xr_array = cast_and_assert(xr.DataArray, xr_array.rename(dm)) 

1080 

1081 if isinstance(xr_array.data, np.ndarray): 

1082 zarr_array[:] = xr_array.data 

1083 elif isinstance(xr_array.data, dask.array.Array): 

1084 dask.array.to_zarr(xr_array.data, zarr_array, compute=True) 

1085 else: 

1086 assert_never(xr_array.data) 

1087 

1088 

1089def _save_tensor_imageio( 

1090 path: Path | ZipPath | FtpUrl, tensor: Tensor, internal_path: PurePath | None 

1091) -> None: 

1092 if internal_path is not None: 1092 ↛ 1093line 1092 didn't jump to line 1093 because the condition on line 1092 was never true

1093 raise NotImplementedError( 

1094 "Saving tensors to subdirectories is not implemented for imageio. Please save to a file path without internal path." 

1095 ) 

1096 

1097 if isinstance(path, ZipPath): 1097 ↛ 1098line 1097 didn't jump to line 1098 because the condition on line 1097 was never true

1098 raise NotImplementedError( 

1099 "Saving tensors to zip files is not implemented for imageio. Please save to an unzipped file path." 

1100 ) 

1101 

1102 logger.debug("Saving tensor to {} with imageio", path) 

1103 if not isinstance(path, FtpUrl): 1103 ↛ 1106line 1103 didn't jump to line 1106 because the condition on line 1103 was always true

1104 path.parent.mkdir(exist_ok=True, parents=True) 

1105 

1106 extension = path.suffix.lower() 

1107 removed_singleton_axes: list[AxisId] = [] 

1108 remove_singletons = { 

1109 AxisId("batch"): [ 

1110 ".tif", 

1111 ".tiff", 

1112 ], # remove singleton batch dim for tiff files 

1113 **{ 

1114 a: [".png", ".jpg", ".jpeg"] for a in tensor.dims 

1115 }, # remove any singleton axis for png and jpg files 

1116 } 

1117 for rm_a, rm_ext in remove_singletons.items(): 

1118 if extension in rm_ext and tensor.tagged_shape.get(rm_a) == 1: 

1119 tensor = tensor[{rm_a: 0}] 

1120 removed_singleton_axes.append(rm_a) 

1121 

1122 if removed_singleton_axes: 

1123 singleton_axes_msg = ( 

1124 f"(with removed singleton axes {list(map(str, removed_singleton_axes))}) " 

1125 ) 

1126 else: 

1127 singleton_axes_msg = "" 

1128 

1129 logger.info( 

1130 "writing tensor {} {}to {}", 

1131 dict(tensor.tagged_shape), 

1132 singleton_axes_msg, 

1133 path, 

1134 ) 

1135 if extension in (".png", ".jpg", ".jpeg") and tensor.dtype in ( 1135 ↛ 1139line 1135 didn't jump to line 1139 because the condition on line 1135 was never true

1136 "float32", 

1137 "float64", 

1138 ): 

1139 logger.warning( 

1140 "converting tensor of dtype {} to uint8 for saving as {}", 

1141 tensor.dtype, 

1142 extension, 

1143 ) 

1144 tensor = ( 

1145 (tensor - (t_min := cast(float, tensor.data.min()))) 

1146 / xr.ufuncs.maximum(cast(float, tensor.data.max()) - t_min, 1e-8) 

1147 * 255 

1148 ).astype("uint8") 

1149 

1150 imwrite(path, tensor, extension=extension) 

1151 

1152 

1153def _save_tensor_numpy(path: Path | ZipPath | FtpUrl, tensor: Tensor) -> None: 

1154 if isinstance(path, FtpUrl): 1154 ↛ 1155line 1154 didn't jump to line 1155 because the condition on line 1154 was never true

1155 raise NotImplementedError( 

1156 "Saving numpy arrays to FTP URLs is not implemented. Please save to a local file path instead." 

1157 ) 

1158 

1159 logger.info("Saving tensor to {} with numpy", path) 

1160 

1161 if isinstance(path, ZipPath): 1161 ↛ 1162line 1161 didn't jump to line 1162 because the condition on line 1161 was never true

1162 folder = path.filename.parent 

1163 else: 

1164 folder = path.parent 

1165 

1166 folder.mkdir(exist_ok=True, parents=True) 

1167 

1168 save_array(path, tensor.to_numpy()) 

1169 

1170 

1171def _save_tensor_tifffile( 

1172 path: Path | ZipPath | FtpUrl, extensions: list[str] | None, tensor: Tensor 

1173) -> None: 

1174 if isinstance(path, FtpUrl): 1174 ↛ 1175line 1174 didn't jump to line 1175 because the condition on line 1174 was never true

1175 raise NotImplementedError( 

1176 "Saving tiff files to FTP URLs is not implemented. Please save to a local file path instead." 

1177 ) 

1178 

1179 import inspect 

1180 

1181 import tifffile 

1182 

1183 tifffile_imwrite_signature = inspect.signature( 

1184 cast(Callable[..., Any], tifffile.imwrite) 

1185 ) 

1186 if "kind" not in tifffile_imwrite_signature.parameters: 

1187 raise NotImplementedError( 

1188 "Saving tensors with tifffile uses tifffile.imwrite(...kind=...) (added in tifffile 2026.5.2)." 

1189 ) 

1190 

1191 logger.trace("tensor dims {}", tensor.dims) 

1192 tensor = tensor.transpose("TZCYX") # tifffile expects TZCYX order 

1193 logger.trace("tensor dims transposed {}", tensor.dims) 

1194 if len(tensor.dims) > 5: 1194 ↛ 1195line 1194 didn't jump to line 1195 because the condition on line 1194 was never true

1195 logger.warning( 

1196 "Saving tensor with more than 5 dimensions ({}) will likely fail." 

1197 ) 

1198 

1199 logger.info("Saving tensor to {} with tifffile", path) 

1200 

1201 scale = tensor.get_physical_scale() 

1202 x_scale = scale.get(AxisId("x")) 

1203 y_scale = scale.get(AxisId("y")) 

1204 z_scale = scale.get(AxisId("z")) 

1205 t_scale = scale.get(AxisId("time")) 

1206 if x_scale is not None and y_scale is not None: 1206 ↛ 1209line 1206 didn't jump to line 1209 because the condition on line 1206 was always true

1207 resolution = (1 / x_scale, 1 / y_scale) 

1208 else: 

1209 resolution = None 

1210 

1211 scale_unit = tensor.get_physical_scale_unit() 

1212 x_unit = scale_unit.get(AxisId("x")) 

1213 y_unit = scale_unit.get(AxisId("y")) 

1214 z_unit = scale_unit.get(AxisId("z")) 

1215 t_unit = scale_unit.get(AxisId("time")) 

1216 if x_unit == "inch": 1216 ↛ 1217line 1216 didn't jump to line 1217 because the condition on line 1216 was never true

1217 resolution_unit = tifffile.RESUNIT.INCH 

1218 elif x_unit in ("centimeter", "cm"): 1218 ↛ 1219line 1218 didn't jump to line 1219 because the condition on line 1218 was never true

1219 resolution_unit = tifffile.RESUNIT.CENTIMETER 

1220 elif x_unit in ("millimeter", "mm"): 1220 ↛ 1221line 1220 didn't jump to line 1221 because the condition on line 1220 was never true

1221 resolution_unit = tifffile.RESUNIT.MILLIMETER 

1222 elif x_unit in ("micrometer", "um", "µm"): 

1223 resolution_unit = tifffile.RESUNIT.MICROMETER 

1224 else: 

1225 resolution_unit = None 

1226 

1227 if isinstance(path, ZipPath): 1227 ↛ 1228line 1227 didn't jump to line 1228 because the condition on line 1227 was never true

1228 folder = path.filename.parent 

1229 else: 

1230 folder = path.parent 

1231 

1232 folder.mkdir(exist_ok=True, parents=True) 

1233 

1234 if extensions is not None and extensions[-2:] in ( 1234 ↛ 1238line 1234 didn't jump to line 1238 because the condition on line 1234 was never true

1235 [".ome", ".tif"], 

1236 [".ome", ".tiff"], 

1237 ): 

1238 metadata = OmeMetadata( 

1239 # "DimensionOrder": "TZCYX", # [::-1]?? # TODO: check if this is needed 

1240 TimeIncrement=t_scale, 

1241 TimeIncrementUnit=t_unit, 

1242 PhysicalSizeX=x_scale, 

1243 PhysicalSizeXUnit=x_unit, 

1244 PhysicalSizeY=y_scale, 

1245 PhysicalSizeYUnit=y_unit, 

1246 PhysicalSizeZ=z_scale, 

1247 PhysicalSizeZUnit=z_unit, 

1248 Channel=None 

1249 if tensor.channel_names is None 

1250 else OmeChannelMetadata(Name=tensor.channel_names), 

1251 ) 

1252 kind = "ome" 

1253 else: 

1254 if not t_unit or t_scale is None: 1254 ↛ 1256line 1254 didn't jump to line 1256 because the condition on line 1254 was always true

1255 pass 

1256 elif isinstance(t_scale, (int, float)): 

1257 if t_unit in ("nanosecond", "nanosecond(s)", "ns"): 

1258 t_scale /= 1_000_000_000 

1259 elif t_unit in ("microsecond", "microsecond(s)", "us", "µsec", "µs"): 

1260 t_scale /= 1_000_000 

1261 elif t_unit in ("millisecond", "millisecond(s)", "ms"): 

1262 t_scale /= 1_000 

1263 elif t_unit in ("minute", "minute(s)", "min"): 

1264 t_scale *= 60 

1265 elif t_unit in ("hour", "hour(s)", "h"): 

1266 t_scale *= 3_600 

1267 elif t_unit in ("day", "day(s)", "d"): 

1268 t_scale *= 86_400 

1269 elif t_unit in ("week", "week(s)", "wk"): 

1270 t_scale *= 604_800 

1271 elif t_unit in ("month", "month(s)", "mo"): 

1272 t_scale *= 2_629_746 # average month length in seconds 

1273 elif t_unit in ("year", "year(s)", "yr"): 

1274 t_scale *= 31_556_952 # average year length in seconds 

1275 elif t_unit not in ("second", "s"): 

1276 logger.warning( 

1277 "Unknown time unit '{}' encountered while saving an imagej tiff with tifffile", 

1278 t_unit, 

1279 ) 

1280 else: 

1281 logger.warning( 

1282 "Failed to convert time interval '{}' of type {} (with time unit '{}') to seconds while saving an imagej tiff with tifffile", 

1283 t_scale, 

1284 type(t_scale), 

1285 t_unit, 

1286 ) 

1287 

1288 metadata = ImageJMetadata( 

1289 unit=x_unit, 

1290 yunit=y_unit, 

1291 zunit=z_unit, 

1292 spacing=z_scale, 

1293 finterval=t_scale, 

1294 fps=None if not isinstance(t_scale, (int, float)) else 1 / t_scale, 

1295 ) 

1296 kind = "imagej" 

1297 

1298 if isinstance(path, ZipPath): 1298 ↛ 1299line 1298 didn't jump to line 1299 because the condition on line 1298 was never true

1299 ctxt = path.open("wb") 

1300 else: 

1301 ctxt = nullcontext(path) 

1302 

1303 with ctxt as f: 

1304 assert not isinstance(f, TextIOWrapper) 

1305 _ = tifffile.imwrite( 

1306 f, 

1307 tensor.to_numpy(), 

1308 kind=kind, 

1309 software="bioimageio.core", 

1310 resolution=resolution, 

1311 metadata={ 

1312 k: v 

1313 for k, v in metadata.model_dump(mode="json", exclude_none=True).items() 

1314 }, 

1315 resolutionunit=resolution_unit, 

1316 ) 

1317 

1318 

1319class _CreateArrayKwargs(TypedDict): 

1320 name: str 

1321 overwrite: bool 

1322 dimension_names: tuple[str, ...] | None 

1323 shards: Literal["auto"] | tuple[int, ...] | None 

1324 chunks: Literal["auto"] | tuple[int, ...] 

1325 

1326 

1327_zarr_url_adapter: TypeAdapter[ZarrUrl] = TypeAdapter(ZarrUrl) 

1328 

1329 

1330def _interprete_tensor_target( 

1331 source: PermissiveFileSource | ZarrSource | ZipPath | FileDescr, 

1332): 

1333 if isinstance(source, FileDescr): 

1334 source = source.source 

1335 

1336 if isinstance(source, (str, pydantic.AnyUrl)): 

1337 try: 

1338 source = _zarr_url_adapter.validate_python(str(source)) 

1339 except Exception: 

1340 if ( 

1341 str(source).startswith("http://") 

1342 or str(source).startswith("https://") 

1343 or str(source).startswith("ftp://") 

1344 ): 

1345 raise 

1346 else: 

1347 source = Path(str(source)) 

1348 

1349 if isinstance(source, (RelativeFilePath, RelativeDirectory)): 

1350 source = source.absolute() 

1351 

1352 return _get_file_source_extension_and_internal_path(source) 

1353 

1354 

1355def _interprete_tensor_source( 

1356 source: PermissiveFileSource | ZarrSource | ZipPath | FileDescr, 

1357): 

1358 

1359 if isinstance(source, FileDescr): 1359 ↛ 1360line 1359 didn't jump to line 1360 because the condition on line 1359 was never true

1360 source = source.source 

1361 

1362 if isinstance(source, (str, pydantic.AnyUrl)): 

1363 source = interprete_file_source(source, allow_zarr=True) 

1364 

1365 if isinstance(source, (RelativeFilePath, RelativeDirectory)): 1365 ↛ 1366line 1365 didn't jump to line 1366 because the condition on line 1365 was never true

1366 source = source.absolute() 

1367 

1368 return _get_file_source_extension_and_internal_path(source) 

1369 

1370 

1371def _get_file_source_extension_and_internal_path( 

1372 source: Path | ZipPath | RootHttpUrl | FtpUrl, 

1373): 

1374 internal_path = None 

1375 if isinstance(source, (HttpUrl, RootHttpUrl, FtpUrl)): 1375 ↛ 1376line 1375 didn't jump to line 1376 because the condition on line 1375 was never true

1376 original_source_path = PosixPath(source.path or "") 

1377 parents: Iterable[FtpUrl | RootHttpUrl | Path | ZipPath] = source.parents 

1378 elif isinstance(source, ZipPath): 1378 ↛ 1379line 1378 didn't jump to line 1379 because the condition on line 1378 was never true

1379 original_source_path = PosixPath(source.filename) 

1380 parents = [] 

1381 p = source 

1382 while p != p.parent: 

1383 parents.append(p.parent) 

1384 p = p.parent 

1385 else: 

1386 original_source_path = PosixPath(source) 

1387 parents = source.parents 

1388 

1389 file_source = source 

1390 for parent in chain([source], parents): 1390 ↛ 1407line 1390 didn't jump to line 1407 because the loop on line 1390 didn't complete

1391 extensions = [s.lower() for s in parent.suffixes] 

1392 if extensions: 

1393 file_source = parent 

1394 if isinstance(parent, (RootHttpUrl, FtpUrl)): 1394 ↛ 1395line 1394 didn't jump to line 1395 because the condition on line 1394 was never true

1395 parent_path = PosixPath(parent.path or "") 

1396 elif isinstance(parent, ZipPath): 1396 ↛ 1397line 1396 didn't jump to line 1397 because the condition on line 1396 was never true

1397 parent_path = PosixPath(parent.filename) 

1398 else: 

1399 parent_path = PosixPath(parent) 

1400 

1401 internal_path = original_source_path.relative_to(parent_path) 

1402 if internal_path == PosixPath("."): 

1403 internal_path = None 

1404 

1405 break 

1406 else: 

1407 extensions = None 

1408 

1409 return file_source, extensions, internal_path 

1410 

1411 

1412def save_sample( 

1413 path: Path | str | PerMember[Path | str], 

1414 sample: Sample, 

1415 *, 

1416 chunk_shape: PerMember[PerAxis[int]] | None = None, 

1417 shard_shape: PerMember[PerAxis[int]] | None = None, 

1418) -> PerMember[Path]: 

1419 """Save a **sample** to a **path** pattern 

1420 or all sample members in the **path** mapping. 

1421 

1422 Args: 

1423 path: A path pattern or a mapping of member_id to path pattern. 

1424 If **path** is a pathlib.Path or a string and the **sample** has multiple members, 

1425 **path** it must contain `{member_id}` (or `{input_id}` or `{output_id}`). 

1426 sample: Sample to save 

1427 chunk_shape: Optional (re-)chunking of the sample data for applicable image formats (zarr). 

1428 shard_shape: Optional sharding of the sample data for applicable image formats (zarr). 

1429 

1430 (Each) **path** may contain `{sample_id}` to be formatted with the **sample** object. 

1431 """ 

1432 

1433 if not isinstance(path, Mapping): 

1434 if len(sample.members) < 2 or any( 1434 ↛ 1439line 1434 didn't jump to line 1439 because the condition on line 1434 was always true

1435 m in str(path) for m in ("{member_id}", "{input_id}", "{output_id}") 

1436 ): 

1437 path = {m: path for m in sample.members} 

1438 else: 

1439 raise ValueError( 

1440 f"path {path} must contain '{{member_id}}' for sample with multiple members {list(sample.members)}." 

1441 ) 

1442 

1443 output_paths = { 

1444 m: Path( 

1445 str(p).format(sample_id=sample.id, member_id=m, input_id=m, output_id=m) 

1446 ) 

1447 for m, p in path.items() 

1448 } 

1449 with ThreadPoolExecutor(max_workers=min(len(sample.members), 4)) as executor: 

1450 for m, p in output_paths.items(): 

1451 _ = executor.submit( 

1452 save_tensor, 

1453 p, 

1454 sample.members[m], 

1455 chunk_shape=chunk_shape[m] if chunk_shape is not None else None, 

1456 shard_shape=shard_shape[m] if shard_shape is not None else None, 

1457 ) 

1458 

1459 return output_paths 

1460 

1461 

1462class _StatEntry(BaseModel, frozen=True, arbitrary_types_allowed=True): 

1463 """Serializable stat entry""" 

1464 

1465 measure: DatasetMeasure | SampleMeasure 

1466 value: MeasureValue 

1467 

1468 

1469class _StatList(RootModel[list[_StatEntry]]): 

1470 """Serializable stat mapping""" 

1471 

1472 

1473def serialize_stat( 

1474 stat: Mapping[DatasetMeasure | SampleMeasure, MeasureValue], 

1475) -> list[JsonValue]: 

1476 """Serialize a stat mapping to a JSON string""" 

1477 stat_list = _StatList([_StatEntry(measure=k, value=v) for k, v in stat.items()]) 

1478 return stat_list.model_dump(mode="json") 

1479 

1480 

1481def save_stat( 

1482 stat: Mapping[DatasetMeasure | SampleMeasure, MeasureValue], 

1483 output: Path | BytesIO, 

1484) -> None: 

1485 """Save sample and dataset statistics as a JSON file""" 

1486 

1487 if isinstance(output, Path): 1487 ↛ 1490line 1487 didn't jump to line 1490 because the condition on line 1487 was always true

1488 ctxt = output.open("wb") 

1489 else: 

1490 ctxt = nullcontext(output) 

1491 

1492 with ctxt as out: 

1493 _ = out.write(json.dumps(serialize_stat(stat), indent=2).encode("utf-8")) 

1494 

1495 

1496def load_stat(source: Path | str | Sequence[JsonValue]) -> Stat: 

1497 """Load sample and dataset statistics from JSON""" 

1498 if isinstance(source, Path): 

1499 source = source.read_text(encoding="utf-8") 

1500 

1501 if isinstance(source, str): 

1502 seq = _StatList.model_validate_json(source) 

1503 else: 

1504 seq = _StatList.model_validate(source) 

1505 

1506 return {e.measure: e.value for e in seq.root} 

1507 

1508 

1509def save_dataset_stat(stat: Mapping[DatasetMeasure, MeasureValue], path: Path) -> None: 

1510 """DEPRECATED alias for save_stat(): use `save_stats()` instead.""" 

1511 warnings.warn("`save_dataset_stat()` is deprecated, use `save_stats()` instead.") 

1512 save_stat({k: v for k, v in stat.items()}, path) 

1513 

1514 

1515def load_dataset_stat(path: Path) -> Stat: 

1516 """DEPRECATED alias for `load_stat()`: use `load_stat()` instead.""" 

1517 warnings.warn("`load_dataset_stat()` is deprecated, use `load_stats()` instead.") 

1518 return load_stat(path) 

1519 

1520 

1521def ensure_unzipped(source: PermissiveFileSource | ZipPath | BytesReader, folder: Path): 

1522 """unzip a (downloaded) **source** to a file in **folder** if source is a zip archive 

1523 otherwise copy **source** to a file in **folder**.""" 

1524 if isinstance(source, BytesReader): 1524 ↛ 1525line 1524 didn't jump to line 1525 because the condition on line 1524 was never true

1525 weights_reader = source 

1526 else: 

1527 weights_reader = get_reader(source) 

1528 

1529 out_path = folder / ( 

1530 weights_reader.original_file_name or f"file{weights_reader.suffix}" 

1531 ) 

1532 

1533 if zipfile.is_zipfile(weights_reader): 1533 ↛ 1541line 1533 didn't jump to line 1541 because the condition on line 1533 was always true

1534 out_path = out_path.with_name(out_path.name + ".unzipped") 

1535 out_path.parent.mkdir(exist_ok=True, parents=True) 

1536 # source itself is a zipfile 

1537 with zipfile.ZipFile(weights_reader, "r") as f: 

1538 f.extractall(out_path) 

1539 

1540 else: 

1541 out_path.parent.mkdir(exist_ok=True, parents=True) 

1542 with out_path.open("wb") as f: 

1543 copyfileobj(weights_reader, f) 

1544 

1545 return out_path 

1546 

1547 

1548def get_suffix(source: ZipPath | FileSource) -> Suffix: 

1549 """DEPRECATED: use source.suffix instead.""" 

1550 return source.suffix 

1551 

1552 

1553# def _determine_chunk_and_shard_shapes( 

1554# shape: PerAxis[int], 

1555# block_shape: PerAxis[int], 

1556# item_size: int, 

1557# ) -> tuple[PerAxis[int], PerAxis[int]]: 

1558# def increase_shard_size(shard_size: int, shape_size: int, chunk_size: int) -> int: 

1559# """increase shard size by chunk_size (or a multiple of chunk_size such that the last two chunks fall into the same shard). 

1560# Will not increase shard size beyond shape_size. 

1561# """ 

1562# while ( 

1563# shard_size < shape_size 

1564# and shape_size % shard_size 

1565# != 0 # last shard is smaller than the shard size 

1566# and shape_size % shard_size 

1567# < 2 * chunk_size # last shard is smaller than 2*inner_size, 

1568# # but the last two blocks may overlap and must land in the same shard 

1569# ): 

1570# shard_size = min(shard_size + chunk_size, shape_size) 

1571 

1572# return shard_size 

1573 

1574# shard_shape: dict[AxisId, int] | None = {} 

1575# # create smallest possible shard shape 

1576# for a, shape_size in shape.items(): 

1577# # The smallest possible shard shape is made up of the inner sizes (that will make up the chunks) 

1578# block_size = block_shape[a] 

1579# shard_shape[a] = increase_shard_size(block_size, shape_size, block_size) 

1580 

1581# shard_byte_size = prod(shard_shape.values()) * item_size 

1582# if ( 

1583# shard_byte_size < SHARDED_INNER_CHUNK_MAX_BYTES // 2 

1584# and BATCH_AXIS_ID in shape 

1585# and unstacked_batch_shape is not None 

1586# ): 

1587# # increase shard shape along a stacked multi-index batch dimension if present (to optimize writing) 

1588# shard_shape[BATCH_AXIS_ID] = max( 

1589# shard_shape[BATCH_AXIS_ID], prod(unstacked_batch_shape.values()) 

1590# ) 

1591 

1592# shard_byte_size = prod(shard_shape.values()) * item_size 

1593# increase_order = [ 

1594# a 

1595# for a in ( 

1596# CHANNEL_AXIS_ID, 

1597# AxisId("x"), 

1598# AxisId("y"), 

1599# AxisId("z"), 

1600# AxisId("t"), 

1601# AxisId("i"), 

1602# ) 

1603# if a in shape 

1604# ] 

1605# increase_order += [a for a in shape if a not in increase_order] 

1606# while ( 

1607# any(shard_shape[a] < s for a, s in shape.items()) 

1608# and shard_byte_size < SHARDED_INNER_CHUNK_MAX_BYTES // 2 

1609# ): 

1610# for a in increase_order: 

1611# shard_shape[a] = increase_shard_size( 

1612# shard_shape[a], shape[a], inner_sizes[a] 

1613# ) 

1614# shard_byte_size = prod(shard_shape.values()) * item_size 

1615# if shard_byte_size >= SHARDED_INNER_CHUNK_MAX_BYTES // 2: 

1616# break