Coverage for src/bioimageio/core/io.py: 72%
692 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 09:46 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 09:46 +0000
1from __future__ import annotations
3import atexit
4import json
5import os
6import warnings
7import zipfile
8from collections.abc import Callable, Iterable, Mapping, Sequence
9from concurrent.futures import ThreadPoolExecutor
10from contextlib import nullcontext
11from io import BytesIO, TextIOWrapper
12from itertools import chain
13from pathlib import Path, PosixPath, PurePath, PurePosixPath
14from shutil import copyfileobj
15from typing import TYPE_CHECKING, Any, Literal, TypeAlias, TypedDict, cast
17import dask.array as da
18import dask.diagnostics
19import numpy as np
20import pydantic
21import xarray as xr
22from exceptiongroup import ExceptionGroup
23from imageio.v3 import (
24 imread, # pyright: ignore[reportUnknownVariableType]
25 imwrite, # type: ignore
26)
27from loguru import logger
28from pydantic import BaseModel, RootModel, TypeAdapter
29from typing_extensions import TypeAliasType as _TypeAliasType
30from typing_extensions import assert_never
32from bioimageio.spec._internal.io import (
33 RelativeDirectory,
34 ZarrSource,
35 ZarrUrl,
36 get_reader,
37 interprete_file_source,
38)
39from bioimageio.spec.common import (
40 BytesReader,
41 FileDescr,
42 FileSource,
43 FtpUrl,
44 HttpUrl,
45 PermissiveFileSource,
46 RelativeFilePath,
47 RootHttpUrl,
48 ZipPath,
49)
50from bioimageio.spec.utils import load_array, save_array
52from ._utils.color import hex_to_rgb
53from ._utils.io_zarr import open_zarr_multiscale_array
54from ._utils.tiff_metadata import (
55 ImageJMetadata,
56 OmeChannelMetadata,
57 OmeMetadata,
58 create_ome_metadata_from_xml_string,
59)
60from ._utils.type_guards import is_sequence, is_tuple
61from ._utils.type_hints import cast_and_assert
62from .axis import (
63 BATCH_AXIS_ID,
64 AxisId,
65 AxisLike,
66 PerAxis,
67 single_letter_dims_if_possible,
68)
69from .common import PerMember
70from .sample import Sample
71from .stat_measures import DatasetMeasure, MeasureValue, SampleMeasure, Stat
72from .tensor import Tensor
74if TYPE_CHECKING:
75 JsonValue: TypeAlias = (
76 bool | int | float | str | None | list["JsonValue"] | dict[str, "JsonValue"]
77 ) # note: order relevant for deserializing
79else:
80 # for pydantic validation we need to use `TypeAliasType`,
81 # see https://docs.pydantic.dev/latest/concepts/types/#named-recursive-types
82 # however this results in a partially unknown type with the current pyright 1.1.388
83 JsonValue: TypeAlias = _TypeAliasType(
84 "JsonValue",
85 bool | int | float | str | None | list["JsonValue"] | dict[str, "JsonValue"],
86 )
88JsonValueReadOnly: TypeAlias = (
89 bool
90 | int
91 | float
92 | str
93 | None
94 | Sequence["JsonValueReadOnly"]
95 | Mapping[str, "JsonValueReadOnly"]
96)
98IO_Lib: TypeAlias = Literal[
99 "bioio", "imageio", "clearscale", "numpy", "tifffile", "h5py"
100]
103def load_tensor(
104 source: FileDescr | PermissiveFileSource | ZipPath,
105 /,
106 axes: Sequence[AxisLike] | None = None,
107 io_lib: IO_Lib | None = None,
108) -> Tensor:
109 """Load an image tensor
111 Args:
112 source: The source of the image data.
113 axes: The (desired) axes of the image data.
114 io_lib: library to use for loading the image.
115 Defaults to trying various libraries depending on file extension.
117 Returns:
118 The loaded tensor.
119 """
120 tensor, _unstacked_dim_order = load_tensor_and_return_unstacked_dim_order(
121 source, axes=axes, io_lib=io_lib
122 )
123 return tensor
126def load_tensor_and_return_unstacked_dim_order(
127 source: FileDescr | PermissiveFileSource | ZipPath,
128 /,
129 axes: Sequence[AxisLike] | None = None,
130 io_lib: IO_Lib | None = None,
131) -> tuple[Tensor, Sequence[AxisId] | None]:
132 """Load an image tensor
134 Args:
135 source: The source of the image data.
136 axes: The (desired) axes of the image data.
137 io_lib: library to use for loading the image.
138 Defaults to trying various libraries depending on file extension.
140 Returns:
141 A tuple of the loaded tensor and the unstacked dimension order if the initially loaded
142 tensor had extra dimensions that were stacked to a multi-index batch dimension.
143 """
144 file_source, extensions, internal_path = _interprete_tensor_source(source)
145 if io_lib is None:
146 try_io_libs = _select_io_libs_based_on_extension(extensions)
148 exceptions: list[Exception] = []
149 for auto_io_lib in try_io_libs: 149 ↛ 160line 149 didn't jump to line 160 because the loop on line 149 didn't complete
150 try:
151 return load_tensor_and_return_unstacked_dim_order(
152 source, axes=axes, io_lib=auto_io_lib
153 )
154 except Exception as e:
155 exceptions.append(e)
156 logger.opt(exception=e).warning(
157 "Failed to load tensor with io_lib={}", auto_io_lib
158 )
160 raise ExceptionGroup(
161 "Failed to load tensor with any available io_lib", exceptions
162 )
164 elif io_lib == "bioio":
165 t = _load_tensor_bioio(
166 file_source, None if internal_path is None else str(internal_path)
167 )
168 elif io_lib == "clearscale":
169 t = _load_tensor_clearscale(source)
170 elif io_lib == "imageio":
171 reader = get_reader(source)
172 array = imread(
173 reader.read(),
174 extension=extensions[-1] if extensions else None, # pyright: ignore[reportArgumentType]
175 )
176 try:
177 t = Tensor.from_array(array, dims=axes)
178 except Exception:
179 if axes is None:
180 raise
182 t = Tensor.from_array(array, dims=None)
183 else:
184 axes = None
185 elif io_lib == "numpy":
186 array = load_array(source)
187 try:
188 t = Tensor.from_array(array, dims=axes)
189 except Exception:
190 if axes is None:
191 raise
193 t = Tensor.from_array(array, dims=None)
194 else:
195 axes = None
197 elif io_lib == "tifffile":
198 t = _load_tensor_tifffile(source)
199 elif io_lib == "h5py":
200 t = _load_tensor_h5py(file_source, internal_path, axes=axes)
201 else:
202 assert_never(io_lib)
204 if axes is None:
205 unstacked_dim_order = None
206 else:
207 t, unstacked_dim_order = t.transpose_and_return_unstacked_dim_order(
208 axes, extra_dims="squeeze_or_stack", missing_dims="unstack_or_expand"
209 )
211 logger.info("Loaded tensor {} with {}", t.tagged_shape, io_lib)
212 return t, unstacked_dim_order
215def _select_io_libs_based_on_extension(
216 extensions: list[str] | None,
217) -> tuple[IO_Lib, ...]:
218 if not extensions: 218 ↛ 219line 218 didn't jump to line 219 because the condition on line 218 was never true
219 return ("clearscale", "bioio", "imageio")
221 if extensions[-2:] in ([".ome", ".tif"], [".ome", ".tiff"]): 221 ↛ 222line 221 didn't jump to line 222 because the condition on line 221 was never true
222 return ("tifffile", "bioio")
223 elif extensions[-1] == ".npy":
224 return ("numpy",)
225 elif extensions[-1] == ".zarr":
226 return ("clearscale", "bioio")
227 elif extensions[-1] in (".tif", ".tiff"):
228 return ("tifffile", "imageio")
229 elif extensions[-1] in (".h4", ".h5", ".hdf", ".hdf4", ".hdf5", ".he5"):
230 return ("h5py", "bioio")
231 else:
232 return ("bioio", "imageio")
235def _load_tensor_h5py(
236 source: ZarrSource | FileSource | ZipPath,
237 internal_path: PurePath | None,
238 *,
239 axes: Sequence[AxisLike] | None = None,
240) -> Tensor:
241 import h5py # pyright: ignore[reportMissingTypeStubs]
243 if isinstance(source, RelativeDirectory): 243 ↛ 244line 243 didn't jump to line 244 because the condition on line 243 was never true
244 source = source.absolute()
246 file_source = (
247 source if isinstance(source, Path) else BytesIO(get_reader(source).read())
248 )
249 f = h5py.File(file_source, "r")
250 # we need to keep the file open for lazy loading, so we close it on exit
251 _ = atexit.register(f.close)
252 if internal_path is not None:
253 key = internal_path.as_posix()
254 elif len(f.keys()) == 0:
255 raise ValueError(f"HDF5 file {source} is empty")
256 else:
257 if "data" in f: 257 ↛ 260line 257 didn't jump to line 260 because the condition on line 257 was always true
258 key = "data"
259 else:
260 key = str(min(f.keys())) # pyright: ignore[reportUnknownArgumentType]
262 if len(f.keys()) > 1:
263 warnings.warn(
264 f"choosing to load dataset '{key}' from HDF5 file with multiple datasets: {list(f.keys())}"
265 )
267 dataset = f[key]
268 if not isinstance(dataset, h5py.Dataset):
269 raise TypeError(f"HDF5 path {internal_path} does not point to a dataset")
271 labels = cast(Any, dataset.attrs.get("DIMENSION_LABELS"))
272 if isinstance(labels, np.ndarray):
273 labels = labels.tolist()
274 if not is_sequence(labels):
275 if axes is None or dataset.ndim == len(axes): 275 ↛ 279line 275 didn't jump to line 279 because the condition on line 275 was always true
276 dims = axes
277 dims_out = None
278 else:
279 dims = None # guess dims
280 dims_out = axes
281 else:
282 dims = tuple(AxisId(str(a)) for a in labels)
283 dims_out = axes
285 tensor = Tensor.from_array(
286 cast_and_assert(da.Array, da.from_array(dataset)), dims=dims
287 )
288 for attr, setter in (
289 ("axis_units", tensor.set_physical_scale_unit),
290 ("axis_scale", tensor.set_physical_scale),
291 ):
292 values = cast(Any, dataset.attrs.get(attr))
293 if isinstance(values, np.ndarray):
294 values = values.tolist()
295 if is_sequence(values):
296 if len(values) != tensor.ndim:
297 raise ValueError(f"HDF5 attribute {attr} must have one value per axis")
298 setter(dict(zip(tensor.dims, values)))
300 if dims_out is not None: 300 ↛ 301line 300 didn't jump to line 301 because the condition on line 300 was never true
301 tensor = tensor.transpose(
302 dims_out, extra_dims="squeeze_or_stack", missing_dims="expand"
303 )
305 return tensor
308def _load_tensor_bioio(
309 source: ZarrSource | FileSource | ZipPath,
310 internal_path: str | None,
311) -> Tensor:
312 """Load an image tensor using bioio"""
314 import bioio
316 logger.debug("Loading tensor from {} with bioio", source)
318 if isinstance(source, FileDescr): 318 ↛ 319line 318 didn't jump to line 319 because the condition on line 318 was never true
319 src = source.source
320 else:
321 src = source
323 if isinstance(src, (RelativeFilePath, RelativeDirectory)): 323 ↛ 324line 323 didn't jump to line 324 because the condition on line 323 was never true
324 src = src.absolute()
326 del source
328 try:
329 img = bioio.BioImage(src)
330 except Exception:
331 # try to open zarr groups along the internal path for local zarr storage
332 if not isinstance(src, Path) or internal_path is None: 332 ↛ 333line 332 didn't jump to line 333 because the condition on line 332 was never true
333 raise
335 img = None
336 for gs in chain([src / internal_path], (src / internal_path).parents): 336 ↛ 353line 336 didn't jump to line 353 because the loop on line 336 didn't complete
337 if gs == src: 337 ↛ 338line 337 didn't jump to line 338 because the condition on line 337 was never true
338 break
339 try:
340 import bioio_ome_zarr
342 img = bioio.BioImage(gs, reader=bioio_ome_zarr.Reader)
343 except Exception:
344 logger.info(f"Failed to load tensor with bioio from {gs}")
345 else:
346 if gs == src / internal_path:
347 internal_path = None
348 else:
349 internal_path = (src / internal_path).relative_to(gs).as_posix()
351 break
353 if img is None: 353 ↛ 354line 353 didn't jump to line 354 because the condition on line 353 was never true
354 raise
356 if internal_path is not None:
357 internal_path_found = False
358 try:
359 import bioio_ome_zarr
361 except ImportError:
362 pass
363 else:
364 if isinstance(img.reader, bioio_ome_zarr.Reader): 364 ↛ 376line 364 didn't jump to line 376 because the condition on line 364 was always true
365 multiscale = img.reader._multiscales_metadata[img.current_scene_index] # pyright: ignore
366 datasets = multiscale.get("datasets", []) # pyright: ignore
367 data_paths = [datasets[i].get("path") for i in img.resolution_levels] # pyright: ignore
368 if internal_path not in data_paths: 368 ↛ 369line 368 didn't jump to line 369 because the condition on line 368 was never true
369 raise ValueError(
370 f"Resolution level {internal_path} not found in multiscales datasets: {data_paths}"
371 )
372 resolution_level = data_paths.index(internal_path)
373 img.set_resolution_level(resolution_level)
374 internal_path_found = True
375 else:
376 raise NotImplementedError(
377 f"Reading from internal path {internal_path} is not implemented for bioio reader {type(img.reader)}" # pyright: ignore[reportUnknownArgumentType]
378 )
380 if not internal_path_found: 380 ↛ 381line 380 didn't jump to line 381 because the condition on line 380 was never true
381 raise ValueError(
382 f"Internal path {internal_path} not found with bioio in {src}"
383 )
385 dataarray_bioio = cast_and_assert(xr.DataArray, img.xarray_dask_data.squeeze())
387 # map bioio TCZYX/YXS axes to AxisId objects
388 # (e.g. "T" -> AxisId("time"), "C" -> AxisId("channel"), "S" -> AxisId("channel"))
389 axes_map = {
390 d: AxisId("channel") if d == "S" else AxisId(d) for d in dataarray_bioio.dims
391 }
392 dataarray = cast_and_assert(xr.DataArray, dataarray_bioio.rename(axes_map))
394 t = Tensor.from_xarray(dataarray)
396 # add physical scale
397 scale_bioio: dict[str, float | None] = {
398 "T": img.scale.T,
399 "C": img.scale.C,
400 "Z": img.scale.Z,
401 "Y": img.scale.Y,
402 "X": img.scale.X,
403 }
404 if (raw_c_unit := img.dimension_properties.C.unit) is None or str( # pyright: ignore[reportUnknownVariableType] 404 ↛ 409line 404 didn't jump to line 409 because the condition on line 404 was always true
405 raw_c_unit # pyright: ignore[reportUnknownArgumentType]
406 ) == "dimensionless":
407 c_unit = None
408 else:
409 c_unit = str(raw_c_unit) # pyright: ignore[reportUnknownArgumentType]
410 unit_bioio: dict[str, str | None] = {
411 "T": img.dimension_properties.T.unit,
412 "C": c_unit,
413 "Z": img.dimension_properties.Z.unit,
414 "Y": img.dimension_properties.Y.unit,
415 "X": img.dimension_properties.X.unit,
416 }
417 scale = {
418 axes_map[k]: v
419 for k, v in scale_bioio.items()
420 if k in axes_map and v is not None
421 }
422 unit = {
423 axes_map[k]: str(v)
424 for k, v in unit_bioio.items()
425 if k in axes_map and v is not None
426 }
427 try:
428 t.set_physical_scale(scale)
429 except Exception as e:
430 logger.opt(exception=e).warning(
431 "Failed to interprete physical scale: {}", scale
432 )
433 try:
434 t.set_physical_scale_unit(unit)
435 except Exception as e:
436 logger.opt(exception=e).warning(
437 "Failed to interprete physical scale unit: {}", unit
438 )
439 return t
442def _load_tensor_clearscale(source: FileDescr | PermissiveFileSource) -> Tensor:
443 """load a single image as numpy array"""
444 import dask.array as da
446 if isinstance(source, FileDescr): 446 ↛ 447line 446 didn't jump to line 447 because the condition on line 446 was never true
447 source = source.source.absolute()
449 if isinstance(source, RelativeFilePath): 449 ↛ 450line 449 didn't jump to line 450 because the condition on line 449 was never true
450 source = source.absolute()
452 if isinstance(source, ZipPath): 452 ↛ 453line 452 didn't jump to line 453 because the condition on line 452 was never true
453 raise NotImplementedError(
454 "clearscale: Loading from zip files is not implemented."
455 )
457 logger.debug("loading tensor from {} with clearscale+zarr-python", source)
458 array, scale = open_zarr_multiscale_array(
459 source.as_posix() if isinstance(source, Path) else str(source)
460 )
461 image = da.from_zarr(array)
462 axes = tuple(str(a) for a in scale.shape)
463 tensor = Tensor.from_xarray(xr.DataArray(image, dims=axes))
464 try:
465 tensor.set_physical_scale(
466 {AxisId(str(a)): value for a, value in scale.pixel_size.items()}
467 )
468 except Exception as e:
469 logger.opt(exception=e).warning(
470 "Failed to interprete clearscale scale.pixel_size: {}", scale.pixel_size
471 )
472 try:
473 tensor.set_physical_scale_unit(
474 {AxisId(str(a)): value for a, value in scale.unit.items()}
475 )
476 except Exception as e:
477 logger.opt(exception=e).warning(
478 "Failed to interprete clearscale scale.unit: {}", scale.unit
479 )
480 return tensor
483def _load_tensor_tifffile(source: FileDescr | PermissiveFileSource) -> Tensor:
484 import tifffile
486 logger.debug("loading tensor from {} with tifffile", source)
487 if isinstance(source, FileDescr): 487 ↛ 488line 487 didn't jump to line 488 because the condition on line 487 was never true
488 source = source.source.absolute()
490 if isinstance(source, RelativeFilePath): 490 ↛ 491line 490 didn't jump to line 491 because the condition on line 490 was never true
491 source = source.absolute()
493 if isinstance(source, ZipPath): 493 ↛ 494line 493 didn't jump to line 494 because the condition on line 493 was never true
494 ctxt = source.open("rb")
495 name = source.name
496 suffixes = source.suffixes
497 else:
498 if isinstance(source, (RootHttpUrl, pydantic.AnyUrl)): 498 ↛ 499line 498 didn't jump to line 499 because the condition on line 498 was never true
499 source = str(source)
501 if isinstance(source, Path): 501 ↛ 504line 501 didn't jump to line 504 because the condition on line 501 was always true
502 suffixes = source.suffixes
503 else:
504 suffixes = PurePosixPath(source).suffixes
506 ctxt = nullcontext(source)
507 name = None
509 with ctxt as f:
510 assert not isinstance(f, TextIOWrapper)
512 tif = tifffile.TiffFile(f, name=name)
513 if not hasattr(tif, "asxarray"):
514 raise ImportError(
515 "tifffile version >= 2026.5.2 is required for loading tensors with tifffile"
516 )
517 with tif:
518 xr_data = tif.asxarray()
519 page = tif.pages[0]
520 if isinstance(page, tifffile.TiffFrame): 520 ↛ 521line 520 didn't jump to line 521 because the condition on line 520 was never true
521 page = None if page.pages is None else page.pages[0]
522 if isinstance(page, tifffile.TiffFrame):
523 page = None
525 x_res = None
526 y_res = None
527 unit = None
528 if page is not None: 528 ↛ 554line 528 didn't jump to line 554 because the condition on line 528 was always true
529 if "XResolution" in page.tags: 529 ↛ 540line 529 didn't jump to line 540 because the condition on line 529 was always true
530 try:
531 x_res = page.tags["XResolution"].value
532 x_res = x_res[0] / x_res[1] if is_tuple(x_res) else x_res
533 x_res = None if x_res is None else float(x_res)
534 except Exception as e:
535 logger.opt(exception=e).warning(
536 "Failed to interprete tiff XResolution tag: {}", x_res
537 )
538 x_res = None
540 if "YResolution" in page.tags: 540 ↛ 551line 540 didn't jump to line 551 because the condition on line 540 was always true
541 try:
542 y_res = page.tags["YResolution"].value
543 y_res = y_res[0] / y_res[1] if is_tuple(y_res) else y_res
544 y_res = None if y_res is None else float(y_res)
545 except Exception as e:
546 logger.opt(exception=e).warning(
547 "Failed to interprete tiff YResolution tag: {}", y_res
548 )
549 y_res = None
551 unit_tag = page.tags.get("ResolutionUnit")
552 unit = unit_tag.value if unit_tag else None
554 ij_metadata = ImageJMetadata(**(tif.imagej_metadata or {}))
555 ome_metadata = None
556 if ( 556 ↛ 560line 556 didn't jump to line 560 because the condition on line 556 was never true
557 suffixes[-2:] in ([".ome", ".tif"], [".ome", ".tiff"])
558 or not ij_metadata
559 ) and tif.ome_metadata is not None:
560 try:
561 ome_metadata = create_ome_metadata_from_xml_string(tif.ome_metadata)
562 except Exception as e:
563 logger.opt(exception=e).warning(
564 "Failed to parse OME-XML metadata from tif.ome_metadata"
565 )
567 if ome_metadata is None: 567 ↛ 570line 567 didn't jump to line 570
568 ome_metadata = OmeMetadata()
570 t = Tensor.from_xarray(xr_data)
572 # set physical scale
573 x_scale = ome_metadata.PhysicalSizeX or (
574 1 / x_res if x_res is not None and x_res != 0 else None
575 )
576 y_scale = ome_metadata.PhysicalSizeY or (
577 1 / y_res if y_res is not None and y_res != 0 else x_scale
578 )
579 z_scale = ome_metadata.PhysicalSizeZ or ij_metadata.spacing or x_scale
580 t_scale = (
581 ome_metadata.TimeIncrement or ij_metadata.finterval or None
582 if ij_metadata.fps is None
583 else 1 / ij_metadata.fps
584 )
585 scale = {
586 AxisId("x"): x_scale,
587 AxisId("y"): y_scale,
588 AxisId("z"): z_scale,
589 AxisId("time"): t_scale,
590 }
591 try:
592 t.set_physical_scale({k: v for k, v in scale.items() if k in t.dims})
593 except Exception as e:
594 logger.opt(exception=e).warning(
595 "Failed to interprete physical scale from tif metadata: {}", scale
596 )
598 # set physical scale unit
599 x_unit = ome_metadata.PhysicalSizeXUnit or ij_metadata.unit
600 if x_unit is None and unit is not None: 600 ↛ 601line 600 didn't jump to line 601 because the condition on line 600 was never true
601 try:
602 x_unit = tifffile.RESUNIT(unit).name.lower()
603 except Exception as e:
604 logger.opt(exception=e).warning(
605 "Failed to interprete tiff ResolutionUnit tag: {}", unit
606 )
607 x_unit = None
609 y_unit = ome_metadata.PhysicalSizeYUnit or ij_metadata.yunit or x_unit
610 z_unit = ome_metadata.PhysicalSizeZUnit or ij_metadata.zunit or x_unit
611 t_unit = ome_metadata.TimeIncrementUnit or "s"
613 scale_unit = {
614 AxisId("x"): x_unit,
615 AxisId("y"): y_unit,
616 AxisId("z"): z_unit,
617 AxisId("time"): t_unit,
618 }
619 try:
620 t.set_physical_scale_unit({k: v for k, v in scale_unit.items() if k in t.dims})
621 except Exception as e:
622 logger.opt(exception=e).warning(
623 "Failed to interprete physical scale unit from tif metadata: {}", scale_unit
624 )
626 # set channel names
627 if ome_metadata.Channel is not None: 627 ↛ 628line 627 didn't jump to line 628 because the condition on line 627 was never true
628 t.channel_names = ome_metadata.Channel.Name
630 return t
633Suffix = str
636def save_tensor(
637 path: Path | str,
638 tensor: Tensor,
639 *,
640 io_lib: IO_Lib | None = None,
641 chunk_shape: PerAxis[int] | None = None,
642 shard_shape: PerAxis[int] | None = None,
643 progressbar: bool = not os.getenv("GITHUB_ACTIONS"),
644) -> None:
645 _ = _save_tensor_impl(
646 path,
647 tensor,
648 io_lib=io_lib,
649 chunk_shape=chunk_shape,
650 shard_shape=shard_shape,
651 progressbar=progressbar,
652 )
655def _save_tensor_impl(
656 path: Path | str,
657 tensor: Tensor,
658 *,
659 io_lib: IO_Lib | None,
660 chunk_shape: PerAxis[int] | None,
661 shard_shape: PerAxis[int] | None,
662 progressbar: bool,
663) -> IO_Lib:
664 try:
665 output_path, extensions, internal_path = _interprete_tensor_source(path)
666 except Exception:
667 output_path, extensions, internal_path = _interprete_tensor_target(path)
669 if (
670 extensions is None
671 or ".npy" not in extensions
672 and tensor.tagged_shape.get(BATCH_AXIS_ID, 2) == 1
673 ):
674 tensor = tensor.squeeze(BATCH_AXIS_ID)
676 if isinstance(output_path, RootHttpUrl): 676 ↛ 677line 676 didn't jump to line 677 because the condition on line 676 was never true
677 raise NotImplementedError(
678 "Saving tensors to HTTP URLs is not implemented. Please save to a local file path."
679 )
681 if io_lib is None:
682 try_io_libs = _select_io_libs_based_on_extension(extensions)
683 exceptions: list[Exception] = []
684 for try_io_lib in try_io_libs: 684 ↛ 700line 684 didn't jump to line 700 because the loop on line 684 didn't complete
685 try:
686 return _save_tensor_impl(
687 path,
688 tensor,
689 io_lib=try_io_lib,
690 chunk_shape=chunk_shape,
691 shard_shape=shard_shape,
692 progressbar=progressbar,
693 )
694 except Exception as e:
695 exceptions.append(e)
696 logger.opt(exception=e).warning(
697 "Failed to save tensor with io_lib={}", try_io_lib
698 )
700 raise ExceptionGroup(
701 "Failed to save tensor with any available io_lib", exceptions
702 )
704 if progressbar: 704 ↛ 705line 704 didn't jump to line 705 because the condition on line 704 was never true
705 pbar_ctxt = dask.diagnostics.ProgressBar() # pyright: ignore[reportPrivateImportUsage]
706 else:
707 pbar_ctxt = nullcontext()
709 with pbar_ctxt:
710 if io_lib == "bioio":
711 _save_tensor_bioio(
712 output_path,
713 tensor,
714 internal_path=internal_path,
715 chunk_shape=chunk_shape,
716 shard_shape=shard_shape,
717 )
718 elif io_lib == "clearscale":
719 _save_tensor_clearscale(
720 output_path,
721 tensor,
722 internal_path=internal_path,
723 chunk_shape=chunk_shape,
724 shard_shape=shard_shape,
725 )
726 elif io_lib == "imageio":
727 _save_tensor_imageio(output_path, tensor, internal_path=internal_path)
728 elif io_lib == "numpy":
729 if internal_path: 729 ↛ 730line 729 didn't jump to line 730 because the condition on line 729 was never true
730 raise NotImplementedError(
731 "Saving to an internal path is not implemented for numpy."
732 )
733 _save_tensor_numpy(output_path, tensor)
734 elif io_lib == "tifffile":
735 if internal_path is not None: 735 ↛ 736line 735 didn't jump to line 736 because the condition on line 735 was never true
736 raise NotImplementedError(
737 "Saving to an internal path is not implemented for tifffile."
738 )
740 _save_tensor_tifffile(output_path, extensions, tensor)
741 elif io_lib == "h5py":
742 _save_tensor_h5py(
743 output_path,
744 tensor,
745 internal_path=internal_path,
746 chunk_shape=chunk_shape,
747 )
748 else:
749 assert_never(io_lib)
751 return io_lib
754def _save_tensor_h5py(
755 path: Path | ZipPath | FtpUrl,
756 tensor: Tensor,
757 internal_path: PurePath | None,
758 chunk_shape: PerAxis[int] | None = None,
759) -> None:
760 import h5py # pyright: ignore[reportMissingTypeStubs]
762 if not isinstance(path, Path): 762 ↛ 763line 762 didn't jump to line 763 because the condition on line 762 was never true
763 raise NotImplementedError("Saving with h5py requires a local file path.")
765 name = internal_path.as_posix() if internal_path is not None else "data"
766 path.parent.mkdir(parents=True, exist_ok=True)
767 with h5py.File(path, "a") as f:
768 if name in f:
769 if not isinstance(f[name], h5py.Dataset): 769 ↛ 770line 769 didn't jump to line 770 because the condition on line 769 was never true
770 raise ValueError(f"HDF5 path {name} does not point to a dataset")
771 del f[name]
773 dataset = f.create_dataset(
774 name,
775 dtype=tensor.dtype,
776 shape=tensor.shape,
777 chunks=tuple(chunk_shape[a] for a in tensor.dims)
778 if chunk_shape is not None
779 else True,
780 compression="lzf",
781 )
782 dataset.attrs["DIMENSION_LABELS"] = [str(a) for a in tensor.dims]
783 units = tensor.get_physical_scale_unit()
784 if any(units.values()): 784 ↛ 786line 784 didn't jump to line 786 because the condition on line 784 was always true
785 dataset.attrs["axis_units"] = [units.get(a, "") for a in tensor.dims]
786 scale = tensor.get_physical_scale()
787 if scale: 787 ↛ 790line 787 didn't jump to line 790 because the condition on line 787 was always true
788 dataset.attrs["axis_scale"] = [scale.get(a, 1.0) for a in tensor.dims]
790 if all(d[0].lower() in "tczxy" for d in tensor.dims): 790 ↛ 806line 790 didn't jump to line 806 because the condition on line 790 was always true
791 # add vigra axis tags
792 dataset.attrs["axistags"] = json.dumps(
793 {
794 "axes": [
795 {
796 "key": (a := str(d[0]).lower()),
797 "typeFlags": 1 if a == "c" else 2 if a in "zxy" else 8,
798 "resolution": scale.get(d, 0),
799 "description": units.get(d, ""),
800 }
801 for d in tensor.dims
802 ]
803 }
804 )
806 data = cast_and_assert(da.Array, tensor.as_dask().data.data)
807 da.store(data, dataset, compute=True, lock=True)
810def _save_tensor_bioio(
811 path: Path | ZipPath | FtpUrl,
812 tensor: Tensor,
813 internal_path: PurePath | None,
814 chunk_shape: PerAxis[int] | None = None,
815 shard_shape: PerAxis[int] | None = None,
816) -> None:
817 """Save an image tensor using bioio"""
819 import bioio_base.types
821 logger.info(
822 "Saving tensor to {}{} with bioio",
823 path,
824 f"/{internal_path}" if internal_path else "",
825 )
826 bioio_axis_map = {
827 "batch": "S",
828 "channel": "C",
829 "index": "S",
830 "time": "T",
831 }
832 scale = tensor.get_physical_scale()
834 if path.suffix.lower() == ".zarr":
835 if internal_path and str(internal_path) != "0":
836 raise NotImplementedError(
837 "Saving to an internal path that is not '0' is not implemented for zarr files with io_lib='bioio'."
838 )
840 import bioio_ome_zarr.writers
842 try:
843 import zarr
844 except ImportError:
845 zarr_format = 3
846 else:
847 zarr_format = zarr.config.get("default_zarr_format")
848 if zarr_format not in (2, 3): 848 ↛ 849line 848 didn't jump to line 849 because the condition on line 848 was never true
849 raise NotImplementedError(
850 f"DEFAULT_ZARR_FORMAT={zarr_format} is not supported. (expected 2 or 3)"
851 )
853 axes_names = single_letter_dims_if_possible(tensor.dims)
854 axes_types = [
855 {"time": "time", "t": "time", "channel": "channel", "c": "channel"}.get(
856 str(a).lower(), "space"
857 )
858 for a in tensor.dims
859 ]
861 channel_names = tensor.channel_names
862 channel_colors = tensor.channel_colors
864 channels = [
865 bioio_ome_zarr.writers.Channel(
866 label=f"channel{i}" if channel_names is None else channel_names[i],
867 color="#FF0000" if channel_colors is None else channel_colors[i],
868 )
869 for i in range(tensor.tagged_shape.get(AxisId("channel"), 0))
870 ]
872 unit = tensor.get_physical_scale_unit()
873 writer = bioio_ome_zarr.writers.OMEZarrWriter(
874 store=path,
875 level_shapes=[[tensor.tagged_shape[d] for d in tensor.dims]],
876 dtype=tensor.dtype,
877 zarr_format=zarr_format,
878 image_name=tensor.name or "Image",
879 axes_names=list(axes_names),
880 axes_types=axes_types,
881 axes_units=[unit.get(a) for a in tensor.dims],
882 physical_pixel_size=[scale.get(a, 1.0) for a in tensor.dims],
883 channels=channels,
884 chunk_shape=[chunk_shape[a] for a in tensor.dims]
885 if chunk_shape is not None
886 else None,
887 shard_shape=[shard_shape[a] for a in tensor.dims]
888 if shard_shape is not None
889 else None,
890 )
891 logger.info("OME-Zarr metadata: {}", writer.preview_metadata())
892 writer.write_full_volume(tensor.data.data)
893 return
894 elif path.suffix.lower() in (".tif", ".tiff"):
895 from bioio_ome_tiff.writers import OmeTiffWriter
897 save = OmeTiffWriter.save # pyright: ignore[reportUnknownVariableType]
898 bioio_axis_letters = "TCZYX"
899 elif path.suffix.lower() in (".gif", ".mp4", ".mkv"): 899 ↛ 900line 899 didn't jump to line 900 because the condition on line 899 was never true
900 from bioio_imageio.writers import TimeseriesWriter
902 tensor = tensor.squeeze()
903 save = TimeseriesWriter.save # pyright: ignore[reportUnknownVariableType]
904 bioio_axis_map["channel"] = "S"
905 bioio_axis_letters = "YXS"
906 elif path.suffix.lower() in ( 906 ↛ 927line 906 didn't jump to line 927 because the condition on line 906 was always true
907 ".png",
908 ".bmp",
909 ".jpg",
910 ".mov",
911 ".avi",
912 ".mpg",
913 ".mpeg",
914 ".mp4",
915 ".mkv",
916 ".wmv",
917 ".ogg",
918 ):
919 from bioio_imageio.writers import TwoDWriter
921 tensor = tensor.squeeze()
922 save = TwoDWriter.save # pyright: ignore[reportUnknownVariableType]
923 # channel dim denoted as 'S'
924 bioio_axis_map["channel"] = "S"
925 bioio_axis_letters = "YXS"
926 else:
927 raise RuntimeError(
928 f"Failed to identify a suitable writer for {path} with suffix={path.suffix}."
929 )
931 bioio_data = tensor.data
932 dim_map = {
933 d: bioio_axis_map.get(str(d).lower(), str(d)[:1].upper()) for d in tensor.dims
934 }
936 bioio_data = cast_and_assert(xr.DataArray, bioio_data.rename(dim_map))
938 if not all(str(d) in bioio_axis_letters for d in bioio_data.dims): 938 ↛ 939line 938 didn't jump to line 939 because the condition on line 938 was never true
939 logger.warning(
940 "Attempting to save tensor with dimensions {} not in '{}' using bioio.",
941 bioio_data.dims,
942 bioio_axis_letters,
943 )
945 dim_order = "".join(str(d) for d in bioio_data.dims)
947 image_name = None if internal_path is None else str(internal_path)
948 physical_scale = tensor.get_physical_scale()
949 _ = save(
950 bioio_data.data,
951 path,
952 dim_order=dim_order,
953 image_name=image_name,
954 channel_names=tensor.channel_names,
955 channel_colors=None
956 if (channel_colors := tensor.channel_colors) is None
957 else [list(hex_to_rgb(c)) for c in channel_colors],
958 physical_pixel_sizes=bioio_base.types.PhysicalPixelSizes(
959 physical_scale.get(AxisId("z")),
960 physical_scale.get(AxisId("y")),
961 physical_scale.get(AxisId("x")),
962 ),
963 )
966def _save_tensor_clearscale(
967 path: Path | ZipPath | FtpUrl,
968 tensor: Tensor,
969 internal_path: PurePath | None,
970 chunk_shape: PerAxis[int] | None = None,
971 shard_shape: PerAxis[int] | None = None,
972) -> None:
973 import clearscale
974 import dask.array
975 import zarr
977 # we parallelize at a higher level, so here we write single-threaded.
978 _ = zarr.config.set({"threading.max_workers": 1})
980 single_letter_dims = single_letter_dims_if_possible(tensor.dims)
982 # prefer canonical order for standard volumes
983 canonical_axes = "tczyx"
984 if all(d in canonical_axes for d in single_letter_dims): 984 ↛ 1003line 984 didn't jump to line 1003 because the condition on line 984 was always true
985 canonical_single_letter_dims = tuple(
986 a for a in canonical_axes if a in single_letter_dims
987 )
988 canonical_order = tuple(
989 tensor.dims[single_letter_dims.index(a)]
990 for a in canonical_single_letter_dims
991 )
992 if tensor.dims != canonical_order: 992 ↛ 993line 992 didn't jump to line 993 because the condition on line 992 was never true
993 logger.debug(
994 "Transposing tensor from {} to canonical order {} for saving with clearscale+zarr-python",
995 tensor.dims,
996 canonical_order,
997 )
998 tensor = tensor.transpose(
999 canonical_order, missing_dims="raise", extra_dims="raise"
1000 )
1001 single_letter_dims = canonical_single_letter_dims
1003 tensor_shape_tuple = tuple(tensor.tagged_shape[d] for d in tensor.dims)
1005 zarr_format = zarr.config.get("default_zarr_format")
1006 scale_key = "s0" if internal_path is None else str(internal_path)
1007 logger.info(
1008 "Saving tensor {} to '{}' of {} with clearscale+zarr-python using zarr_format={} and dims={}",
1009 tensor.tagged_shape,
1010 scale_key,
1011 path,
1012 zarr_format,
1013 single_letter_dims,
1014 )
1016 dim_map = dict(zip(tensor.dims, single_letter_dims))
1017 multiscale: Any = clearscale.Multiscale(
1018 {
1019 scale_key: clearscale.Scale(
1020 shape=clearscale.Shape(
1021 **dict(zip(dim_map.values(), tensor_shape_tuple))
1022 ),
1023 pixel_size={
1024 dim_map[k]: v for k, v in tensor.get_physical_scale().items()
1025 }
1026 or None,
1027 unit={
1028 dim_map[k]: v for k, v in tensor.get_physical_scale_unit().items()
1029 }
1030 or None,
1031 )
1032 }
1033 )
1034 if zarr_format == 2:
1035 zarr_attrs: dict[str, JsonValueReadOnly] = {
1036 "multiscales": [
1037 multiscale.to_ome_zarr(version="0.4")
1038 ], # TODO: add axis semantics
1039 }
1040 elif zarr_format == 3: 1040 ↛ 1050line 1040 didn't jump to line 1050 because the condition on line 1040 was always true
1041 zarr_attrs: dict[str, JsonValueReadOnly] = {
1042 "ome": {
1043 "version": "0.5",
1044 "multiscales": [
1045 multiscale.to_ome_zarr(version="0.5")
1046 ], # TODO: add axis semantics
1047 }
1048 }
1049 else:
1050 raise NotImplementedError(f"Zarr format {zarr_format} is not supported.")
1052 zarr_group = zarr.open_group(
1053 str(path),
1054 mode="w",
1055 zarr_format=zarr_format,
1056 attributes=zarr_attrs,
1057 )
1059 create_array_kwargs = _CreateArrayKwargs(
1060 name=scale_key,
1061 overwrite=True,
1062 dimension_names=None if zarr_format == 2 else tuple(dim_map.values()),
1063 chunks="auto"
1064 if chunk_shape is None
1065 else tuple(chunk_shape[d] for d in tensor.dims),
1066 shards=None
1067 if zarr_format == 2
1068 else "auto"
1069 if shard_shape is None
1070 else tuple(shard_shape[d] for d in tensor.dims),
1071 )
1072 zarr_array = zarr_group.require_array(
1073 **create_array_kwargs,
1074 shape=[tensor.tagged_shape[d] for d in tensor.dims],
1075 dtype=tensor.dtype,
1076 )
1077 xr_array = tensor.data
1078 if dm := {k: v for k, v in dim_map.items() if k != v}:
1079 xr_array = cast_and_assert(xr.DataArray, xr_array.rename(dm))
1081 if isinstance(xr_array.data, np.ndarray):
1082 zarr_array[:] = xr_array.data
1083 elif isinstance(xr_array.data, dask.array.Array):
1084 dask.array.to_zarr(xr_array.data, zarr_array, compute=True)
1085 else:
1086 assert_never(xr_array.data)
1089def _save_tensor_imageio(
1090 path: Path | ZipPath | FtpUrl, tensor: Tensor, internal_path: PurePath | None
1091) -> None:
1092 if internal_path is not None: 1092 ↛ 1093line 1092 didn't jump to line 1093 because the condition on line 1092 was never true
1093 raise NotImplementedError(
1094 "Saving tensors to subdirectories is not implemented for imageio. Please save to a file path without internal path."
1095 )
1097 if isinstance(path, ZipPath): 1097 ↛ 1098line 1097 didn't jump to line 1098 because the condition on line 1097 was never true
1098 raise NotImplementedError(
1099 "Saving tensors to zip files is not implemented for imageio. Please save to an unzipped file path."
1100 )
1102 logger.debug("Saving tensor to {} with imageio", path)
1103 if not isinstance(path, FtpUrl): 1103 ↛ 1106line 1103 didn't jump to line 1106 because the condition on line 1103 was always true
1104 path.parent.mkdir(exist_ok=True, parents=True)
1106 extension = path.suffix.lower()
1107 removed_singleton_axes: list[AxisId] = []
1108 remove_singletons = {
1109 AxisId("batch"): [
1110 ".tif",
1111 ".tiff",
1112 ], # remove singleton batch dim for tiff files
1113 **{
1114 a: [".png", ".jpg", ".jpeg"] for a in tensor.dims
1115 }, # remove any singleton axis for png and jpg files
1116 }
1117 for rm_a, rm_ext in remove_singletons.items():
1118 if extension in rm_ext and tensor.tagged_shape.get(rm_a) == 1:
1119 tensor = tensor[{rm_a: 0}]
1120 removed_singleton_axes.append(rm_a)
1122 if removed_singleton_axes:
1123 singleton_axes_msg = (
1124 f"(with removed singleton axes {list(map(str, removed_singleton_axes))}) "
1125 )
1126 else:
1127 singleton_axes_msg = ""
1129 logger.info(
1130 "writing tensor {} {}to {}",
1131 dict(tensor.tagged_shape),
1132 singleton_axes_msg,
1133 path,
1134 )
1135 if extension in (".png", ".jpg", ".jpeg") and tensor.dtype in ( 1135 ↛ 1139line 1135 didn't jump to line 1139 because the condition on line 1135 was never true
1136 "float32",
1137 "float64",
1138 ):
1139 logger.warning(
1140 "converting tensor of dtype {} to uint8 for saving as {}",
1141 tensor.dtype,
1142 extension,
1143 )
1144 tensor = (
1145 (tensor - (t_min := cast(float, tensor.data.min())))
1146 / xr.ufuncs.maximum(cast(float, tensor.data.max()) - t_min, 1e-8)
1147 * 255
1148 ).astype("uint8")
1150 imwrite(path, tensor, extension=extension)
1153def _save_tensor_numpy(path: Path | ZipPath | FtpUrl, tensor: Tensor) -> None:
1154 if isinstance(path, FtpUrl): 1154 ↛ 1155line 1154 didn't jump to line 1155 because the condition on line 1154 was never true
1155 raise NotImplementedError(
1156 "Saving numpy arrays to FTP URLs is not implemented. Please save to a local file path instead."
1157 )
1159 logger.info("Saving tensor to {} with numpy", path)
1161 if isinstance(path, ZipPath): 1161 ↛ 1162line 1161 didn't jump to line 1162 because the condition on line 1161 was never true
1162 folder = path.filename.parent
1163 else:
1164 folder = path.parent
1166 folder.mkdir(exist_ok=True, parents=True)
1168 save_array(path, tensor.to_numpy())
1171def _save_tensor_tifffile(
1172 path: Path | ZipPath | FtpUrl, extensions: list[str] | None, tensor: Tensor
1173) -> None:
1174 if isinstance(path, FtpUrl): 1174 ↛ 1175line 1174 didn't jump to line 1175 because the condition on line 1174 was never true
1175 raise NotImplementedError(
1176 "Saving tiff files to FTP URLs is not implemented. Please save to a local file path instead."
1177 )
1179 import inspect
1181 import tifffile
1183 tifffile_imwrite_signature = inspect.signature(
1184 cast(Callable[..., Any], tifffile.imwrite)
1185 )
1186 if "kind" not in tifffile_imwrite_signature.parameters:
1187 raise NotImplementedError(
1188 "Saving tensors with tifffile uses tifffile.imwrite(...kind=...) (added in tifffile 2026.5.2)."
1189 )
1191 logger.trace("tensor dims {}", tensor.dims)
1192 tensor = tensor.transpose("TZCYX") # tifffile expects TZCYX order
1193 logger.trace("tensor dims transposed {}", tensor.dims)
1194 if len(tensor.dims) > 5: 1194 ↛ 1195line 1194 didn't jump to line 1195 because the condition on line 1194 was never true
1195 logger.warning(
1196 "Saving tensor with more than 5 dimensions ({}) will likely fail."
1197 )
1199 logger.info("Saving tensor to {} with tifffile", path)
1201 scale = tensor.get_physical_scale()
1202 x_scale = scale.get(AxisId("x"))
1203 y_scale = scale.get(AxisId("y"))
1204 z_scale = scale.get(AxisId("z"))
1205 t_scale = scale.get(AxisId("time"))
1206 if x_scale is not None and y_scale is not None: 1206 ↛ 1209line 1206 didn't jump to line 1209 because the condition on line 1206 was always true
1207 resolution = (1 / x_scale, 1 / y_scale)
1208 else:
1209 resolution = None
1211 scale_unit = tensor.get_physical_scale_unit()
1212 x_unit = scale_unit.get(AxisId("x"))
1213 y_unit = scale_unit.get(AxisId("y"))
1214 z_unit = scale_unit.get(AxisId("z"))
1215 t_unit = scale_unit.get(AxisId("time"))
1216 if x_unit == "inch": 1216 ↛ 1217line 1216 didn't jump to line 1217 because the condition on line 1216 was never true
1217 resolution_unit = tifffile.RESUNIT.INCH
1218 elif x_unit in ("centimeter", "cm"): 1218 ↛ 1219line 1218 didn't jump to line 1219 because the condition on line 1218 was never true
1219 resolution_unit = tifffile.RESUNIT.CENTIMETER
1220 elif x_unit in ("millimeter", "mm"): 1220 ↛ 1221line 1220 didn't jump to line 1221 because the condition on line 1220 was never true
1221 resolution_unit = tifffile.RESUNIT.MILLIMETER
1222 elif x_unit in ("micrometer", "um", "µm"):
1223 resolution_unit = tifffile.RESUNIT.MICROMETER
1224 else:
1225 resolution_unit = None
1227 if isinstance(path, ZipPath): 1227 ↛ 1228line 1227 didn't jump to line 1228 because the condition on line 1227 was never true
1228 folder = path.filename.parent
1229 else:
1230 folder = path.parent
1232 folder.mkdir(exist_ok=True, parents=True)
1234 if extensions is not None and extensions[-2:] in ( 1234 ↛ 1238line 1234 didn't jump to line 1238 because the condition on line 1234 was never true
1235 [".ome", ".tif"],
1236 [".ome", ".tiff"],
1237 ):
1238 metadata = OmeMetadata(
1239 # "DimensionOrder": "TZCYX", # [::-1]?? # TODO: check if this is needed
1240 TimeIncrement=t_scale,
1241 TimeIncrementUnit=t_unit,
1242 PhysicalSizeX=x_scale,
1243 PhysicalSizeXUnit=x_unit,
1244 PhysicalSizeY=y_scale,
1245 PhysicalSizeYUnit=y_unit,
1246 PhysicalSizeZ=z_scale,
1247 PhysicalSizeZUnit=z_unit,
1248 Channel=None
1249 if tensor.channel_names is None
1250 else OmeChannelMetadata(Name=tensor.channel_names),
1251 )
1252 kind = "ome"
1253 else:
1254 if not t_unit or t_scale is None: 1254 ↛ 1256line 1254 didn't jump to line 1256 because the condition on line 1254 was always true
1255 pass
1256 elif isinstance(t_scale, (int, float)):
1257 if t_unit in ("nanosecond", "nanosecond(s)", "ns"):
1258 t_scale /= 1_000_000_000
1259 elif t_unit in ("microsecond", "microsecond(s)", "us", "µsec", "µs"):
1260 t_scale /= 1_000_000
1261 elif t_unit in ("millisecond", "millisecond(s)", "ms"):
1262 t_scale /= 1_000
1263 elif t_unit in ("minute", "minute(s)", "min"):
1264 t_scale *= 60
1265 elif t_unit in ("hour", "hour(s)", "h"):
1266 t_scale *= 3_600
1267 elif t_unit in ("day", "day(s)", "d"):
1268 t_scale *= 86_400
1269 elif t_unit in ("week", "week(s)", "wk"):
1270 t_scale *= 604_800
1271 elif t_unit in ("month", "month(s)", "mo"):
1272 t_scale *= 2_629_746 # average month length in seconds
1273 elif t_unit in ("year", "year(s)", "yr"):
1274 t_scale *= 31_556_952 # average year length in seconds
1275 elif t_unit not in ("second", "s"):
1276 logger.warning(
1277 "Unknown time unit '{}' encountered while saving an imagej tiff with tifffile",
1278 t_unit,
1279 )
1280 else:
1281 logger.warning(
1282 "Failed to convert time interval '{}' of type {} (with time unit '{}') to seconds while saving an imagej tiff with tifffile",
1283 t_scale,
1284 type(t_scale),
1285 t_unit,
1286 )
1288 metadata = ImageJMetadata(
1289 unit=x_unit,
1290 yunit=y_unit,
1291 zunit=z_unit,
1292 spacing=z_scale,
1293 finterval=t_scale,
1294 fps=None if not isinstance(t_scale, (int, float)) else 1 / t_scale,
1295 )
1296 kind = "imagej"
1298 if isinstance(path, ZipPath): 1298 ↛ 1299line 1298 didn't jump to line 1299 because the condition on line 1298 was never true
1299 ctxt = path.open("wb")
1300 else:
1301 ctxt = nullcontext(path)
1303 with ctxt as f:
1304 assert not isinstance(f, TextIOWrapper)
1305 _ = tifffile.imwrite(
1306 f,
1307 tensor.to_numpy(),
1308 kind=kind,
1309 software="bioimageio.core",
1310 resolution=resolution,
1311 metadata={
1312 k: v
1313 for k, v in metadata.model_dump(mode="json", exclude_none=True).items()
1314 },
1315 resolutionunit=resolution_unit,
1316 )
1319class _CreateArrayKwargs(TypedDict):
1320 name: str
1321 overwrite: bool
1322 dimension_names: tuple[str, ...] | None
1323 shards: Literal["auto"] | tuple[int, ...] | None
1324 chunks: Literal["auto"] | tuple[int, ...]
1327_zarr_url_adapter: TypeAdapter[ZarrUrl] = TypeAdapter(ZarrUrl)
1330def _interprete_tensor_target(
1331 source: PermissiveFileSource | ZarrSource | ZipPath | FileDescr,
1332):
1333 if isinstance(source, FileDescr):
1334 source = source.source
1336 if isinstance(source, (str, pydantic.AnyUrl)):
1337 try:
1338 source = _zarr_url_adapter.validate_python(str(source))
1339 except Exception:
1340 if (
1341 str(source).startswith("http://")
1342 or str(source).startswith("https://")
1343 or str(source).startswith("ftp://")
1344 ):
1345 raise
1346 else:
1347 source = Path(str(source))
1349 if isinstance(source, (RelativeFilePath, RelativeDirectory)):
1350 source = source.absolute()
1352 return _get_file_source_extension_and_internal_path(source)
1355def _interprete_tensor_source(
1356 source: PermissiveFileSource | ZarrSource | ZipPath | FileDescr,
1357):
1359 if isinstance(source, FileDescr): 1359 ↛ 1360line 1359 didn't jump to line 1360 because the condition on line 1359 was never true
1360 source = source.source
1362 if isinstance(source, (str, pydantic.AnyUrl)):
1363 source = interprete_file_source(source, allow_zarr=True)
1365 if isinstance(source, (RelativeFilePath, RelativeDirectory)): 1365 ↛ 1366line 1365 didn't jump to line 1366 because the condition on line 1365 was never true
1366 source = source.absolute()
1368 return _get_file_source_extension_and_internal_path(source)
1371def _get_file_source_extension_and_internal_path(
1372 source: Path | ZipPath | RootHttpUrl | FtpUrl,
1373):
1374 internal_path = None
1375 if isinstance(source, (HttpUrl, RootHttpUrl, FtpUrl)): 1375 ↛ 1376line 1375 didn't jump to line 1376 because the condition on line 1375 was never true
1376 original_source_path = PosixPath(source.path or "")
1377 parents: Iterable[FtpUrl | RootHttpUrl | Path | ZipPath] = source.parents
1378 elif isinstance(source, ZipPath): 1378 ↛ 1379line 1378 didn't jump to line 1379 because the condition on line 1378 was never true
1379 original_source_path = PosixPath(source.filename)
1380 parents = []
1381 p = source
1382 while p != p.parent:
1383 parents.append(p.parent)
1384 p = p.parent
1385 else:
1386 original_source_path = PosixPath(source)
1387 parents = source.parents
1389 file_source = source
1390 for parent in chain([source], parents): 1390 ↛ 1407line 1390 didn't jump to line 1407 because the loop on line 1390 didn't complete
1391 extensions = [s.lower() for s in parent.suffixes]
1392 if extensions:
1393 file_source = parent
1394 if isinstance(parent, (RootHttpUrl, FtpUrl)): 1394 ↛ 1395line 1394 didn't jump to line 1395 because the condition on line 1394 was never true
1395 parent_path = PosixPath(parent.path or "")
1396 elif isinstance(parent, ZipPath): 1396 ↛ 1397line 1396 didn't jump to line 1397 because the condition on line 1396 was never true
1397 parent_path = PosixPath(parent.filename)
1398 else:
1399 parent_path = PosixPath(parent)
1401 internal_path = original_source_path.relative_to(parent_path)
1402 if internal_path == PosixPath("."):
1403 internal_path = None
1405 break
1406 else:
1407 extensions = None
1409 return file_source, extensions, internal_path
1412def save_sample(
1413 path: Path | str | PerMember[Path | str],
1414 sample: Sample,
1415 *,
1416 chunk_shape: PerMember[PerAxis[int]] | None = None,
1417 shard_shape: PerMember[PerAxis[int]] | None = None,
1418) -> PerMember[Path]:
1419 """Save a **sample** to a **path** pattern
1420 or all sample members in the **path** mapping.
1422 Args:
1423 path: A path pattern or a mapping of member_id to path pattern.
1424 If **path** is a pathlib.Path or a string and the **sample** has multiple members,
1425 **path** it must contain `{member_id}` (or `{input_id}` or `{output_id}`).
1426 sample: Sample to save
1427 chunk_shape: Optional (re-)chunking of the sample data for applicable image formats (zarr).
1428 shard_shape: Optional sharding of the sample data for applicable image formats (zarr).
1430 (Each) **path** may contain `{sample_id}` to be formatted with the **sample** object.
1431 """
1433 if not isinstance(path, Mapping):
1434 if len(sample.members) < 2 or any( 1434 ↛ 1439line 1434 didn't jump to line 1439 because the condition on line 1434 was always true
1435 m in str(path) for m in ("{member_id}", "{input_id}", "{output_id}")
1436 ):
1437 path = {m: path for m in sample.members}
1438 else:
1439 raise ValueError(
1440 f"path {path} must contain '{{member_id}}' for sample with multiple members {list(sample.members)}."
1441 )
1443 output_paths = {
1444 m: Path(
1445 str(p).format(sample_id=sample.id, member_id=m, input_id=m, output_id=m)
1446 )
1447 for m, p in path.items()
1448 }
1449 with ThreadPoolExecutor(max_workers=min(len(sample.members), 4)) as executor:
1450 for m, p in output_paths.items():
1451 _ = executor.submit(
1452 save_tensor,
1453 p,
1454 sample.members[m],
1455 chunk_shape=chunk_shape[m] if chunk_shape is not None else None,
1456 shard_shape=shard_shape[m] if shard_shape is not None else None,
1457 )
1459 return output_paths
1462class _StatEntry(BaseModel, frozen=True, arbitrary_types_allowed=True):
1463 """Serializable stat entry"""
1465 measure: DatasetMeasure | SampleMeasure
1466 value: MeasureValue
1469class _StatList(RootModel[list[_StatEntry]]):
1470 """Serializable stat mapping"""
1473def serialize_stat(
1474 stat: Mapping[DatasetMeasure | SampleMeasure, MeasureValue],
1475) -> list[JsonValue]:
1476 """Serialize a stat mapping to a JSON string"""
1477 stat_list = _StatList([_StatEntry(measure=k, value=v) for k, v in stat.items()])
1478 return stat_list.model_dump(mode="json")
1481def save_stat(
1482 stat: Mapping[DatasetMeasure | SampleMeasure, MeasureValue],
1483 output: Path | BytesIO,
1484) -> None:
1485 """Save sample and dataset statistics as a JSON file"""
1487 if isinstance(output, Path): 1487 ↛ 1490line 1487 didn't jump to line 1490 because the condition on line 1487 was always true
1488 ctxt = output.open("wb")
1489 else:
1490 ctxt = nullcontext(output)
1492 with ctxt as out:
1493 _ = out.write(json.dumps(serialize_stat(stat), indent=2).encode("utf-8"))
1496def load_stat(source: Path | str | Sequence[JsonValue]) -> Stat:
1497 """Load sample and dataset statistics from JSON"""
1498 if isinstance(source, Path):
1499 source = source.read_text(encoding="utf-8")
1501 if isinstance(source, str):
1502 seq = _StatList.model_validate_json(source)
1503 else:
1504 seq = _StatList.model_validate(source)
1506 return {e.measure: e.value for e in seq.root}
1509def save_dataset_stat(stat: Mapping[DatasetMeasure, MeasureValue], path: Path) -> None:
1510 """DEPRECATED alias for save_stat(): use `save_stats()` instead."""
1511 warnings.warn("`save_dataset_stat()` is deprecated, use `save_stats()` instead.")
1512 save_stat({k: v for k, v in stat.items()}, path)
1515def load_dataset_stat(path: Path) -> Stat:
1516 """DEPRECATED alias for `load_stat()`: use `load_stat()` instead."""
1517 warnings.warn("`load_dataset_stat()` is deprecated, use `load_stats()` instead.")
1518 return load_stat(path)
1521def ensure_unzipped(source: PermissiveFileSource | ZipPath | BytesReader, folder: Path):
1522 """unzip a (downloaded) **source** to a file in **folder** if source is a zip archive
1523 otherwise copy **source** to a file in **folder**."""
1524 if isinstance(source, BytesReader): 1524 ↛ 1525line 1524 didn't jump to line 1525 because the condition on line 1524 was never true
1525 weights_reader = source
1526 else:
1527 weights_reader = get_reader(source)
1529 out_path = folder / (
1530 weights_reader.original_file_name or f"file{weights_reader.suffix}"
1531 )
1533 if zipfile.is_zipfile(weights_reader): 1533 ↛ 1541line 1533 didn't jump to line 1541 because the condition on line 1533 was always true
1534 out_path = out_path.with_name(out_path.name + ".unzipped")
1535 out_path.parent.mkdir(exist_ok=True, parents=True)
1536 # source itself is a zipfile
1537 with zipfile.ZipFile(weights_reader, "r") as f:
1538 f.extractall(out_path)
1540 else:
1541 out_path.parent.mkdir(exist_ok=True, parents=True)
1542 with out_path.open("wb") as f:
1543 copyfileobj(weights_reader, f)
1545 return out_path
1548def get_suffix(source: ZipPath | FileSource) -> Suffix:
1549 """DEPRECATED: use source.suffix instead."""
1550 return source.suffix
1553# def _determine_chunk_and_shard_shapes(
1554# shape: PerAxis[int],
1555# block_shape: PerAxis[int],
1556# item_size: int,
1557# ) -> tuple[PerAxis[int], PerAxis[int]]:
1558# def increase_shard_size(shard_size: int, shape_size: int, chunk_size: int) -> int:
1559# """increase shard size by chunk_size (or a multiple of chunk_size such that the last two chunks fall into the same shard).
1560# Will not increase shard size beyond shape_size.
1561# """
1562# while (
1563# shard_size < shape_size
1564# and shape_size % shard_size
1565# != 0 # last shard is smaller than the shard size
1566# and shape_size % shard_size
1567# < 2 * chunk_size # last shard is smaller than 2*inner_size,
1568# # but the last two blocks may overlap and must land in the same shard
1569# ):
1570# shard_size = min(shard_size + chunk_size, shape_size)
1572# return shard_size
1574# shard_shape: dict[AxisId, int] | None = {}
1575# # create smallest possible shard shape
1576# for a, shape_size in shape.items():
1577# # The smallest possible shard shape is made up of the inner sizes (that will make up the chunks)
1578# block_size = block_shape[a]
1579# shard_shape[a] = increase_shard_size(block_size, shape_size, block_size)
1581# shard_byte_size = prod(shard_shape.values()) * item_size
1582# if (
1583# shard_byte_size < SHARDED_INNER_CHUNK_MAX_BYTES // 2
1584# and BATCH_AXIS_ID in shape
1585# and unstacked_batch_shape is not None
1586# ):
1587# # increase shard shape along a stacked multi-index batch dimension if present (to optimize writing)
1588# shard_shape[BATCH_AXIS_ID] = max(
1589# shard_shape[BATCH_AXIS_ID], prod(unstacked_batch_shape.values())
1590# )
1592# shard_byte_size = prod(shard_shape.values()) * item_size
1593# increase_order = [
1594# a
1595# for a in (
1596# CHANNEL_AXIS_ID,
1597# AxisId("x"),
1598# AxisId("y"),
1599# AxisId("z"),
1600# AxisId("t"),
1601# AxisId("i"),
1602# )
1603# if a in shape
1604# ]
1605# increase_order += [a for a in shape if a not in increase_order]
1606# while (
1607# any(shard_shape[a] < s for a, s in shape.items())
1608# and shard_byte_size < SHARDED_INNER_CHUNK_MAX_BYTES // 2
1609# ):
1610# for a in increase_order:
1611# shard_shape[a] = increase_shard_size(
1612# shard_shape[a], shape[a], inner_sizes[a]
1613# )
1614# shard_byte_size = prod(shard_shape.values()) * item_size
1615# if shard_byte_size >= SHARDED_INNER_CHUNK_MAX_BYTES // 2:
1616# break