Coverage for src/bioimageio/core/_model_adapter.py: 96%
188 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 09:46 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 09:46 +0000
1from __future__ import annotations
3import gc
4import time
5import warnings
6from abc import ABC, abstractmethod
7from collections.abc import Callable, Sequence
8from dataclasses import dataclass, field
9from queue import LifoQueue
10from typing import (
11 Any,
12 Generic,
13 Literal,
14 cast,
15)
17import dask.array as da
18from dask.delayed import Delayed, delayed # pyright: ignore[reportUnknownVariableType]
19from exceptiongroup import ExceptionGroup
20from loguru import logger
21from numpy.typing import NDArray
22from typing_extensions import TypeVar, assert_never
24from bioimageio.spec import ValidationSummary
25from bioimageio.spec.model import AnyModelDescr, v0_4
27from ._sample_serializer import SampleSerializer, SerializedSampleBlockType
28from ._utils.type_hints import NestedList, cast_and_assert
29from .axis import AxisId, PerAxis
30from .common import MemberId, PerMember
31from .digest_spec import (
32 get_axes_infos,
33 get_block_transform,
34 get_member_ids,
35 get_unstacked_output_dim_orders_for_model,
36)
37from .sample import Sample
38from .tensor import Array, MultiIndex, Tensor, restore_batch_multi_index
41class ModelAdapter(ABC):
42 """
43 Represents model *without* any preprocessing or postprocessing.
45 ```
46 from bioimageio.core import load_description
48 model = load_description(...)
50 # option 1:
51 adapter = create_model_adapter(model)
52 adapter.forward(...)
53 adapter.unload()
55 # option 2:
56 with create_model_adapter(model) as adapter:
57 adapter.forward(...)
58 ```
59 """
61 def __init__(
62 self,
63 model_description: AnyModelDescr,
64 devices: Sequence[str] | None,
65 computation_mode: Literal["eager", "lazy"] = "lazy",
66 ):
67 super().__init__()
68 self._model_descr = model_description
69 self._input_ids = get_member_ids(model_description.inputs)
70 self._output_ids = get_member_ids(model_description.outputs)
71 self._input_axes = [
72 tuple(a.id for a in get_axes_infos(t)) for t in model_description.inputs
73 ]
74 self._output_axes = [
75 tuple(a.id for a in get_axes_infos(t)) for t in model_description.outputs
76 ]
77 if isinstance(model_description, v0_4.ModelDescr):
78 if computation_mode == "lazy": 78 ↛ 84line 78 didn't jump to line 84 because the condition on line 78 was always true
79 warnings.warn(
80 "'lazy' computation mode is not supported for v0.4 model descriptions. Falling back to 'eager' computation mode."
81 )
82 computation_mode = "eager"
84 self._input_is_optional = [False] * len(model_description.inputs)
85 else:
86 self._input_is_optional = [ipt.optional for ipt in model_description.inputs]
88 self.computation_mode: Literal["eager", "lazy"] = computation_mode
90 self._devices = devices
91 self._loaded = False
93 def get_output_shape(
94 self,
95 input_shape: PerMember[PerAxis[int]],
96 ) -> PerMember[PerAxis[int]]:
97 if isinstance(self._model_descr, v0_4.ModelDescr): 97 ↛ 98line 97 didn't jump to line 98 because the condition on line 97 was never true
98 output_shapes = {
99 MemberId(m): {
100 AxisId(d): s if isinstance(s, int) else -1
101 for d, s in s_per_a.items()
102 }
103 for m, s_per_a in self._model_descr.get_output_tensor_sizes(
104 {
105 v0_4.TensorName(m): {str(a)[0]: s for a, s in s_per_a.items()}
106 for m, s_per_a in input_shape.items()
107 }
108 ).items()
109 }
110 else:
111 output_shapes = self._model_descr.get_output_tensor_sizes(input_shape)
112 for member, axes in get_block_transform(self._model_descr).items():
113 if (channel_transform := axes.get(AxisId("channel"))) is not None:
114 output_shapes[member][AxisId("channel")] = (
115 channel_transform
116 if isinstance(channel_transform, int)
117 else channel_transform.compute(
118 input_shape[channel_transform.member][
119 channel_transform.axis
120 ]
121 )
122 )
124 return {
125 m: {d: s if isinstance(s, int) else -1 for d, s in s_per_a.items()}
126 for m, s_per_a in output_shapes.items()
127 }
129 @property
130 def model_descr(self) -> AnyModelDescr:
131 return self._model_descr
133 @property
134 def is_loaded(self) -> bool:
135 """`True` if the model adapter is loaded and ready for inference."""
136 return self._loaded
138 @abstractmethod
139 def load(self) -> None:
140 """Load the model adapter and prepare it for inference."""
141 self._loaded = True
143 @abstractmethod
144 def forward(
145 self, inputs: PerMember[Tensor | Array | None]
146 ) -> PerMember[Tensor | None]:
147 """
148 Run forward pass of model to get model predictions
150 Args:
151 inputs: input tensors for the model, keyed by member id
152 """
154 @abstractmethod
155 def unload(self):
156 """Unload model from any devices, freeing their memory.
158 Note:
159 The model adapter should be considered unusable afterwards.
160 """
161 self._loaded = False
163 def close(self):
164 """Close the model adapter, freeing any resources.
166 Note:
167 The moder adapter should be considered unusable afterwards.
168 """
169 self.unload()
172DeviceType = TypeVar("DeviceType")
173ModelType = TypeVar("ModelType")
176@dataclass
177class _LocalModelAdapterStats:
178 forward_calls: dict[str, int] = field(
179 default_factory=cast(Callable[[], dict[str, int]], dict)
180 )
181 """forward calls per device"""
182 forward_time: dict[str, float] = field(
183 default_factory=cast(Callable[[], dict[str, float]], dict)
184 )
185 """total forward time per device"""
187 def __str__(self) -> str:
188 forward_time = ", ".join(
189 f"{device}: {time:.2g}" for device, time in self.forward_time.items()
190 )
191 forward_time_per_call = ", ".join(
192 f"{device}: {time / self.forward_calls[device]:.2g}"
193 for device, time in self.forward_time.items()
194 if device in self.forward_calls
195 )
196 return (
197 f"forward calls: {self.forward_calls}, "
198 f"forward time: {forward_time}, "
199 f"forward time per call: {forward_time_per_call}"
200 )
203class LocalModelAdapter(ModelAdapter, ABC, Generic[DeviceType, ModelType]):
204 def __init__(
205 self,
206 model_description: AnyModelDescr,
207 devices: Sequence[str] | None,
208 computation_mode: Literal["eager", "lazy"] = "lazy",
209 ):
210 super().__init__(model_description, devices, computation_mode)
211 self._stats = _LocalModelAdapterStats()
213 def load(self) -> None:
214 devices = self._devices
215 self._model_queue: LifoQueue[tuple[DeviceType, ModelType]] = LifoQueue()
216 parsed_devices = self._parse_devices(devices)
217 assert parsed_devices
218 # prioritize devices by order specified by user
219 device_exceptions: dict[str, Exception] = {}
220 self._initialized_devices: list[str] = []
221 for d in parsed_devices[::-1]:
222 try:
223 model = self._init_model_on_device(d)
224 except Exception as e:
225 device_exceptions[str(d)] = e
226 else:
227 self._model_queue.put((d, model))
228 self._initialized_devices.insert(0, str(d))
230 if self._model_queue.empty():
231 if len(device_exceptions) == 1:
232 raise next(iter(device_exceptions.values()))
233 else:
234 raise ExceptionGroup(
235 "Failed to initialize model on any of the requested devices.",
236 list(device_exceptions.values())[::-1],
237 )
239 if device_exceptions:
240 logger.warning(
241 "Failed to initialize model on some of the requested devices. Successfully initialized on {}, but got the following errors for other devices: {}",
242 self._initialized_devices,
243 device_exceptions,
244 )
246 super().load()
248 @abstractmethod
249 def _parse_devices(self, devices: Sequence[str] | None) -> Sequence[DeviceType]:
250 """Parse devices
252 Note:
253 - May not return an empty sequence.
254 - The order of devices in the returned sequence determines the priority of device usage in the forward pass.
255 First devices has highgest priority, last device has lowest priority.
256 """
258 @abstractmethod
259 def _init_model_on_device(self, device: DeviceType) -> ModelType: ...
261 def forward(
262 self, inputs: PerMember[Tensor | Array | None]
263 ) -> PerMember[Tensor | None]:
264 """
265 Run forward pass of model to get model predictions
267 Args:
268 inputs: input tensors for the model, keyed by member id
269 """
271 if not self._loaded:
272 raise RuntimeError("Model must be loaded before calling forward()")
274 unexpected = [mid for mid in inputs if mid not in self._input_ids]
275 if unexpected: 275 ↛ 276line 275 didn't jump to line 276 because the condition on line 275 was never true
276 warnings.warn(f"Got unexpected input tensor IDs: {unexpected}")
278 multi_batch_indices: dict[str, MultiIndex] = {}
279 input_arrays: list[Array | None] = []
280 for in_id, in_order in zip(self._input_ids, self._input_axes):
281 if isinstance((a := inputs.get(in_id)), Tensor): 281 ↛ 291line 281 didn't jump to line 291 because the condition on line 281 was always true
282 a = a.transpose(
283 in_order,
284 extra_dims="stack" if a.batch_multi_index is None else "raise",
285 )
286 if a.batch_multi_index is not None:
287 multi_batch_indices[in_id] = a.batch_multi_index
289 a = a.data.data
291 input_arrays.append(a)
293 logger.trace(
294 "NN input shapes: {}",
295 [a.shape if a is not None else None for a in input_arrays],
296 )
298 if self.computation_mode == "eager":
299 output_arrays = self._forward_arrays(*input_arrays)
300 logger.trace(
301 "NN output shapes: {}",
302 [a.shape if a is not None else None for a in output_arrays],
303 )
304 if len(output_arrays) > len(self._output_ids): 304 ↛ 305line 304 didn't jump to line 305 because the condition on line 304 was never true
305 warnings.warn(
306 f"Model produced more outputs ({len(output_arrays)}) than specified in the model description ({len(self._output_ids)}). Extra outputs will be ignored."
307 )
308 output_arrays = output_arrays[: len(self._output_ids)]
310 output_tensors = [
311 None if a is None else Tensor(a, dims=d)
312 for a, d in zip(output_arrays, self._output_axes)
313 ]
314 outputs = {
315 tid: out
316 for tid, out in zip(
317 self._output_ids,
318 output_tensors,
319 )
320 if out is not None
321 }
322 elif self.computation_mode == "lazy":
323 deferred_inputs = [
324 cast_and_assert(
325 Delayed,
326 delayed(a),
327 )
328 if isinstance(a, da.Array)
329 else a
330 for a in input_arrays
331 ]
333 output_arrays = cast(
334 Sequence[Delayed | None],
335 delayed(
336 self._forward_arrays,
337 pure=False,
338 nout=len(self._output_ids),
339 )(*deferred_inputs),
340 )
342 in_shape = {
343 m: {d: int(s) for d, s in zip(dims, a.shape)}
344 for m, a, dims in zip(self._input_ids, input_arrays, self._input_axes)
345 if a is not None
346 }
347 out_shape = self.get_output_shape(in_shape)
348 outputs = {
349 m: Tensor(
350 da.from_delayed(
351 a, shape=[out_shape[m][d] for d in dims], dtype=dtype
352 ),
353 dims=dims,
354 )
355 for m, a, dims, dtype in zip(
356 self._output_ids,
357 output_arrays,
358 self._output_axes,
359 [out.dtype for out in self._model_descr.outputs],
360 )
361 if a is not None
362 }
363 else:
364 assert_never(self.computation_mode)
366 if multi_batch_indices:
367 outputs = restore_batch_multi_index(multi_batch_indices.values(), outputs)
368 # Reorder unstacked output dims to match the order of the unstacked input dims, if possible
369 unstacked_output_dim_orders = get_unstacked_output_dim_orders_for_model(
370 self._model_descr,
371 {
372 m: in_tensor.dims
373 for m, in_tensor in inputs.items()
374 if isinstance(in_tensor, Tensor)
375 },
376 stacked_output_dims={
377 m: [
378 AxisId(name)
379 for name in out.batch_multi_index.names
380 if name is not None
381 ]
382 for m, out in outputs.items()
383 if out.batch_multi_index is not None
384 },
385 )
386 outputs = {
387 m: out.transpose(unstacked_output_dim_orders[m])
388 for m, out in outputs.items()
389 }
391 return outputs
393 def _forward_arrays(self, *input_arrays: Array | None):
394 if not self._loaded:
395 raise RuntimeError(
396 "Cannot compute model outputs because the model adapter is unloaded. "
397 + "Compute lazy outputs inside the prediction pipeline context "
398 + "(`with pred_pipe as pp:`), before the model is unloaded."
399 )
400 device, model = self._model_queue.get()
401 self._stats.forward_calls[str(device)] = (
402 self._stats.forward_calls.get(str(device), 0) + 1
403 )
404 start = time.perf_counter()
405 try:
406 return self._forward_impl(
407 device,
408 model,
409 [
410 cast(NDArray[Any], a.compute()) if isinstance(a, da.Array) else a
411 for a in input_arrays
412 ],
413 )
414 finally:
415 self._stats.forward_time[str(device)] = self._stats.forward_time.get(
416 str(device), 0.0
417 ) + (time.perf_counter() - start)
418 self._model_queue.put((device, model))
420 @abstractmethod
421 def _forward_impl(
422 self,
423 device: DeviceType,
424 model: ModelType,
425 input_arrays: Sequence[NDArray[Any] | None],
426 ) -> list[NDArray[Any] | None] | tuple[NDArray[Any] | None, ...]:
427 """framework specific forward implementation"""
429 def unload(self):
430 if not self.is_loaded:
431 return
433 logger.info("model adapter stats: {}", self._stats)
434 for _ in range(len(self._initialized_devices)):
435 device, model = self._model_queue.get()
436 try:
437 self._cleanup_pre_model_deletion(device, model)
438 except Exception as e:
439 logger.warning(
440 "Got error during pre-deletion cleanup on device {}: {}", device, e
441 )
442 finally:
443 del model
444 try:
445 self._cleanup_post_model_deletion(device)
446 except Exception as e:
447 logger.warning(
448 "Got error during post-deletion cleanup on device {}: {}", device, e
449 )
451 _ = gc.collect() # deallocate memory
452 super().unload()
454 @abstractmethod
455 def _cleanup_pre_model_deletion(self, device: DeviceType, model: ModelType) -> None:
456 """Clean up before model reference deletion"""
458 @abstractmethod
459 def _cleanup_post_model_deletion(self, device: DeviceType) -> None:
460 """Clean up after model reference deletion"""
463class RemoteModelAdapter(ModelAdapter, ABC, Generic[SerializedSampleBlockType]):
464 """Model adapter to use a remote service for model inference."""
466 def __init__(
467 self,
468 model_description: AnyModelDescr,
469 server: str,
470 sample_serializer: SampleSerializer[SerializedSampleBlockType],
471 ):
472 super().__init__(model_description, devices=None)
473 self._server = server
474 self._serializer = sample_serializer
476 @property
477 def server(self) -> str:
478 return self._server
480 def forward(
481 self, inputs: PerMember[Tensor | Array | None]
482 ) -> PerMember[Tensor | None]:
483 input_dims = dict(zip(self._input_ids, self._input_axes))
484 serialized_input = self._serializer.serialize_sample(
485 Sample(
486 members={
487 k: v if isinstance(v, Tensor) else Tensor(v, dims=input_dims[k])
488 for k, v in inputs.items()
489 if v is not None
490 },
491 stat={},
492 id=None,
493 )
494 )
495 serialized_output = self._forward_impl(serialized_input)
496 output = self._serializer.deserialize_sample(serialized_output).members
497 return output
499 @abstractmethod
500 def _forward_impl(
501 self, serialized_input_sample: NestedList[SerializedSampleBlockType]
502 ) -> NestedList[SerializedSampleBlockType]: ...
504 @abstractmethod
505 def test(self) -> ValidationSummary | None:
506 """Run the bioimageio model test."""