Coverage for src/bioimageio/core/_model_adapter.py: 96%

188 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 09:46 +0000

1from __future__ import annotations 

2 

3import gc 

4import time 

5import warnings 

6from abc import ABC, abstractmethod 

7from collections.abc import Callable, Sequence 

8from dataclasses import dataclass, field 

9from queue import LifoQueue 

10from typing import ( 

11 Any, 

12 Generic, 

13 Literal, 

14 cast, 

15) 

16 

17import dask.array as da 

18from dask.delayed import Delayed, delayed # pyright: ignore[reportUnknownVariableType] 

19from exceptiongroup import ExceptionGroup 

20from loguru import logger 

21from numpy.typing import NDArray 

22from typing_extensions import TypeVar, assert_never 

23 

24from bioimageio.spec import ValidationSummary 

25from bioimageio.spec.model import AnyModelDescr, v0_4 

26 

27from ._sample_serializer import SampleSerializer, SerializedSampleBlockType 

28from ._utils.type_hints import NestedList, cast_and_assert 

29from .axis import AxisId, PerAxis 

30from .common import MemberId, PerMember 

31from .digest_spec import ( 

32 get_axes_infos, 

33 get_block_transform, 

34 get_member_ids, 

35 get_unstacked_output_dim_orders_for_model, 

36) 

37from .sample import Sample 

38from .tensor import Array, MultiIndex, Tensor, restore_batch_multi_index 

39 

40 

41class ModelAdapter(ABC): 

42 """ 

43 Represents model *without* any preprocessing or postprocessing. 

44 

45 ``` 

46 from bioimageio.core import load_description 

47 

48 model = load_description(...) 

49 

50 # option 1: 

51 adapter = create_model_adapter(model) 

52 adapter.forward(...) 

53 adapter.unload() 

54 

55 # option 2: 

56 with create_model_adapter(model) as adapter: 

57 adapter.forward(...) 

58 ``` 

59 """ 

60 

61 def __init__( 

62 self, 

63 model_description: AnyModelDescr, 

64 devices: Sequence[str] | None, 

65 computation_mode: Literal["eager", "lazy"] = "lazy", 

66 ): 

67 super().__init__() 

68 self._model_descr = model_description 

69 self._input_ids = get_member_ids(model_description.inputs) 

70 self._output_ids = get_member_ids(model_description.outputs) 

71 self._input_axes = [ 

72 tuple(a.id for a in get_axes_infos(t)) for t in model_description.inputs 

73 ] 

74 self._output_axes = [ 

75 tuple(a.id for a in get_axes_infos(t)) for t in model_description.outputs 

76 ] 

77 if isinstance(model_description, v0_4.ModelDescr): 

78 if computation_mode == "lazy": 78 ↛ 84line 78 didn't jump to line 84 because the condition on line 78 was always true

79 warnings.warn( 

80 "'lazy' computation mode is not supported for v0.4 model descriptions. Falling back to 'eager' computation mode." 

81 ) 

82 computation_mode = "eager" 

83 

84 self._input_is_optional = [False] * len(model_description.inputs) 

85 else: 

86 self._input_is_optional = [ipt.optional for ipt in model_description.inputs] 

87 

88 self.computation_mode: Literal["eager", "lazy"] = computation_mode 

89 

90 self._devices = devices 

91 self._loaded = False 

92 

93 def get_output_shape( 

94 self, 

95 input_shape: PerMember[PerAxis[int]], 

96 ) -> PerMember[PerAxis[int]]: 

97 if isinstance(self._model_descr, v0_4.ModelDescr): 97 ↛ 98line 97 didn't jump to line 98 because the condition on line 97 was never true

98 output_shapes = { 

99 MemberId(m): { 

100 AxisId(d): s if isinstance(s, int) else -1 

101 for d, s in s_per_a.items() 

102 } 

103 for m, s_per_a in self._model_descr.get_output_tensor_sizes( 

104 { 

105 v0_4.TensorName(m): {str(a)[0]: s for a, s in s_per_a.items()} 

106 for m, s_per_a in input_shape.items() 

107 } 

108 ).items() 

109 } 

110 else: 

111 output_shapes = self._model_descr.get_output_tensor_sizes(input_shape) 

112 for member, axes in get_block_transform(self._model_descr).items(): 

113 if (channel_transform := axes.get(AxisId("channel"))) is not None: 

114 output_shapes[member][AxisId("channel")] = ( 

115 channel_transform 

116 if isinstance(channel_transform, int) 

117 else channel_transform.compute( 

118 input_shape[channel_transform.member][ 

119 channel_transform.axis 

120 ] 

121 ) 

122 ) 

123 

124 return { 

125 m: {d: s if isinstance(s, int) else -1 for d, s in s_per_a.items()} 

126 for m, s_per_a in output_shapes.items() 

127 } 

128 

129 @property 

130 def model_descr(self) -> AnyModelDescr: 

131 return self._model_descr 

132 

133 @property 

134 def is_loaded(self) -> bool: 

135 """`True` if the model adapter is loaded and ready for inference.""" 

136 return self._loaded 

137 

138 @abstractmethod 

139 def load(self) -> None: 

140 """Load the model adapter and prepare it for inference.""" 

141 self._loaded = True 

142 

143 @abstractmethod 

144 def forward( 

145 self, inputs: PerMember[Tensor | Array | None] 

146 ) -> PerMember[Tensor | None]: 

147 """ 

148 Run forward pass of model to get model predictions 

149 

150 Args: 

151 inputs: input tensors for the model, keyed by member id 

152 """ 

153 

154 @abstractmethod 

155 def unload(self): 

156 """Unload model from any devices, freeing their memory. 

157 

158 Note: 

159 The model adapter should be considered unusable afterwards. 

160 """ 

161 self._loaded = False 

162 

163 def close(self): 

164 """Close the model adapter, freeing any resources. 

165 

166 Note: 

167 The moder adapter should be considered unusable afterwards. 

168 """ 

169 self.unload() 

170 

171 

172DeviceType = TypeVar("DeviceType") 

173ModelType = TypeVar("ModelType") 

174 

175 

176@dataclass 

177class _LocalModelAdapterStats: 

178 forward_calls: dict[str, int] = field( 

179 default_factory=cast(Callable[[], dict[str, int]], dict) 

180 ) 

181 """forward calls per device""" 

182 forward_time: dict[str, float] = field( 

183 default_factory=cast(Callable[[], dict[str, float]], dict) 

184 ) 

185 """total forward time per device""" 

186 

187 def __str__(self) -> str: 

188 forward_time = ", ".join( 

189 f"{device}: {time:.2g}" for device, time in self.forward_time.items() 

190 ) 

191 forward_time_per_call = ", ".join( 

192 f"{device}: {time / self.forward_calls[device]:.2g}" 

193 for device, time in self.forward_time.items() 

194 if device in self.forward_calls 

195 ) 

196 return ( 

197 f"forward calls: {self.forward_calls}, " 

198 f"forward time: {forward_time}, " 

199 f"forward time per call: {forward_time_per_call}" 

200 ) 

201 

202 

203class LocalModelAdapter(ModelAdapter, ABC, Generic[DeviceType, ModelType]): 

204 def __init__( 

205 self, 

206 model_description: AnyModelDescr, 

207 devices: Sequence[str] | None, 

208 computation_mode: Literal["eager", "lazy"] = "lazy", 

209 ): 

210 super().__init__(model_description, devices, computation_mode) 

211 self._stats = _LocalModelAdapterStats() 

212 

213 def load(self) -> None: 

214 devices = self._devices 

215 self._model_queue: LifoQueue[tuple[DeviceType, ModelType]] = LifoQueue() 

216 parsed_devices = self._parse_devices(devices) 

217 assert parsed_devices 

218 # prioritize devices by order specified by user 

219 device_exceptions: dict[str, Exception] = {} 

220 self._initialized_devices: list[str] = [] 

221 for d in parsed_devices[::-1]: 

222 try: 

223 model = self._init_model_on_device(d) 

224 except Exception as e: 

225 device_exceptions[str(d)] = e 

226 else: 

227 self._model_queue.put((d, model)) 

228 self._initialized_devices.insert(0, str(d)) 

229 

230 if self._model_queue.empty(): 

231 if len(device_exceptions) == 1: 

232 raise next(iter(device_exceptions.values())) 

233 else: 

234 raise ExceptionGroup( 

235 "Failed to initialize model on any of the requested devices.", 

236 list(device_exceptions.values())[::-1], 

237 ) 

238 

239 if device_exceptions: 

240 logger.warning( 

241 "Failed to initialize model on some of the requested devices. Successfully initialized on {}, but got the following errors for other devices: {}", 

242 self._initialized_devices, 

243 device_exceptions, 

244 ) 

245 

246 super().load() 

247 

248 @abstractmethod 

249 def _parse_devices(self, devices: Sequence[str] | None) -> Sequence[DeviceType]: 

250 """Parse devices 

251 

252 Note: 

253 - May not return an empty sequence. 

254 - The order of devices in the returned sequence determines the priority of device usage in the forward pass. 

255 First devices has highgest priority, last device has lowest priority. 

256 """ 

257 

258 @abstractmethod 

259 def _init_model_on_device(self, device: DeviceType) -> ModelType: ... 

260 

261 def forward( 

262 self, inputs: PerMember[Tensor | Array | None] 

263 ) -> PerMember[Tensor | None]: 

264 """ 

265 Run forward pass of model to get model predictions 

266 

267 Args: 

268 inputs: input tensors for the model, keyed by member id 

269 """ 

270 

271 if not self._loaded: 

272 raise RuntimeError("Model must be loaded before calling forward()") 

273 

274 unexpected = [mid for mid in inputs if mid not in self._input_ids] 

275 if unexpected: 275 ↛ 276line 275 didn't jump to line 276 because the condition on line 275 was never true

276 warnings.warn(f"Got unexpected input tensor IDs: {unexpected}") 

277 

278 multi_batch_indices: dict[str, MultiIndex] = {} 

279 input_arrays: list[Array | None] = [] 

280 for in_id, in_order in zip(self._input_ids, self._input_axes): 

281 if isinstance((a := inputs.get(in_id)), Tensor): 281 ↛ 291line 281 didn't jump to line 291 because the condition on line 281 was always true

282 a = a.transpose( 

283 in_order, 

284 extra_dims="stack" if a.batch_multi_index is None else "raise", 

285 ) 

286 if a.batch_multi_index is not None: 

287 multi_batch_indices[in_id] = a.batch_multi_index 

288 

289 a = a.data.data 

290 

291 input_arrays.append(a) 

292 

293 logger.trace( 

294 "NN input shapes: {}", 

295 [a.shape if a is not None else None for a in input_arrays], 

296 ) 

297 

298 if self.computation_mode == "eager": 

299 output_arrays = self._forward_arrays(*input_arrays) 

300 logger.trace( 

301 "NN output shapes: {}", 

302 [a.shape if a is not None else None for a in output_arrays], 

303 ) 

304 if len(output_arrays) > len(self._output_ids): 304 ↛ 305line 304 didn't jump to line 305 because the condition on line 304 was never true

305 warnings.warn( 

306 f"Model produced more outputs ({len(output_arrays)}) than specified in the model description ({len(self._output_ids)}). Extra outputs will be ignored." 

307 ) 

308 output_arrays = output_arrays[: len(self._output_ids)] 

309 

310 output_tensors = [ 

311 None if a is None else Tensor(a, dims=d) 

312 for a, d in zip(output_arrays, self._output_axes) 

313 ] 

314 outputs = { 

315 tid: out 

316 for tid, out in zip( 

317 self._output_ids, 

318 output_tensors, 

319 ) 

320 if out is not None 

321 } 

322 elif self.computation_mode == "lazy": 

323 deferred_inputs = [ 

324 cast_and_assert( 

325 Delayed, 

326 delayed(a), 

327 ) 

328 if isinstance(a, da.Array) 

329 else a 

330 for a in input_arrays 

331 ] 

332 

333 output_arrays = cast( 

334 Sequence[Delayed | None], 

335 delayed( 

336 self._forward_arrays, 

337 pure=False, 

338 nout=len(self._output_ids), 

339 )(*deferred_inputs), 

340 ) 

341 

342 in_shape = { 

343 m: {d: int(s) for d, s in zip(dims, a.shape)} 

344 for m, a, dims in zip(self._input_ids, input_arrays, self._input_axes) 

345 if a is not None 

346 } 

347 out_shape = self.get_output_shape(in_shape) 

348 outputs = { 

349 m: Tensor( 

350 da.from_delayed( 

351 a, shape=[out_shape[m][d] for d in dims], dtype=dtype 

352 ), 

353 dims=dims, 

354 ) 

355 for m, a, dims, dtype in zip( 

356 self._output_ids, 

357 output_arrays, 

358 self._output_axes, 

359 [out.dtype for out in self._model_descr.outputs], 

360 ) 

361 if a is not None 

362 } 

363 else: 

364 assert_never(self.computation_mode) 

365 

366 if multi_batch_indices: 

367 outputs = restore_batch_multi_index(multi_batch_indices.values(), outputs) 

368 # Reorder unstacked output dims to match the order of the unstacked input dims, if possible 

369 unstacked_output_dim_orders = get_unstacked_output_dim_orders_for_model( 

370 self._model_descr, 

371 { 

372 m: in_tensor.dims 

373 for m, in_tensor in inputs.items() 

374 if isinstance(in_tensor, Tensor) 

375 }, 

376 stacked_output_dims={ 

377 m: [ 

378 AxisId(name) 

379 for name in out.batch_multi_index.names 

380 if name is not None 

381 ] 

382 for m, out in outputs.items() 

383 if out.batch_multi_index is not None 

384 }, 

385 ) 

386 outputs = { 

387 m: out.transpose(unstacked_output_dim_orders[m]) 

388 for m, out in outputs.items() 

389 } 

390 

391 return outputs 

392 

393 def _forward_arrays(self, *input_arrays: Array | None): 

394 if not self._loaded: 

395 raise RuntimeError( 

396 "Cannot compute model outputs because the model adapter is unloaded. " 

397 + "Compute lazy outputs inside the prediction pipeline context " 

398 + "(`with pred_pipe as pp:`), before the model is unloaded." 

399 ) 

400 device, model = self._model_queue.get() 

401 self._stats.forward_calls[str(device)] = ( 

402 self._stats.forward_calls.get(str(device), 0) + 1 

403 ) 

404 start = time.perf_counter() 

405 try: 

406 return self._forward_impl( 

407 device, 

408 model, 

409 [ 

410 cast(NDArray[Any], a.compute()) if isinstance(a, da.Array) else a 

411 for a in input_arrays 

412 ], 

413 ) 

414 finally: 

415 self._stats.forward_time[str(device)] = self._stats.forward_time.get( 

416 str(device), 0.0 

417 ) + (time.perf_counter() - start) 

418 self._model_queue.put((device, model)) 

419 

420 @abstractmethod 

421 def _forward_impl( 

422 self, 

423 device: DeviceType, 

424 model: ModelType, 

425 input_arrays: Sequence[NDArray[Any] | None], 

426 ) -> list[NDArray[Any] | None] | tuple[NDArray[Any] | None, ...]: 

427 """framework specific forward implementation""" 

428 

429 def unload(self): 

430 if not self.is_loaded: 

431 return 

432 

433 logger.info("model adapter stats: {}", self._stats) 

434 for _ in range(len(self._initialized_devices)): 

435 device, model = self._model_queue.get() 

436 try: 

437 self._cleanup_pre_model_deletion(device, model) 

438 except Exception as e: 

439 logger.warning( 

440 "Got error during pre-deletion cleanup on device {}: {}", device, e 

441 ) 

442 finally: 

443 del model 

444 try: 

445 self._cleanup_post_model_deletion(device) 

446 except Exception as e: 

447 logger.warning( 

448 "Got error during post-deletion cleanup on device {}: {}", device, e 

449 ) 

450 

451 _ = gc.collect() # deallocate memory 

452 super().unload() 

453 

454 @abstractmethod 

455 def _cleanup_pre_model_deletion(self, device: DeviceType, model: ModelType) -> None: 

456 """Clean up before model reference deletion""" 

457 

458 @abstractmethod 

459 def _cleanup_post_model_deletion(self, device: DeviceType) -> None: 

460 """Clean up after model reference deletion""" 

461 

462 

463class RemoteModelAdapter(ModelAdapter, ABC, Generic[SerializedSampleBlockType]): 

464 """Model adapter to use a remote service for model inference.""" 

465 

466 def __init__( 

467 self, 

468 model_description: AnyModelDescr, 

469 server: str, 

470 sample_serializer: SampleSerializer[SerializedSampleBlockType], 

471 ): 

472 super().__init__(model_description, devices=None) 

473 self._server = server 

474 self._serializer = sample_serializer 

475 

476 @property 

477 def server(self) -> str: 

478 return self._server 

479 

480 def forward( 

481 self, inputs: PerMember[Tensor | Array | None] 

482 ) -> PerMember[Tensor | None]: 

483 input_dims = dict(zip(self._input_ids, self._input_axes)) 

484 serialized_input = self._serializer.serialize_sample( 

485 Sample( 

486 members={ 

487 k: v if isinstance(v, Tensor) else Tensor(v, dims=input_dims[k]) 

488 for k, v in inputs.items() 

489 if v is not None 

490 }, 

491 stat={}, 

492 id=None, 

493 ) 

494 ) 

495 serialized_output = self._forward_impl(serialized_input) 

496 output = self._serializer.deserialize_sample(serialized_output).members 

497 return output 

498 

499 @abstractmethod 

500 def _forward_impl( 

501 self, serialized_input_sample: NestedList[SerializedSampleBlockType] 

502 ) -> NestedList[SerializedSampleBlockType]: ... 

503 

504 @abstractmethod 

505 def test(self) -> ValidationSummary | None: 

506 """Run the bioimageio model test."""