Coverage for src/bioimageio/core/block.py: 88%

324 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 09:46 +0000

1from __future__ import annotations 

2 

3import itertools 

4from collections.abc import Callable, Mapping, Sequence 

5from copy import deepcopy 

6from dataclasses import dataclass 

7from functools import partial 

8from math import ceil, floor, prod 

9from types import MappingProxyType 

10from typing import ( 

11 Annotated, 

12 Generic, 

13 Literal, 

14 TypeVar, 

15 cast, 

16) 

17 

18import dask.array as da 

19import numpy as np 

20from loguru import logger 

21from numpy.typing import NDArray 

22from typing_extensions import assert_never 

23 

24from ._axis_annotations import PerAxisAnno 

25from ._settings import settings 

26from ._utils.type_hints import NestedList, cast_and_assert 

27from .axis import TIME_AXIS_ID, AxisId, AxisSize, PerAxis 

28from .common import ( 

29 BlockLoc, 

30 Halo, 

31 HaloLike, 

32 MemberId, 

33 PadMode, 

34 PadWidth, 

35 PerMember, 

36 SliceInfo, 

37 SliceInfoWithOverlap, 

38) 

39from .tensor import Array, Tensor 

40 

41 

42@dataclass 

43class LinearAxisTransform: 

44 member: MemberId 

45 axis: AxisId 

46 scale: float 

47 offset: int 

48 

49 def compute(self, s: int, round: Callable[[float], int] = floor) -> int: 

50 return round(s * self.scale) + self.offset 

51 

52 

53@dataclass(frozen=True, slots=True) 

54class BlockMeta: 

55 """A block of data within a sample 

56 

57 Figure for illustration: 

58 The first 2d block (dashed) of a sample member (**bold**). 

59 The inner slice (thin) is expanded by a halo in both dimensions on both sides. 

60 The outer slice reaches from the sample member origin (0, 0) to the right halo point. 

61 

62 ``` 

63 first block (at the sample origin) 

64 ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─┐ 

65 ╷ halo(left) ╷ 

66 ╷ padding outside the sample ╷ 

67 ╷ (0, 0)┏━━━━━━━━━━━━━━━━━┯━━━━━━━━━┯━━━➔ 

68 ╷ ┃ │ ╷ sample member 

69 ╷ ┃ inner │ outer ╷ 

70 ╷ ┃ region │ region ╷ 

71 ╷ ┃ /slice │ /slice ╷ 

72 ╷ ┃ │ ╷ 

73 ╷ ┣─────────────────┘ ╷ 

74 ╷ ┃ outer region/slice ╷ 

75 ╷ ┃ halo(right) ╷ 

76 └ ─ ─ ─ ─┃─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─┘ 

77 ⬇ 

78 ``` 

79 

80 Note: 

81 - Inner and outer slices are specified in sample member coordinates. 

82 - The outer_slice of a block at the sample edge may overlap by more than the 

83 halo with the neighboring block (the inner slices will not overlap though). 

84 

85 """ 

86 

87 parent: PerAxis[int] | Tensor 

88 """the full tensor this block is a part of, or its shape if the full tensor is not available""" 

89 

90 inner_slice: PerAxisAnno[SliceInfo | SliceInfoWithOverlap] 

91 """inner region (without halo) wrt the full tensor""" 

92 

93 halo: PerAxisAnno[Halo] 

94 """halo enlarging the inner region to the block's sizes""" 

95 

96 # block_loc: BlockLoc 

97 # """the block index within the full tensor""" 

98 

99 # block_loc_shape: BlockLocShape # TODO: rename block_loc_shape 

100 # """The full tensor shape in numbers of blocks""" 

101 

102 @property 

103 def full_shape(self) -> PerAxis[int]: 

104 """shape of the full tensor this block is a part of""" 

105 if isinstance(self.parent, Tensor): 105 ↛ 108line 105 didn't jump to line 108 because the condition on line 105 was always true

106 return self.parent.tagged_shape 

107 else: 

108 return self.parent 

109 

110 @property 

111 def shape(self) -> PerAxis[int]: 

112 """shape of the block (including halo)""" 

113 return MappingProxyType( 

114 { 

115 a: s.stop - s.start + (sum(self.halo[a]) if a in self.halo else 0) 

116 for a, s in self.inner_slice.items() 

117 } 

118 ) 

119 

120 @property 

121 def padding(self) -> PerAxis[PadWidth]: 

122 """padding to realize the halo at the sample edge 

123 where we cannot simply enlarge the inner slice""" 

124 return MappingProxyType( 

125 { 

126 a: p 

127 for a in self.inner_slice 

128 if ( 

129 p := PadWidth( 

130 ( 

131 self.halo[a].left 

132 - (self.inner_slice[a].start - self.outer_slice[a].start) 

133 if a in self.halo 

134 else 0 

135 ), 

136 ( 

137 self.halo[a].right 

138 - (self.outer_slice[a].stop - self.inner_slice[a].stop) 

139 if a in self.halo 

140 else 0 

141 ), 

142 ) 

143 ) 

144 != PadWidth(0, 0) 

145 } 

146 ) 

147 

148 @property 

149 def outer_slice(self) -> PerAxis[SliceInfo]: 

150 """slice of the outer block (including halo, excluding padding) wrt the sample""" 

151 return MappingProxyType( 

152 { 

153 a: SliceInfo( 

154 max( 

155 0, 

156 self.inner_slice[a].start 

157 - (self.halo[a].left if a in self.halo else 0), 

158 ), 

159 min( 

160 self.full_shape[a], 

161 self.inner_slice[a].stop 

162 + (self.halo[a].right if a in self.halo else 0), 

163 ), 

164 ) 

165 for a in self.inner_slice 

166 } 

167 ) 

168 

169 @property 

170 def inner_shape(self) -> PerAxis[int]: 

171 """axis lengths of the inner region (without halo)""" 

172 return MappingProxyType( 

173 {a: s.stop - s.start for a, s in self.inner_slice.items()} 

174 ) 

175 

176 @property 

177 def local_slice(self) -> PerAxis[SliceInfo]: 

178 """inner slice wrt the block, **not** the sample""" 

179 return MappingProxyType( 

180 { 

181 a: ( 

182 SliceInfo( 

183 h.left, 

184 h.left + self.inner_shape[a], 

185 ) 

186 if (h := self.halo.get(a)) is not None 

187 else SliceInfo(0, self.inner_shape[a]) 

188 ) 

189 for a in self.inner_slice 

190 } 

191 ) 

192 

193 @property 

194 def dims(self) -> tuple[AxisId, ...]: 

195 return tuple(self.inner_shape) 

196 

197 @property 

198 def tagged_shape(self) -> PerAxis[int]: 

199 """alias for shape""" 

200 return self.shape 

201 

202 @property 

203 def inner_slice_wo_overlap(self) -> PerAxis[SliceInfo]: 

204 """subslice of the inner slice, such that all `inner_slice_wo_overlap` can be 

205 stiched together trivially to form the original sample. 

206 

207 This can also be used to calculate statistics 

208 without overrepresenting block edge regions.""" 

209 return { 

210 a: SliceInfo(s.start_wo_overlap, s.stop_wo_overlap) 

211 if isinstance(s, SliceInfoWithOverlap) 

212 else SliceInfo(s.start, s.stop) 

213 for a, s in self.inner_slice.items() 

214 } 

215 

216 @property 

217 def inner_shape_wo_overlap(self) -> PerAxis[int]: 

218 """axis lengths of the inner region (without halo and without overlap)""" 

219 return MappingProxyType( 

220 {a: s.stop - s.start for a, s in self.inner_slice_wo_overlap.items()} 

221 ) 

222 

223 def __post_init__(self): 

224 assert all(a in self.full_shape for a in self.inner_slice), ( 

225 "block has axes not present in its full tensor shape" 

226 ) 

227 

228 assert all(a in self.inner_slice for a in self.halo), ( 

229 "halo has axes not present in block" 

230 ) 

231 

232 if any( 232 ↛ 236line 232 didn't jump to line 236 because the condition on line 232 was never true

233 s > self.full_shape[a] + sum(self.padding.get(a, PadWidth(0, 0))) 

234 for a, s in self.shape.items() 

235 ): 

236 logger.warning( 

237 "block {} larger than its full tensor shape {}", 

238 self.shape, 

239 self.full_shape, 

240 ) 

241 

242 # def get_transformed(self, new_axes: PerAxis[LinearAxisTransform | int]) -> Self: 

243 # assert BATCH_AXIS_ID not in new_axes, "cannot transform the batch axis" 

244 

245 # full_shape = { 

246 # a: (trf if isinstance(trf, int) else trf.compute(self.full_shape[trf.axis])) 

247 # for a, trf in new_axes.items() 

248 # } 

249 # assert full_shape.get(BATCH_AXIS_ID) == self.full_shape.get(BATCH_AXIS_ID), ( 

250 # "block transform must not change the batch axis size" 

251 # ) 

252 

253 # inner_slice = { 

254 # a: ( 

255 # SliceInfo(0, trf) 

256 # if isinstance(trf, int) 

257 # else SliceInfo( 

258 # trf.compute(self.inner_slice[trf.axis].start), 

259 # trf.compute(self.inner_slice[trf.axis].stop), 

260 # ) 

261 # ) 

262 # for a, trf in new_axes.items() 

263 # } 

264 # assert inner_slice.get(BATCH_AXIS_ID) == self.inner_slice.get(BATCH_AXIS_ID), ( 

265 # "block transform must not change the batch axis slice" 

266 # ) 

267 

268 # return self.__class__( 

269 # parent=full_shape, 

270 # inner_slice=inner_slice, 

271 # halo={ 

272 # a: ( 

273 # Halo(0, 0) 

274 # if isinstance(trf, int) 

275 # else Halo(self.halo[trf.axis].left, self.halo[trf.axis].right) 

276 # ) 

277 # for a, trf in new_axes.items() 

278 # }, 

279 # ) 

280 

281 # def get_transformed( 

282 # self, transformed_tensor: Tensor, new_axes: PerAxis[LinearAxisTransform | int] 

283 # ) -> Self: 

284 # assert BATCH_AXIS_ID not in new_axes, "cannot transform the batch axis" 

285 

286 # full_shape = { 

287 # a: (trf if isinstance(trf, int) else trf.compute(self.full_shape[trf.axis])) 

288 # for a, trf in new_axes.items() 

289 # } 

290 # assert full_shape.get(BATCH_AXIS_ID) == self.full_shape.get(BATCH_AXIS_ID), ( 

291 # "block transform must not change the batch axis size" 

292 # ) 

293 

294 # inner_slice = { 

295 # a: ( 

296 # SliceInfo(0, trf) 

297 # if isinstance(trf, int) 

298 # else SliceInfo( 

299 # trf.compute(self.inner_slice[trf.axis].start), 

300 # trf.compute(self.inner_slice[trf.axis].stop), 

301 # ) 

302 # ) 

303 # for a, trf in new_axes.items() 

304 # } 

305 # assert inner_slice.get(BATCH_AXIS_ID) == self.inner_slice.get(BATCH_AXIS_ID), ( 

306 # "block transform must not change the batch axis slice" 

307 # ) 

308 

309 # return self.__class__( 

310 # full_tensor=transformed_tensor, 

311 # inner_slice=inner_slice, 

312 # halo={ 

313 # a: ( 

314 # Halo(0, 0) 

315 # if isinstance(trf, int) 

316 # else Halo(self.halo[trf.axis].left, self.halo[trf.axis].right) 

317 # ) 

318 # for a, trf in new_axes.items() 

319 # }, 

320 # block_loc=self.block_loc, 

321 # block_loc_shape=self.block_loc_shape, 

322 # ) 

323 

324 

325@dataclass(frozen=True, slots=True) 

326class Block(BlockMeta): 

327 """A block of data within a sample 

328 

329 Figure for illustration: 

330 The first 2d block (dashed) of a sample member (**bold**). 

331 The inner slice (thin) is expanded by a halo in both dimensions on both sides. 

332 The outer slice reaches from the sample member origin (0, 0) to the right halo point. 

333 

334 ``` 

335 first block (at the sample origin) 

336 ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─┐ 

337 ╷ halo(left) ╷ 

338 ╷ padding outside the sample ╷ 

339 ╷ (0, 0)┏━━━━━━━━━━━━━━━━━┯━━━━━━━━━┯━━━➔ 

340 ╷ ┃ │ ╷ sample member 

341 ╷ ┃ inner │ outer ╷ 

342 ╷ ┃ region │ region ╷ 

343 ╷ ┃ /slice │ /slice ╷ 

344 ╷ ┃ │ ╷ 

345 ╷ ┣─────────────────┘ ╷ 

346 ╷ ┃ outer region/slice ╷ 

347 ╷ ┃ halo(right) ╷ 

348 └ ─ ─ ─ ─┃─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─┘ 

349 ⬇ 

350 ``` 

351 

352 Note: 

353 - Inner and outer slices are specified in sample member coordinates. 

354 - The outer_slice of a block at the sample edge may overlap by more than the 

355 halo with the neighboring block (the inner slices will not overlap though). 

356 

357 """ 

358 

359 parent: Tensor 

360 """the full tensor this block is a part of""" 

361 

362 pad_mode: PadMode 

363 """The padding mode to use when the halo extends beyond the tensor boundary.""" 

364 

365 @property 

366 def outer_tensor(self) -> Tensor: 

367 """the outer tensor of the block (including halo and possibly padding)""" 

368 outer_tensor = self.parent[self.outer_slice] 

369 if self.padding: 

370 outer_tensor = outer_tensor.pad(self.padding, mode=self.pad_mode) 

371 

372 return outer_tensor 

373 

374 

375_GridItem = TypeVar("_GridItem") 

376 

377 

378@dataclass(frozen=True, slots=True) 

379class NestedListWithNamedDims(Generic[_GridItem]): 

380 """A nested list of values with names for each nesting dimension. 

381 

382 `dims` maps each nested-list level in `values` to an axis. 

383 """ 

384 

385 dims: tuple[AxisId, ...] 

386 """Axis corresponding to each nested-list level in `values`.""" 

387 

388 values: NestedList[_GridItem] 

389 """Values arranged in a rectangular nested list whose dimensions are named by `dims`.""" 

390 

391 

392BlockArray = Annotated[NDArray[np.object_], "NDArray[Block]"] 

393"""An n-dimensional object array of tensor `Block`s. 

394 

395Unlike `NestedListWithNamedDims`, its Block elements have the same dimensionality as the overall array. 

396 

397I.e. all array dimensions are explicitly present in the `Block` objects, even if they have length 1. 

398This is in contrast to a `NestedListWithNamedDims[Blocks]`, where each block has the same dimensions, 

399but the nested list structure may skip singleton dimensions and add additional dimensions. 

400""" 

401 

402 

403def split_tensor_into_blocks( 

404 tensor: Tensor, 

405 *, 

406 block_shape: PerAxis[int | AxisSize], 

407 halo: PerAxis[HaloLike], 

408 pad_mode: PadMode, 

409 pad_boundary: Literal["allow", True, False] = True, 

410) -> BlockArray: 

411 """Split a tensor, optionally padding single blocks to a valid axis size. 

412 

413 Args: 

414 tensor: the tensor to split into blocks 

415 block_shape: the block shape, omitted axes are treated as having a block size equal to the tensor axis size. 

416 To allow for variable block sizes along a given axis, 

417 the block size can be specified as an `AxisSize` with a `min` `max` and `step` value. 

418 halo: the halo around each block, omitted axes are treated as having a halo of 0. 

419 pad_mode: the padding mode to use when the halo extends beyond the tensor boundary. 

420 pad_boundary: 

421 True: pad boundary halos, possibly more to make a single block valid; 

422 False: forbid boundary padding 

423 "allow": pad only to make a single block valid, no to account for boundary halos. 

424 """ 

425 if unknown_dims := [ 425 ↛ 428line 425 didn't jump to line 428 because the condition on line 425 was never true

426 a for a in block_shape if a not in tensor.dims and block_shape[a] != 1 

427 ]: 

428 raise ValueError( 

429 f"unknown dims in block_shape: {unknown_dims} for tensor {tensor.dims}" 

430 ) 

431 

432 if unknown_halo_dims := [a for a in halo if a not in tensor.dims]: 432 ↛ 433line 432 didn't jump to line 433 because the condition on line 432 was never true

433 raise ValueError( 

434 f"unknown dims in halo: {unknown_halo_dims} for tensor {tensor.dims}" 

435 ) 

436 

437 # fill in default halo (0) 

438 halo = {d: Halo.create(halo.get(d, 0)) for d in tensor.dims} 

439 

440 main_block_shape: dict[AxisId, int] = {} 

441 single_block_dims: set[AxisId] = set() 

442 for d, s in tensor.tagged_shape.items(): 

443 outer_size = s + (sum(halo[d]) if pad_boundary is True else 0) 

444 bs = block_shape.get(d, outer_size) 

445 if isinstance(bs, AxisSize): 

446 step = bs.step or 1 

447 padded_size = bs.min + max(0, ceil((outer_size - bs.min) / step)) * step 

448 limit = min(outer_size, bs.max or outer_size) 

449 main_block_shape[d] = limit - (limit - bs.min) % step 

450 fits_single_block = bs.max is None or padded_size <= bs.max 

451 else: 

452 padded_size = bs 

453 main_block_shape[d] = min(bs, outer_size) if pad_boundary is False else bs 

454 fits_single_block = outer_size <= bs 

455 

456 if pad_boundary is not False and fits_single_block: 

457 main_block_shape[d] = padded_size 

458 single_block_dims.add(d) 

459 

460 if large_block_dims := { 

461 a 

462 for a, bs in block_shape.items() 

463 if main_block_shape.get(a, 1) < (bs.min if isinstance(bs, AxisSize) else bs) 

464 }: 

465 raise ValueError( 

466 f"Block shape {block_shape} larger than tensor shape + halo {main_block_shape} along {large_block_dims}.", 

467 ) 

468 

469 inner_1d_slices: dict[AxisId, list[SliceInfo | SliceInfoWithOverlap]] = {} 

470 inner_sizes = {d: main_block_shape[d] - sum(halo[d]) for d in tensor.dims} 

471 for d in tensor.dims: 

472 inner_first_start = halo[d].left if pad_boundary is not True else 0 

473 inner_last_stop = tensor.tagged_shape[d] - ( 

474 halo[d].right if pad_boundary is not True else 0 

475 ) 

476 inner_size = inner_sizes[d] 

477 if inner_size <= 0: 477 ↛ 478line 477 didn't jump to line 478 because the condition on line 477 was never true

478 raise ValueError(f"Block size must be larger than halo along {d}") 

479 if d in single_block_dims: 

480 padded_stop = inner_first_start + inner_size 

481 inner_1d_slices[d] = [ 

482 SliceInfo(inner_first_start, inner_last_stop) 

483 if padded_stop == inner_last_stop 

484 else SliceInfoWithOverlap( 

485 inner_first_start, padded_stop, inner_first_start, inner_last_stop 

486 ) 

487 ] 

488 continue 

489 

490 outer_size = block_shape.get(d, tensor.tagged_shape[d] + sum(halo[d])) 

491 if isinstance(outer_size, int): 

492 outer_size = AxisSize(min=outer_size) 

493 

494 outer_min = outer_size.min 

495 step = outer_size.step or 1 

496 

497 inner_1d_slices[d] = [] 

498 for start in range(inner_first_start, inner_last_stop, inner_size): 

499 if start + inner_size <= inner_last_stop: 

500 inner_1d_slices[d].append(SliceInfo(start, start + inner_size)) 

501 else: 

502 inner_remainder = inner_last_stop - start 

503 outer_remainder = inner_remainder + sum(halo[d]) 

504 if ( 

505 outer_remainder >= outer_min 

506 and (outer_remainder - outer_min) % step == 0 

507 ): 

508 # last block has valid shape 

509 inner_1d_slices[d].append(SliceInfo(start, inner_last_stop)) 

510 else: 

511 # last block needs to overlap to have a valid shape 

512 minimum_outer_remainder = outer_min + ( 

513 step 

514 * max( 

515 0, 

516 ceil((outer_remainder - outer_min) / step), 

517 ) 

518 ) 

519 valid_inner_start = inner_last_stop - ( 

520 minimum_outer_remainder - sum(halo[d]) 

521 ) 

522 inner_1d_slices[d].append( 

523 SliceInfoWithOverlap( 

524 valid_inner_start, inner_last_stop, start, inner_last_stop 

525 ) 

526 ) 

527 

528 out_shape = tuple(len(slices) for slices in inner_1d_slices.values()) 

529 out: BlockArray = np.empty(out_shape, dtype=object) 

530 for idx in itertools.product(*(range(s) for s in out_shape)): 

531 block_loc: BlockLoc = idx 

532 inner_slice = {d: inner_1d_slices[d][i] for d, i in zip(tensor.dims, block_loc)} 

533 out[idx] = Block( 

534 parent=tensor, inner_slice=inner_slice, halo=halo, pad_mode=pad_mode 

535 ) 

536 

537 return out 

538 

539 

540def split_multiple_tensors_into_blocks( 

541 tensors: PerMember[Tensor], 

542 block_shape: PerMember[PerAxis[int | AxisSize]], 

543 *, 

544 halo: PerMember[PerAxis[HaloLike]], 

545 broadcast: bool = False, 

546 pad_mode: PadMode | PerMember[PadMode], 

547 pad_boundary: Literal["allow", True, False] = True, 

548) -> dict[MemberId, BlockArray]: 

549 """Split multiple tensors (e.g. sample members) into aligned blocks. 

550 

551 Args: 

552 tensors: the tensors to split into blocks 

553 block_shape: the block shape per sample member (depending on `broaadcast` the block shapes must result in the same number of blocks for all tensors). 

554 Omitted axes are treated as having a block size equal to the tensor axis size. 

555 To allow for variable block sizes along a given axis, the block size can be specified as an `AxisSize` with a `min` `max` and `step` value. 

556 Any such parameterized axis may result in blocks overlapping along that axis, if the tensor shape does not divide by the maximum block size with a valid remainder. 

557 halo: the halo around each block per sample member, omitted axes are treated as having a halo of 0. 

558 broadcast: whether to allow broadcasting the resulting tensor blocks to each other, e.g. if one tensor does not have a dimension along which another is blocked up. 

559 pad_mode: the padding mode (optionally per sample member) to use when the halo extends beyond the tensor boundary. 

560 pad_boundary: 

561 True: pad boundary halos, possibly more to make a single block valid; 

562 False: forbid boundary padding 

563 "allow": pad only to make a single block valid, no to account for boundary halos. 

564 Raises: 

565 ValueError: If the tensor shape cannot be divided into blocks according to `block_shape`. 

566 """ 

567 

568 if unknown_blocks := [m for m in block_shape if m not in tensors]: 568 ↛ 569line 568 didn't jump to line 569 because the condition on line 568 was never true

569 raise ValueError( 

570 f"block shape specified for unknown tensors: {unknown_blocks}." 

571 ) 

572 

573 if not block_shape: 573 ↛ 574line 573 didn't jump to line 574 because the condition on line 573 was never true

574 block_shape = {m: t.tagged_shape for m, t in tensors.items()} 

575 

576 if not broadcast and ( 576 ↛ 579line 576 didn't jump to line 579 because the condition on line 576 was never true

577 missing_blocks := [m for m in tensors if m not in block_shape] 

578 ): 

579 raise ValueError( 

580 f"no block shape specified for {missing_blocks}." 

581 + " Set `broadcast` to True if these tensors should be repeated" 

582 + " as a whole for each block." 

583 ) 

584 

585 if extra_halo := [m for m in halo if m not in block_shape]: 585 ↛ 586line 585 didn't jump to line 586 because the condition on line 585 was never true

586 raise ValueError( 

587 f"`halo` specified for tensors without block shape: {extra_halo}." 

588 ) 

589 

590 blocks: PerMember[BlockArray] = {} 

591 for m in block_shape: 

592 blocks[m] = split_tensor_into_blocks( 

593 tensor=tensors[m], 

594 block_shape=block_shape[m], 

595 halo=halo.get(m, {}), 

596 pad_mode=pad_mode.get(m, "symmetric") 

597 if isinstance(pad_mode, Mapping) 

598 else pad_mode, 

599 pad_boundary=pad_boundary, 

600 ) 

601 

602 assert len(blocks) > 0, blocks 

603 unique_block_ndims = {b.shape for b in blocks.values()} 

604 

605 if len(unique_block_ndims) != 1: 605 ↛ 606line 605 didn't jump to line 606 because the condition on line 605 was never true

606 if not broadcast: 

607 raise ValueError( 

608 f"Mismatch for block layout across members: { {t: b.shape for t, b in blocks.items()} }. Set `broadcast` to True if this is expected." 

609 ) 

610 

611 blocks = dict(zip(blocks.keys(), np.broadcast_arrays(*blocks.values()))) 

612 block_locs = {m: b.shape for m, b in blocks.items()} 

613 unique_block_locs = set(block_locs.values()) 

614 assert len(unique_block_locs) == 1, ( 

615 f"Mismatch for total number of blocks (after braodcasting?!): {block_locs}" 

616 ) 

617 

618 return blocks 

619 

620 

621def apply_blockwise( 

622 func: Callable[ 

623 [PerMember[Tensor]], 

624 PerMember[Tensor | None], # TODO: return non-optional outputs only? 

625 ], 

626 inputs: PerMember[Tensor], 

627 *, 

628 block_shape: PerMember[PerAxis[int | AxisSize]], 

629 pad_mode: PadMode | PerMember[PadMode], 

630 pad_boundary: Literal["allow", True, False] = True, 

631 input_halo: PerMember[PerAxis[Halo]], 

632 output_halo: PerMember[PerAxis[Halo]], 

633 get_output_shape: Callable[[PerMember[PerAxis[int]]], PerMember[PerAxis[int]]], 

634 chunk_shape: PerMember[PerAxis[int]] | int = settings.processing_chunk_min_bytes, 

635 blockwise_processing_mode: Literal["parallel", "sequential", "auto"] = "auto", 

636) -> dict[MemberId, Tensor]: 

637 """Apply `func` to `inputs` blockwise. 

638 

639 

640 Args: 

641 func: the function to apply to each block 

642 inputs: the input tensors to process 

643 block_shape: 

644 The shape of the blocks to be processed by func. 

645 May be expressed using `AxisSize` objects to allow for variable block shapes. 

646 Missing axes are treated as having a block size equal to the tensor axis size. 

647 pad_mode: 

648 The padding mode (optionally per input member) to use when the halo extends beyond the tensor boundary 

649 pad_boundary: 

650 True: pad boundary halos, possibly more to make a single block valid; 

651 False: forbid boundary padding 

652 "allow": pad only to make a single block valid, no to account for boundary halos. 

653 input_halo: the halo around each input block (e.g. to compenstate for `output_halo` such that cropped output blocks can be stitched together) 

654 Within the sample halo is realized as overlap between neighboring blocks, outside the sample halo is realized as padding. 

655 output_halo: the halo to be cropped from each output block 

656 get_output_shape: a function to compute the output shape based on the input shape. 

657 May return values <1 for output axes without halo, which means the output will be concatenated along those axes. 

658 (Only used if the input shape does not evenly divide by the block shape to correct overlapping outputs.) 

659 chunk_shape: the shape of the chunks to process. 

660 Chunks are larger than blocks and help to reduce the number of tasks in the dask graph for 'parallel' processing. 

661 Must be larger than `block_shape`. 

662 If an integer is given, it is interpreted as the minimum number of bytes of any input chunk tensor. 

663 blockwise_processing_mode: (WIP) processing mode wihin a (parallelly processed) dask chunk. 

664 Options are 

665 - 'parallel': nest blockwise tasks inside the chunk task) 

666 - 'sequential': (not yet implemented) full chunk is a single task 

667 - 'auto': automatically choose between 'parallel' and 'sequential' based on the number of chunks to avoid large task graphs. 

668 """ 

669 # add missing block dims explicitly 

670 block_shape = { 

671 m: {d: block_shape.get(m, {}).get(d, t.tagged_shape[d]) for d in t.dims} 

672 for m, t in inputs.items() 

673 } 

674 

675 max_block_shape: dict[MemberId, dict[AxisId, int]] = {} 

676 for m, b in block_shape.items(): 

677 max_block_shape[m] = {} 

678 for d, bs in b.items(): 

679 if isinstance(bs, AxisSize): 

680 max_tensor_size = inputs[m].tagged_shape.get(d, 1) 

681 if pad_boundary is True: 

682 max_tensor_size += sum(input_halo.get(m, {}).get(d, Halo(0, 0))) 

683 effective_bs = max( 

684 bs.min, 

685 max_tensor_size if bs.max is None else min(bs.max, max_tensor_size), 

686 ) 

687 if ( 

688 not isinstance(chunk_shape, int) 

689 and m in chunk_shape 

690 and d in chunk_shape[m] 

691 ): 

692 effective_bs = min(effective_bs, chunk_shape[m][d]) 

693 else: 

694 effective_bs = bs 

695 

696 max_block_shape[m][d] = effective_bs 

697 

698 if isinstance(chunk_shape, int): 

699 chunk_shape = get_chunk_shape( 

700 inputs, 

701 block_shape=max_block_shape, 

702 chunk_min_bytes=chunk_shape, 

703 ) 

704 

705 if any( 705 ↛ 711line 705 didn't jump to line 711 because the condition on line 705 was never true

706 max_block_shape[m][d] > chunk_shape[m][d] 

707 for m in chunk_shape 

708 for d in chunk_shape[m] 

709 if m in max_block_shape and d in max_block_shape[m] 

710 ): 

711 raise ValueError( 

712 f"block_shape {block_shape} must not be larger than chunk_shape {chunk_shape}" 

713 ) 

714 

715 if blockwise_processing_mode == "auto": 

716 n_input_chunks = max( 

717 prod( 

718 ceil( 

719 inputs[m].tagged_shape[d] 

720 / ( 

721 bs 

722 if isinstance((bs := block_shape[m][d]), int) 

723 else bs.max or bs.min 

724 ) 

725 ) 

726 for d in block_shape[m] 

727 ) 

728 for m in block_shape 

729 ) 

730 if n_input_chunks > 100: 730 ↛ 731line 730 didn't jump to line 731 because the condition on line 730 was never true

731 blockwise_processing_mode = "sequential" 

732 else: 

733 blockwise_processing_mode = "parallel" 

734 

735 logger.info( 

736 "using blockwise_processing_mode='{}' within each of the {} input chunks", 

737 blockwise_processing_mode, 

738 n_input_chunks, 

739 ) 

740 

741 process_chunk = partial( 

742 _apply_blockwise_impl, 

743 func, 

744 block_shape=block_shape, 

745 pad_mode=pad_mode, 

746 pad_boundary="allow" if pad_boundary is not False else False, 

747 input_halo=input_halo, 

748 output_halo=output_halo, 

749 processing_mode=blockwise_processing_mode, 

750 get_output_shape=get_output_shape, 

751 ) 

752 

753 return _apply_blockwise_impl( 

754 process_chunk, 

755 inputs, 

756 block_shape=chunk_shape, 

757 pad_mode=pad_mode, 

758 pad_boundary=pad_boundary, 

759 input_halo=input_halo, 

760 output_halo=output_halo, 

761 processing_mode="parallel", 

762 get_output_shape=get_output_shape, 

763 ) 

764 

765 

766def _apply_blockwise_impl( 

767 func: Callable[[PerMember[Tensor]], PerMember[Tensor | None]], 

768 inputs: PerMember[Tensor], 

769 *, 

770 block_shape: PerMember[PerAxis[int | AxisSize]], 

771 pad_mode: PadMode | PerMember[PadMode], 

772 pad_boundary: Literal["allow", True, False] = True, 

773 input_halo: PerMember[PerAxis[HaloLike]], 

774 output_halo: PerMember[PerAxis[HaloLike]], 

775 get_output_shape: Callable[[PerMember[PerAxis[int]]], PerMember[PerAxis[int]]], 

776 processing_mode: Literal["parallel", "sequential"] = "parallel", 

777) -> dict[MemberId, Tensor]: 

778 """ 

779 Apply a function to a mapping (dict) of tensors, processing them in blocks. 

780 

781 Args: 

782 func: the function to apply to each block 

783 inputs: the input tensors to process 

784 block_shape: the shape of the blocks to be processed by func. May be expressed using `AxisSize` objects to allow for variable block shapes. 

785 pad_mode: the padding mode (optionally per input member) to use when the halo extends beyond the tensor boundary 

786 pad_boundary: 

787 True: pad boundary halos, possibly more to make a single block valid; 

788 False: forbid boundary padding 

789 "allow": pad only to make a single block valid, no to account for boundary halos. 

790 input_halo: the halo around each input block (e.g. to compenstate for `output_halo` such that cropped output blocks can be stitched together) 

791 Within the sample halo is realized as overlap between neighboring blocks, outside the sample halo is realized as padding. 

792 output_halo: the halo to be cropped from each output block 

793 get_output_shape: a function to compute the output shape based on the input shape 

794 (only used if the input shape does not evenly divide by the block shape to correct overlapping outputs) 

795 processing_mode: the mode of blockwise processing ('parallel' (via dask graph) or 'sequential') 

796 """ 

797 input_blocks = split_multiple_tensors_into_blocks( 

798 inputs, 

799 block_shape=block_shape, 

800 halo=input_halo, 

801 broadcast=True, 

802 pad_mode=pad_mode, 

803 pad_boundary=pad_boundary, 

804 ) 

805 

806 return apply_to_block_array( 

807 func, 

808 input_blocks, 

809 output_halo=output_halo, 

810 keep_output_halo_at_sample_edge=pad_boundary is not True, 

811 processing_mode=processing_mode, 

812 get_output_shape=get_output_shape, 

813 ) 

814 

815 

816def apply_to_block_array( 

817 func: Callable[[PerMember[Tensor]], PerMember[Tensor | None]], 

818 input_blocks: PerMember[BlockArray], 

819 *, 

820 output_halo: PerMember[PerAxis[HaloLike]], 

821 keep_output_halo_at_sample_edge: bool, 

822 processing_mode: Literal["parallel", "sequential"] = "parallel", 

823 get_output_shape: Callable[[PerMember[PerAxis[int]]], PerMember[PerAxis[int]]], 

824) -> dict[MemberId, Tensor]: 

825 

826 output_grid = apply_to_blocks( 

827 func, 

828 input_blocks, 

829 output_halo=output_halo, 

830 get_output_shape=get_output_shape, 

831 keep_output_halo_at_sample_edge=keep_output_halo_at_sample_edge, 

832 ) 

833 nested_outputs = output_grid.values 

834 

835 def select_arrays( 

836 nested: NestedList[dict[MemberId, Tensor]], member: MemberId 

837 ) -> NestedList[Array]: 

838 if isinstance(nested, list): 

839 return [select_arrays(item, member) for item in nested] 

840 else: 

841 return nested[member].data.data 

842 

843 first_block = nested_outputs 

844 while isinstance(first_block, list): 

845 first_block = first_block[0] 

846 

847 if processing_mode == "parallel": 

848 concatenate = cast(Callable[[Sequence[Array], int], Array], da.concatenate) 

849 elif processing_mode == "sequential": 

850 concatenate = cast(Callable[[Sequence[Array], int], Array], np.concatenate) 

851 else: 

852 assert_never(processing_mode) 

853 

854 def concatenate_blocks( 

855 nested: NestedList[Array], output_dims: Sequence[AxisId], level: int = 0 

856 ) -> Array: 

857 if not isinstance(nested, list): 

858 return nested 

859 

860 arrays = [concatenate_blocks(item, output_dims, level + 1) for item in nested] 

861 if len(arrays) == 1: 

862 return arrays[0] 

863 

864 grid_dim = output_grid.dims[level] 

865 if grid_dim not in output_dims: 865 ↛ 866line 865 didn't jump to line 866 because the condition on line 865 was never true

866 raise ValueError( 

867 f"Cannot concatenate blocks along input axis {grid_dim!r}, which is " 

868 + f"missing from output axes {tuple(output_dims)}" 

869 ) 

870 

871 return concatenate(arrays, output_dims.index(grid_dim)) 

872 

873 outputs = { 

874 output_id: Tensor( 

875 concatenate_blocks( 

876 select_arrays(nested_outputs, output_id), 

877 first_block[output_id].dims, 

878 ), 

879 dims=first_block[output_id].dims, 

880 ) 

881 for output_id in first_block 

882 } 

883 for output_id, tensor in outputs.items(): 

884 tensor.set_metadata(first_block[output_id].get_metadata()) 

885 return outputs 

886 

887 

888def apply_to_blocks( 

889 func: Callable[[PerMember[Tensor]], PerMember[Tensor | None]], 

890 blocks: PerMember[BlockArray], 

891 *, 

892 output_halo: PerMember[PerAxis[HaloLike]], 

893 get_output_shape: Callable[[PerMember[PerAxis[int]]], PerMember[PerAxis[int]]], 

894 keep_output_halo_at_sample_edge: bool, 

895) -> NestedListWithNamedDims[dict[MemberId, Tensor]]: 

896 if not all(b.size > 0 for b in blocks.values()): 896 ↛ 897line 896 didn't jump to line 897 because the condition on line 896 was never true

897 raise ValueError( 

898 "Sample is not split into blocks. Please provide `block_shapes` or split the sample first." 

899 ) 

900 

901 block_ndims = {m: b.shape for m, b in blocks.items()} 

902 if len(set(block_ndims.values())) != 1: 902 ↛ 903line 902 didn't jump to line 903 because the condition on line 902 was never true

903 raise ValueError(f"Input block grids do not match: {block_ndims}") 

904 

905 block_grid_shape = next(iter(block_ndims.values())) 

906 first_block = cast(Sequence[Block], next(iter(blocks.values())).flat)[0] 

907 block_grid_dims = first_block.dims 

908 

909 output_halo = { 

910 m: {a: Halo.create(h) for a, h in h_per_a.items()} 

911 for m, h_per_a in output_halo.items() 

912 } 

913 

914 def func_blocks( 

915 axis: int = 0, loc: tuple[int, ...] = () 

916 ) -> NestedList[dict[MemberId, Tensor]]: 

917 if axis < len(block_grid_shape): 

918 grid_size = block_grid_shape[axis] 

919 return [func_blocks(axis + 1, (*loc, i)) for i in range(grid_size)] 

920 

921 left_input_overlap = { 

922 a 

923 for b in blocks.values() 

924 for a, s in cast_and_assert(Block, b[loc]).inner_slice.items() 

925 if isinstance(s, SliceInfoWithOverlap) and s.start_wo_overlap != s.start 

926 } 

927 right_input_overlap = { 

928 a 

929 for b in blocks.values() 

930 for a, s in cast_and_assert(Block, b[loc]).inner_slice.items() 

931 if isinstance(s, SliceInfoWithOverlap) and s.stop_wo_overlap != s.stop 

932 } 

933 valid_input_shape = { 

934 m: b[loc].inner_shape_wo_overlap for m, b in blocks.items() 

935 } 

936 valid_output_shape = get_output_shape(valid_input_shape) 

937 input_members = {m: b[loc].outer_tensor for m, b in blocks.items()} 

938 ret: dict[MemberId, Tensor] = {} 

939 for m, t in func(input_members).items(): 

940 if t is None: 940 ↛ 941line 940 didn't jump to line 941 because the condition on line 940 was never true

941 continue 

942 

943 # crop halo to keep inner output 

944 keep_halo_here: dict[AxisId, Halo] = {} 

945 if (crop_halo_here := output_halo.get(m)) is not None: 

946 crop_halo_here = dict(crop_halo_here) 

947 if keep_output_halo_at_sample_edge: 

948 assert len(block_grid_dims) == len(block_grid_shape) == len(loc) 

949 for a, grid_size, loc_idx in zip( 

950 block_grid_dims, block_grid_shape, loc 

951 ): 

952 if a not in crop_halo_here: 

953 continue 

954 

955 if loc_idx == 0 and loc_idx == grid_size - 1: 

956 # single block is first and last block 

957 keep_halo_here[a] = crop_halo_here[a] 

958 crop_halo_here[a] = Halo(0, 0) 

959 elif loc_idx == 0: 

960 # first block 

961 keep_halo_here[a] = Halo(crop_halo_here[a].left, 0) 

962 crop_halo_here[a] = Halo(0, crop_halo_here[a].right) 

963 elif loc_idx == grid_size - 1: 

964 # last block 

965 keep_halo_here[a] = Halo(0, crop_halo_here[a].right) 

966 crop_halo_here[a] = Halo(crop_halo_here[a].left, 0) 

967 

968 t = t.crop_halo(crop_halo_here) 

969 

970 overlap_halo = { 

971 a: Halo(0, overlap) if a in right_input_overlap else Halo(overlap, 0) 

972 for a, vs in valid_output_shape[m].items() 

973 if ( 

974 overlap := t.tagged_shape[a] 

975 - sum( 

976 keep_halo_here.get(a, Halo(0, 0)) 

977 ) # consider the edge halo we explicitly keep to be valid, so we do not error consider it overlap 

978 - vs 

979 ) 

980 != 0 

981 } 

982 assert set(overlap_halo) == left_input_overlap | right_input_overlap, ( 

983 f"need to crop overlap {overlap_halo} from {t.tagged_shape} to " 

984 + f"{valid_output_shape[m]}, but expected overlap in " 

985 + f"{left_input_overlap | right_input_overlap}" 

986 ) 

987 t = t.crop_halo(overlap_halo) 

988 

989 ret[m] = t 

990 

991 return ret 

992 

993 return NestedListWithNamedDims(dims=block_grid_dims, values=func_blocks()) 

994 

995 

996def get_chunk_shape( 

997 tensors: PerMember[Tensor], 

998 block_shape: PerMember[PerAxis[int]], 

999 chunk_min_bytes: int = settings.processing_chunk_min_bytes, 

1000) -> PerMember[PerAxis[int]]: 

1001 if unknown_members := [m for m in block_shape if m not in tensors]: 1001 ↛ 1002line 1001 didn't jump to line 1002 because the condition on line 1001 was never true

1002 raise ValueError( 

1003 f"Block shape specified for unknown tensors: {unknown_members}." 

1004 ) 

1005 

1006 # default to tensor shape for missing block dimensions or block dimensions set to -1 

1007 block_shape = { 

1008 m: { 

1009 d: t.tagged_shape[d] 

1010 if (b_s := block_shape.get(m, {}).get(d, -1)) == -1 

1011 else b_s 

1012 for d in t.dims 

1013 } 

1014 for m, t in tensors.items() 

1015 } 

1016 dim_sizes = {d: d_s for b_s in block_shape.values() for d, d_s in b_s.items()} 

1017 

1018 if not any( 1018 ↛ 1021line 1018 didn't jump to line 1021 because the condition on line 1018 was never true

1019 a in tensors[m].tagged_shape for m, b_s in block_shape.items() for a in b_s 

1020 ): 

1021 raise ValueError( 

1022 f"Block shape {block_shape} has dimensions not present in sample shape { {m: tensors[m].tagged_shape for m in tensors} }" 

1023 ) 

1024 

1025 if inconsistent := { 1025 ↛ 1031line 1025 didn't jump to line 1031 because the condition on line 1025 was never true

1026 d: f"{s} != {bs[d]} (from {m})" 

1027 for m, bs in block_shape.items() 

1028 for d, s in dim_sizes.items() 

1029 if d in bs and bs[d] != s 

1030 }: 

1031 raise ValueError( 

1032 f"Block shape has inconsistent sizes across members: {inconsistent}" 

1033 ) 

1034 

1035 chunk_shape = deepcopy(block_shape) 

1036 item_sizes = {m: np.dtype(t.dtype).itemsize for m, t in tensors.items()} 

1037 

1038 def get_max_chunk_size(): 

1039 return max(prod(chunk_shape[m].values()) * item_sizes[m] for m in chunk_shape) 

1040 

1041 def increment_chunk_dims(incr_dims: Sequence[AxisId]): 

1042 incr_dims = list(incr_dims) 

1043 for _ in range(1_000_000): 

1044 for a in list(incr_dims): 

1045 if get_max_chunk_size() >= chunk_min_bytes: 

1046 return 

1047 max_chunk_sizes = { 

1048 m: max(block_shape[m][a], tensors[m].tagged_shape[a]) 

1049 for m in tensors 

1050 if a in tensors[m].tagged_shape 

1051 } 

1052 for m in tensors: 

1053 if a not in tensors[m].dims: 1053 ↛ 1054line 1053 didn't jump to line 1054 because the condition on line 1053 was never true

1054 continue 

1055 

1056 chunk_shape[m][a] = min( 

1057 max_chunk_sizes[m], 

1058 chunk_shape[m][a] + block_shape[m][a], 

1059 ) 

1060 

1061 if all( 

1062 chunk_shape[m][a] == max_chunk_sizes[m] 

1063 for m in chunk_shape 

1064 if a in chunk_shape[m] 

1065 ): 

1066 incr_dims.remove(a) 

1067 

1068 if not incr_dims: 

1069 break 

1070 

1071 # increment chunk dimensions in order of non-singleton spatial, time, other, then any singleton dim 

1072 if dim_sizes.get(AxisId("z"), 0) <= 1: 1072 ↛ 1081line 1072 didn't jump to line 1081 because the condition on line 1072 was always true

1073 non_singleton_xy = [ 

1074 d for d in (AxisId("x"), AxisId("y")) if dim_sizes.get(d, 0) > 1 

1075 ] 

1076 increment_chunk_dims(non_singleton_xy) 

1077 

1078 if dim_sizes.get((d := AxisId("z")), 0) > 1: 1078 ↛ 1079line 1078 didn't jump to line 1079 because the condition on line 1078 was never true

1079 increment_chunk_dims((d,)) 

1080 else: 

1081 non_singleton_space_dims = [ 

1082 d 

1083 for d in (AxisId("x"), AxisId("y"), AxisId("z")) 

1084 if dim_sizes.get(d, 0) > 1 

1085 ] 

1086 increment_chunk_dims(non_singleton_space_dims) 

1087 

1088 non_singleton_time_dims = [d for d in (TIME_AXIS_ID,) if dim_sizes.get(d, 0) > 1] 

1089 increment_chunk_dims(non_singleton_time_dims) 

1090 

1091 other_non_singleton_dims = [d for d, s in dim_sizes.items() if s > 1] 

1092 increment_chunk_dims(other_non_singleton_dims[::-1]) 

1093 

1094 singleton_dims = [d for d, s in dim_sizes.items() if s <= 1] 

1095 increment_chunk_dims(singleton_dims[::-1]) 

1096 

1097 logger.info( 

1098 "Calculated chunk shape {} ({} {}) for block shape {} and sample shape {}", 

1099 chunk_shape, 

1100 c 

1101 if (c := get_max_chunk_size()) < 1024 

1102 else c // 1024 

1103 if c < 1024**2 

1104 else c // 1024**2 

1105 if c < 1024**3 

1106 else c // 1024**3, 

1107 "B" if c < 1024 else "KiB" if c < 1024**2 else "MiB" if c < 1024**3 else "GiB", 

1108 block_shape, 

1109 {m: dict(t.tagged_shape) for m, t in tensors.items()}, 

1110 ) 

1111 return chunk_shape