Coverage for src/bioimageio/core/proc_ops.py: 82%

541 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 09:46 +0000

1from __future__ import annotations 

2 

3import collections.abc 

4import json 

5from collections.abc import Callable, Collection, Mapping, Sequence 

6from dataclasses import dataclass, field, fields 

7from functools import partial 

8from types import MappingProxyType 

9from typing import ( 

10 Annotated, 

11 Any, 

12 Literal, 

13 cast, 

14) 

15 

16import numpy as np 

17import scipy # pyright: ignore[reportMissingTypeStubs] 

18import xarray as xr 

19from numpy.typing import NDArray 

20from pydantic import BaseModel, ConfigDict, Discriminator, Tag, TypeAdapter 

21from typing_extensions import Self, assert_never 

22 

23from bioimageio.spec.model import v0_4, v0_5 

24from bioimageio.spec.model.v0_5 import ( 

25 _convert_proc, # pyright: ignore[reportPrivateUsage] 

26) 

27 

28from ._op_base import ( 

29 BlockwiseOperator, 

30 Operator, 

31 SamplewiseOperator, 

32 SimpleOperator, 

33 get_relevant_preceding_ops, 

34) 

35from ._ops_cellpose import CellposeFlowDynamics 

36from ._ops_stardist import StardistPostprocessing2D, StardistPostprocessing3D 

37from ._utils.type_hints import cast_and_assert 

38from .axis import CHANNEL_AXIS_ID, AxisId 

39from .common import DTypeStr, MemberId 

40from .digest_spec import get_member_id, import_callable 

41from .sample import Sample 

42from .stat_calculators import StatsCalculator 

43from .stat_measures import ( 

44 DatasetMean, 

45 DatasetMeasureBase, 

46 DatasetQuantile, 

47 DatasetStd, 

48 MeanMeasure, 

49 Measure, 

50 MeasureValue, 

51 SampleMean, 

52 SampleQuantile, 

53 SampleStd, 

54 Stat, 

55 StdMeasure, 

56) 

57from .tensor import Tensor 

58 

59 

60def _convert_axis_ids( 

61 axes: v0_4.AxesInCZYX, 

62 mode: Literal["per_sample", "per_dataset"], 

63) -> tuple[AxisId, ...]: 

64 if not isinstance(axes, str): 64 ↛ 65line 64 didn't jump to line 65 because the condition on line 64 was never true

65 return tuple(axes) 

66 

67 if mode == "per_sample": 67 ↛ 69line 67 didn't jump to line 69 because the condition on line 67 was always true

68 ret = [] 

69 elif mode == "per_dataset": 

70 ret = [AxisId(v0_5.BATCH_AXIS_ID)] 

71 else: 

72 assert_never(mode) 

73 

74 ret.extend([AxisId(a) for a in axes]) 

75 return tuple(ret) 

76 

77 

78@dataclass(frozen=True) 

79class AddKnownDatasetStats(BlockwiseOperator): 

80 dataset_stats: Mapping[Measure, MeasureValue] 

81 

82 @property 

83 def affects_statistics(self) -> bool: 

84 return False 

85 

86 def __post_init__(self): 

87 # keep only dataset measures 

88 object.__setattr__( 

89 self, 

90 "dataset_stats", 

91 {k: v for k, v in self.dataset_stats.items() if k.scope == "dataset"}, 

92 ) 

93 

94 def __eq__(self, other: object) -> bool: 

95 """Compare statistics by value, evaluating lazy tensors if necessary.""" 

96 if type(self) is not type(other) or not isinstance(other, AddKnownDatasetStats): 

97 return NotImplemented 

98 if self.dataset_stats.keys() != other.dataset_stats.keys(): 

99 return False 

100 for measure, value in self.dataset_stats.items(): 

101 other_value = other.dataset_stats[measure] 

102 if isinstance(value, Tensor): 

103 if ( 

104 not isinstance(other_value, Tensor) 

105 or value.dims != other_value.dims 

106 or not np.array_equal( 

107 value.to_numpy(), other_value.to_numpy(), equal_nan=True 

108 ) 

109 ): 

110 return False 

111 elif isinstance(other_value, Tensor) or not ( 111 ↛ 114line 111 didn't jump to line 114 because the condition on line 111 was never true

112 value == other_value or (np.isnan(value) and np.isnan(other_value)) 

113 ): 

114 return False 

115 return True 

116 

117 def __call__(self, sample: Sample) -> None: # | SampleBlock 

118 sample.stat.update(self.dataset_stats) 

119 sample.applied_ops += (self,) 

120 

121 

122class _StatsCalculatorReference(BaseModel, frozen=True): 

123 model_config = ConfigDict(extra="forbid") 

124 

125 measures: tuple[Measure, ...] 

126 dataset_fits_in_memory: bool 

127 

128 

129@dataclass(frozen=True) 

130class UpdateStats(SamplewiseOperator): 

131 """Calculates sample and/or dataset measures""" 

132 

133 stats_calculator: StatsCalculator | _StatsCalculatorReference = field( 

134 repr=False, compare=False 

135 ) 

136 """Calculator, or a non-executable reference after deserialization.""" 

137 keep_updating_initial_dataset_stats: bool = False 

138 """indicates if operator calls should keep updating initial dataset statistics or not; 

139 if the `stats_calculator` was not provided with any initial dataset statistics, 

140 these are always updated with every new sample. 

141 """ 

142 _keep_updating_dataset_stats: bool = field(init=False, compare=False, repr=False) 

143 calculator_reference: _StatsCalculatorReference = field(init=False, repr=False) 

144 

145 @property 

146 def affects_statistics(self) -> bool: 

147 return False 

148 

149 def __post_init__(self): 

150 calculator = self.stats_calculator 

151 reference = ( 

152 _StatsCalculatorReference( 

153 measures=calculator.measures, 

154 dataset_fits_in_memory=calculator.dataset_fits_in_memory, 

155 ) 

156 if isinstance(calculator, StatsCalculator) 

157 else calculator 

158 ) 

159 object.__setattr__(self, "calculator_reference", reference) 

160 object.__setattr__( 

161 self, 

162 "_keep_updating_dataset_stats", 

163 self.keep_updating_initial_dataset_stats 

164 or ( 

165 isinstance(calculator, StatsCalculator) 

166 and not calculator.has_dataset_measures 

167 ), 

168 ) 

169 

170 def __call__(self, sample: Sample) -> None: 

171 if not isinstance(self.stats_calculator, StatsCalculator): 

172 raise RuntimeError( # noqa: TRY004 

173 "A deserialized UpdateStats is a reference and cannot run" 

174 ) 

175 if self._keep_updating_dataset_stats: 

176 sample.stat.update(self.stats_calculator.update_and_get_all(sample)) 

177 else: 

178 sample.stat.update(self.stats_calculator.skip_update_and_get_all(sample)) 

179 sample.applied_ops += (self,) 

180 

181 

182@dataclass(frozen=True) 

183class Binarize(SimpleOperator): 

184 """'output = tensor > threshold'.""" 

185 

186 threshold: float | Sequence[float] 

187 axis: AxisId | None = None 

188 

189 def __post_init__(self): 

190 if not isinstance(self.threshold, (int, float)): 

191 object.__setattr__(self, "threshold", tuple(self.threshold)) 

192 

193 def _apply(self, x: Tensor, stat: Stat) -> Tensor: 

194 return x > self.threshold 

195 

196 def get_output_shape( 

197 self, input_shape: Mapping[AxisId, int] 

198 ) -> Mapping[AxisId, int]: 

199 return input_shape 

200 

201 def get_descr(self): 

202 if self.axis is None: 

203 assert isinstance(self.threshold, (int, float)) 

204 kwargs = v0_5.BinarizeKwargs(threshold=float(self.threshold)) 

205 else: 

206 assert not isinstance(self.threshold, (int, float)) 

207 kwargs = v0_5.BinarizeAlongAxisKwargs( 

208 threshold=list(self.threshold), axis=self.axis 

209 ) 

210 

211 return v0_5.BinarizeDescr(kwargs=kwargs) 

212 

213 @classmethod 

214 def from_proc_descr( 

215 cls, descr: v0_4.BinarizeDescr | v0_5.BinarizeDescr, member_id: MemberId 

216 ) -> Self: 

217 if isinstance(descr.kwargs, (v0_4.BinarizeKwargs, v0_5.BinarizeKwargs)): 

218 return cls( 

219 input=member_id, output=member_id, threshold=descr.kwargs.threshold 

220 ) 

221 elif isinstance(descr.kwargs, v0_5.BinarizeAlongAxisKwargs): 

222 return cls( 

223 input=member_id, 

224 output=member_id, 

225 threshold=descr.kwargs.threshold, 

226 axis=descr.kwargs.axis, 

227 ) 

228 else: 

229 assert_never(descr.kwargs) 

230 

231 

232@dataclass(frozen=True) 

233class Clip(SimpleOperator): 

234 min: float | SampleQuantile | DatasetQuantile | None = None 

235 """minimum value for clipping""" 

236 max: float | SampleQuantile | DatasetQuantile | None = None 

237 """maximum value for clipping""" 

238 

239 def __post_init__(self): 

240 if self.min is None and self.max is None: 240 ↛ 241line 240 didn't jump to line 241 because the condition on line 240 was never true

241 raise ValueError("missing min or max value") 

242 

243 if ( 243 ↛ 248line 243 didn't jump to line 248 because the condition on line 243 was never true

244 isinstance(self.min, float) 

245 and isinstance(self.max, float) 

246 and self.min >= self.max 

247 ): 

248 raise ValueError(f"expected min < max, but {self.min} >= {self.max}") 

249 

250 if isinstance(self.min, (SampleQuantile, DatasetQuantile)) and isinstance( 

251 self.max, (SampleQuantile, DatasetQuantile) 

252 ): 

253 if self.min.axes != self.max.axes: 253 ↛ 254line 253 didn't jump to line 254 because the condition on line 253 was never true

254 raise NotImplementedError( 

255 f"expected min and max quantiles with same axes, but got {self.min.axes} and {self.max.axes}" 

256 ) 

257 if self.min.q >= self.max.q: 257 ↛ 258line 257 didn't jump to line 258 because the condition on line 257 was never true

258 raise ValueError( 

259 f"expected min quantile < max quantile, but {self.min.q} >= {self.max.q}" 

260 ) 

261 

262 def get_required_measures(self) -> Collection[Measure]: 

263 return { 

264 arg 

265 for arg in (self.min, self.max) 

266 if isinstance(arg, (SampleQuantile, DatasetQuantile)) 

267 } 

268 

269 def _apply(self, x: Tensor, stat: Stat) -> Tensor: 

270 if isinstance(self.min, (SampleQuantile, DatasetQuantile)): 

271 min_value = stat[self.min] 

272 if isinstance(min_value, (int, float)): 

273 # use clip for scalar value 

274 min_clip_arg = min_value 

275 else: 

276 # clip does not support non-scalar values 

277 x = Tensor.from_xarray( 

278 cast_and_assert( 

279 xr.DataArray, 

280 x.data.where(x.data >= min_value.data, min_value.data), 

281 ) 

282 ) 

283 min_clip_arg = None 

284 else: 

285 min_clip_arg = self.min 

286 

287 if isinstance(self.max, (SampleQuantile, DatasetQuantile)): 

288 max_value = stat[self.max] 

289 if isinstance(max_value, (int, float)): 

290 # use clip for scalar value 

291 max_clip_arg = max_value 

292 else: 

293 # clip does not support non-scalar values 

294 x = Tensor.from_xarray( 

295 cast_and_assert( 

296 xr.DataArray, 

297 x.data.where(x.data <= max_value.data, max_value.data), 

298 ) 

299 ) 

300 max_clip_arg = None 

301 else: 

302 max_clip_arg = self.max 

303 

304 if min_clip_arg is not None or max_clip_arg is not None: 

305 x = x.clip(min_clip_arg, max_clip_arg) 

306 

307 return x 

308 

309 def get_output_shape( 

310 self, input_shape: Mapping[AxisId, int] 

311 ) -> Mapping[AxisId, int]: 

312 return input_shape 

313 

314 def get_descr(self): 

315 kwargs: dict[str, Any] = {} 

316 quantile: SampleQuantile | DatasetQuantile | None = None 

317 if isinstance(self.min, (SampleQuantile, DatasetQuantile)): 

318 kwargs["min_percentile"] = self.min.q * 100 

319 quantile = self.min 

320 else: 

321 kwargs["min"] = self.min 

322 

323 if isinstance(self.max, (SampleQuantile, DatasetQuantile)): 

324 kwargs["max_percentile"] = self.max.q * 100 

325 quantile = self.max 

326 else: 

327 kwargs["max"] = self.max 

328 

329 if quantile is not None: 

330 kwargs["axes"] = quantile.axes 

331 

332 return v0_5.ClipDescr(kwargs=v0_5.ClipKwargs(**kwargs)) 

333 

334 @classmethod 

335 def from_proc_descr( 

336 cls, 

337 descr: v0_4.ClipDescr | v0_5.ClipDescr, 

338 member_id: MemberId, 

339 preceding_ops: Sequence[Operator] = (), 

340 ) -> Self: 

341 if isinstance(descr, v0_5.ClipDescr): 341 ↛ 374line 341 didn't jump to line 374 because the condition on line 341 was always true

342 dataset_mode, axes = _get_axes(descr.kwargs) 

343 if dataset_mode: 

344 Quantile = DatasetQuantile 

345 else: 

346 Quantile = partial(SampleQuantile, method="inverted_cdf") 

347 

348 relevant_ops = get_relevant_preceding_ops(member_id, preceding_ops) 

349 

350 if descr.kwargs.min is not None: 

351 min_arg = descr.kwargs.min 

352 elif descr.kwargs.min_percentile is not None: 352 ↛ 360line 352 didn't jump to line 360 because the condition on line 352 was always true

353 min_arg = Quantile( 

354 q=descr.kwargs.min_percentile / 100, 

355 axes=axes, 

356 member_id=member_id, 

357 preceding_ops=relevant_ops, 

358 ) 

359 else: 

360 min_arg = None 

361 

362 if descr.kwargs.max is not None: 

363 max_arg = descr.kwargs.max 

364 elif descr.kwargs.max_percentile is not None: 364 ↛ 372line 364 didn't jump to line 372 because the condition on line 364 was always true

365 max_arg = Quantile( 

366 q=descr.kwargs.max_percentile / 100, 

367 axes=axes, 

368 member_id=member_id, 

369 preceding_ops=relevant_ops, 

370 ) 

371 else: 

372 max_arg = None 

373 

374 elif isinstance(descr, v0_4.ClipDescr): 

375 min_arg = descr.kwargs.min 

376 max_arg = descr.kwargs.max 

377 else: 

378 assert_never(descr) 

379 

380 return cls( 

381 input=member_id, 

382 output=member_id, 

383 min=min_arg, 

384 max=max_arg, 

385 ) 

386 

387 

388@dataclass(frozen=True) 

389class EnsureDtype(SimpleOperator): 

390 dtype: DTypeStr 

391 

392 @classmethod 

393 def from_proc_descr(cls, descr: v0_5.EnsureDtypeDescr, member_id: MemberId): 

394 return cls(input=member_id, output=member_id, dtype=descr.kwargs.dtype) 

395 

396 def get_descr(self): 

397 return v0_5.EnsureDtypeDescr(kwargs=v0_5.EnsureDtypeKwargs(dtype=self.dtype)) 

398 

399 def get_output_shape( 

400 self, input_shape: Mapping[AxisId, int] 

401 ) -> Mapping[AxisId, int]: 

402 return input_shape 

403 

404 def _apply(self, x: Tensor, stat: Stat) -> Tensor: 

405 return x.astype(self.dtype) 

406 

407 

408@dataclass(frozen=True) 

409class ScaleLinear(SimpleOperator): 

410 gain: float | tuple[float, ...] = 1.0 

411 """multiplicative factor""" 

412 

413 offset: float | tuple[float, ...] = 0.0 

414 """additive term""" 

415 axis: AxisId | None = None 

416 """Axis for tuple-valued gain or offset.""" 

417 

418 def __post_init__(self): 

419 if not isinstance(self.gain, (int, float, tuple)) or not isinstance( 

420 self.offset, (int, float, tuple) 

421 ): 

422 raise TypeError("gain and offset must be scalars or tuples") 

423 if self.axis is None and ( 423 ↛ 426line 423 didn't jump to line 426 because the condition on line 423 was never true

424 isinstance(self.gain, tuple) or isinstance(self.offset, tuple) 

425 ): 

426 raise ValueError("tuple-valued gain or offset requires an axis") 

427 

428 def _apply(self, x: Tensor, stat: Stat) -> Tensor: 

429 gain = ( 

430 xr.DataArray(list(self.gain), dims=self.axis) 

431 if isinstance(self.gain, tuple) 

432 else self.gain 

433 ) 

434 offset = ( 

435 xr.DataArray(list(self.offset), dims=self.axis) 

436 if isinstance(self.offset, tuple) 

437 else self.offset 

438 ) 

439 return x * gain + offset 

440 

441 def get_output_shape( 

442 self, input_shape: Mapping[AxisId, int] 

443 ) -> Mapping[AxisId, int]: 

444 return input_shape 

445 

446 def get_descr(self): 

447 if self.axis is not None: 447 ↛ 456line 447 didn't jump to line 456 because the condition on line 447 was always true

448 kwargs = v0_5.ScaleLinearAlongAxisKwargs( 

449 axis=self.axis, 

450 gain=list(self.gain) if isinstance(self.gain, tuple) else self.gain, 

451 offset=list(self.offset) 

452 if isinstance(self.offset, tuple) 

453 else self.offset, 

454 ) 

455 else: 

456 assert isinstance(self.gain, (int, float)) and isinstance( 

457 self.offset, (int, float) 

458 ) 

459 kwargs = v0_5.ScaleLinearKwargs(gain=self.gain, offset=self.offset) 

460 

461 return v0_5.ScaleLinearDescr(kwargs=kwargs) 

462 

463 @classmethod 

464 def from_proc_descr( 

465 cls, 

466 descr: v0_4.ScaleLinearDescr | v0_5.ScaleLinearDescr, 

467 member_id: MemberId, 

468 ) -> Self: 

469 kwargs = descr.kwargs 

470 if isinstance(kwargs, v0_5.ScaleLinearKwargs): 470 ↛ 471line 470 didn't jump to line 471 because the condition on line 470 was never true

471 axis = None 

472 elif isinstance(kwargs, v0_5.ScaleLinearAlongAxisKwargs): 472 ↛ 474line 472 didn't jump to line 474 because the condition on line 472 was always true

473 axis = kwargs.axis 

474 elif isinstance(kwargs, v0_4.ScaleLinearKwargs): 

475 if kwargs.axes is not None: 

476 raise NotImplementedError( 

477 "model.v0_4.ScaleLinearKwargs with axes not implemented, please consider updating the model to v0_5." 

478 ) 

479 axis = None 

480 else: 

481 assert_never(kwargs) 

482 

483 if axis is not None: 483 ↛ 495line 483 didn't jump to line 495 because the condition on line 483 was always true

484 gain = ( 

485 kwargs.gain 

486 if isinstance(kwargs.gain, (float, int)) 

487 else tuple(kwargs.gain) 

488 ) 

489 offset = ( 

490 kwargs.offset 

491 if isinstance(kwargs.offset, (float, int)) 

492 else tuple(kwargs.offset) 

493 ) 

494 else: 

495 assert isinstance(kwargs.gain, (float, int)) or len(kwargs.gain) == 1, ( 

496 kwargs.gain 

497 ) 

498 gain = ( 

499 kwargs.gain if isinstance(kwargs.gain, (float, int)) else kwargs.gain[0] 

500 ) 

501 assert isinstance(kwargs.offset, (float, int)) or len(kwargs.offset) == 1 

502 offset = ( 

503 kwargs.offset 

504 if isinstance(kwargs.offset, (float, int)) 

505 else kwargs.offset[0] 

506 ) 

507 

508 return cls( 

509 input=member_id, output=member_id, gain=gain, offset=offset, axis=axis 

510 ) 

511 

512 

513@dataclass(frozen=True) 

514class ScaleMeanVariance(SimpleOperator): 

515 axes: Sequence[AxisId] | None = None 

516 reference_tensor: MemberId | None = None 

517 eps: float = 1e-6 

518 mean: MeanMeasure = field(init=False) 

519 std: StdMeasure = field(init=False) 

520 ref_mean: MeanMeasure = field(init=False) 

521 ref_std: StdMeasure = field(init=False) 

522 _mean: MeanMeasure | None = field(default=None, repr=False, compare=False) 

523 _std: StdMeasure | None = field(default=None, repr=False, compare=False) 

524 _ref_mean: MeanMeasure | None = field(default=None, repr=False, compare=False) 

525 _ref_std: StdMeasure | None = field(default=None, repr=False, compare=False) 

526 

527 def get_required_measures(self) -> Collection[Measure]: 

528 return {self.mean, self.std, self.ref_mean, self.ref_std} 

529 

530 def __post_init__(self): 

531 axes = None if self.axes is None else tuple(self.axes) 

532 ref_tensor = self.reference_tensor or self.input 

533 if axes is None or AxisId("batch") not in axes: 533 ↛ 537line 533 didn't jump to line 537 because the condition on line 533 was always true

534 Mean = SampleMean 

535 Std = SampleStd 

536 else: 

537 Mean = DatasetMean 

538 Std = DatasetStd 

539 

540 object.__setattr__(self, "axes", axes) 

541 object.__setattr__( 

542 self, 

543 "mean", 

544 self._mean 

545 if self._mean is not None 

546 else Mean(member_id=self.input, axes=axes), 

547 ) 

548 object.__setattr__( 

549 self, 

550 "std", 

551 self._std 

552 if self._std is not None 

553 else Std(member_id=self.input, axes=axes), 

554 ) 

555 object.__setattr__( 

556 self, 

557 "ref_mean", 

558 self._ref_mean 

559 if self._ref_mean is not None 

560 else Mean(member_id=ref_tensor, axes=axes), 

561 ) 

562 object.__setattr__( 

563 self, 

564 "ref_std", 

565 self._ref_std 

566 if self._ref_std is not None 

567 else Std(member_id=ref_tensor, axes=axes), 

568 ) 

569 if ( 

570 any( 

571 m.axes != axes 

572 for m in (self.mean, self.std, self.ref_mean, self.ref_std) 

573 ) 

574 or any(m.member_id != self.input for m in (self.mean, self.std)) 

575 or any(m.member_id != ref_tensor for m in (self.ref_mean, self.ref_std)) 

576 or not isinstance(self.mean, Mean) 

577 or not isinstance(self.std, Std) 

578 or not isinstance(self.ref_mean, Mean) 

579 or not isinstance(self.ref_std, Std) 

580 ): 

581 raise ValueError( 

582 "measures must match the operator's members, axes and scope" 

583 ) 

584 

585 def _apply(self, x: Tensor, stat: Stat) -> Tensor: 

586 mean = stat[self.mean] 

587 std = stat[self.std] + self.eps 

588 ref_mean = stat[self.ref_mean] 

589 ref_std = stat[self.ref_std] + self.eps 

590 return (x - mean) / std * ref_std + ref_mean 

591 

592 def get_output_shape( 

593 self, input_shape: Mapping[AxisId, int] 

594 ) -> Mapping[AxisId, int]: 

595 return input_shape 

596 

597 def get_descr(self): 

598 return v0_5.ScaleMeanVarianceDescr( 

599 kwargs=v0_5.ScaleMeanVarianceKwargs( 

600 reference_tensor=self.reference_tensor or self.input, 

601 axes=self.axes, 

602 eps=self.eps, 

603 ) 

604 ) 

605 

606 @classmethod 

607 def from_proc_descr( 

608 cls, 

609 descr: v0_4.ScaleMeanVarianceDescr | v0_5.ScaleMeanVarianceDescr, 

610 member_id: MemberId, 

611 preceding_ops: Sequence[Operator] = (), 

612 ) -> Self: 

613 kwargs = descr.kwargs 

614 _, axes = _get_axes(descr.kwargs) 

615 ref_tensor = MemberId(str(kwargs.reference_tensor)) 

616 if axes is None or AxisId("batch") not in axes: 616 ↛ 619line 616 didn't jump to line 619 because the condition on line 616 was always true

617 Mean, Std = SampleMean, SampleStd 

618 else: 

619 Mean, Std = DatasetMean, DatasetStd 

620 input_ops = get_relevant_preceding_ops(member_id, preceding_ops) 

621 ref_ops = get_relevant_preceding_ops(ref_tensor, preceding_ops) 

622 

623 return cls( 

624 input=member_id, 

625 output=member_id, 

626 reference_tensor=ref_tensor, 

627 axes=axes, 

628 eps=kwargs.eps, 

629 _mean=Mean(member_id=member_id, axes=axes, preceding_ops=input_ops), 

630 _std=Std(member_id=member_id, axes=axes, preceding_ops=input_ops), 

631 _ref_mean=Mean(member_id=ref_tensor, axes=axes, preceding_ops=ref_ops), 

632 _ref_std=Std(member_id=ref_tensor, axes=axes, preceding_ops=ref_ops), 

633 ) 

634 

635 

636def _get_axes( 

637 kwargs: v0_4.ZeroMeanUnitVarianceKwargs 

638 | v0_5.ZeroMeanUnitVarianceKwargs 

639 | v0_4.ScaleRangeKwargs 

640 | v0_5.ScaleRangeKwargs 

641 | v0_4.ScaleMeanVarianceKwargs 

642 | v0_5.ScaleMeanVarianceKwargs 

643 | v0_5.ClipKwargs, 

644) -> tuple[bool, tuple[AxisId, ...] | None]: 

645 if kwargs.axes is None: 

646 return True, None 

647 elif isinstance(kwargs.axes, str): 

648 axes = _convert_axis_ids(kwargs.axes, kwargs["mode"]) 

649 return AxisId("b") in axes, axes 

650 elif isinstance(kwargs.axes, collections.abc.Sequence): 

651 axes = tuple(kwargs.axes) 

652 return AxisId("batch") in axes, axes 

653 else: 

654 assert_never(kwargs.axes) 

655 

656 

657@dataclass(frozen=True) 

658class ScaleRange(SimpleOperator): 

659 _lower: SampleQuantile | DatasetQuantile | None = field( 

660 default=None, repr=False, compare=False 

661 ) 

662 _upper: SampleQuantile | DatasetQuantile | None = field( 

663 default=None, repr=False, compare=False 

664 ) 

665 lower: SampleQuantile | DatasetQuantile = field(init=False) 

666 upper: SampleQuantile | DatasetQuantile = field(init=False) 

667 

668 eps: float = 1e-6 

669 

670 def __post_init__( 

671 self, 

672 ): 

673 if self._lower is None: 

674 m = self.input if self._upper is None else self._upper.member_id 

675 object.__setattr__(self, "lower", DatasetQuantile(q=0.0, member_id=m)) 

676 else: 

677 object.__setattr__(self, "lower", self._lower) 

678 

679 if self._upper is None: 

680 object.__setattr__( 

681 self, 

682 "upper", 

683 DatasetQuantile(q=1.0, member_id=self.lower.member_id), 

684 ) 

685 else: 

686 object.__setattr__(self, "upper", self._upper) 

687 

688 assert self.lower.member_id == self.upper.member_id 

689 assert self.lower.q < self.upper.q 

690 assert self.lower.axes == self.upper.axes 

691 

692 def get_required_measures(self) -> Collection[Measure]: 

693 return {self.lower, self.upper} 

694 

695 def get_output_shape( 

696 self, input_shape: Mapping[AxisId, int] 

697 ) -> Mapping[AxisId, int]: 

698 return input_shape 

699 

700 @classmethod 

701 def from_proc_descr( 

702 cls, 

703 descr: v0_4.ScaleRangeDescr | v0_5.ScaleRangeDescr, 

704 member_id: MemberId, 

705 preceding_ops: Sequence[Operator] = (), 

706 ): 

707 kwargs = descr.kwargs 

708 ref_tensor = ( 

709 member_id 

710 if kwargs.reference_tensor is None 

711 else MemberId(str(kwargs.reference_tensor)) 

712 ) 

713 dataset_mode, axes = _get_axes(descr.kwargs) 

714 if dataset_mode: 

715 Quantile = DatasetQuantile 

716 else: 

717 Quantile = partial(SampleQuantile, method="linear") 

718 

719 relevant_ops = get_relevant_preceding_ops(ref_tensor, preceding_ops) 

720 return cls( 

721 input=member_id, 

722 output=member_id, 

723 _lower=Quantile( 

724 q=kwargs.min_percentile / 100, 

725 axes=axes, 

726 member_id=ref_tensor, 

727 preceding_ops=relevant_ops, 

728 ), 

729 _upper=Quantile( 

730 q=kwargs.max_percentile / 100, 

731 axes=axes, 

732 member_id=ref_tensor, 

733 preceding_ops=relevant_ops, 

734 ), 

735 ) 

736 

737 def _apply(self, x: Tensor, stat: Stat) -> Tensor: 

738 lower = stat[self.lower] 

739 upper = stat[self.upper] 

740 return (x - lower) / (upper - lower + self.eps) 

741 

742 def get_descr(self): 

743 assert self.lower.axes == self.upper.axes 

744 assert self.lower.member_id == self.upper.member_id 

745 

746 return v0_5.ScaleRangeDescr( 

747 kwargs=v0_5.ScaleRangeKwargs( 

748 axes=None 

749 if self.lower.axes is None 

750 else [v0_5.AxisId(a) for a in self.lower.axes], 

751 min_percentile=self.lower.q * 100, 

752 max_percentile=self.upper.q * 100, 

753 eps=self.eps, 

754 reference_tensor=self.lower.member_id, 

755 ) 

756 ) 

757 

758 

759@dataclass(frozen=True) 

760class Sigmoid(SimpleOperator): 

761 """1 / (1 + e^(-input)).""" 

762 

763 def _apply(self, x: Tensor, stat: Stat) -> Tensor: 

764 return Tensor.from_xarray( 

765 cast_and_assert( 

766 xr.DataArray, x.data.copy(data=1.0 / (1.0 + np.exp(-x.data.data))) 

767 ) 

768 ) 

769 

770 def get_output_shape( 

771 self, input_shape: Mapping[AxisId, int] 

772 ) -> Mapping[AxisId, int]: 

773 return input_shape 

774 

775 @classmethod 

776 def from_proc_descr( 

777 cls, descr: v0_4.SigmoidDescr | v0_5.SigmoidDescr, member_id: MemberId 

778 ) -> Self: 

779 assert isinstance(descr, (v0_4.SigmoidDescr, v0_5.SigmoidDescr)) 

780 return cls(input=member_id, output=member_id) 

781 

782 def get_descr(self): 

783 return v0_5.SigmoidDescr() 

784 

785 

786@dataclass(frozen=True) 

787class Softmax(SimpleOperator): 

788 """Softmax activation function.""" 

789 

790 axis: AxisId = CHANNEL_AXIS_ID 

791 

792 def _apply(self, x: Tensor, stat: Stat) -> Tensor: 

793 axis_idx = x.dims.index(self.axis) 

794 result = scipy.special.softmax(x.data, axis=axis_idx) 

795 result_xr = cast_and_assert(xr.DataArray, x.data.copy(data=result)) 

796 return Tensor.from_xarray(result_xr) 

797 

798 def get_output_shape( 

799 self, input_shape: Mapping[AxisId, int] 

800 ) -> Mapping[AxisId, int]: 

801 return input_shape 

802 

803 @classmethod 

804 def from_proc_descr(cls, descr: v0_5.SoftmaxDescr, member_id: MemberId) -> Self: 

805 assert isinstance(descr, v0_5.SoftmaxDescr) 

806 return cls(input=member_id, output=member_id, axis=descr.kwargs.axis) 

807 

808 def get_descr(self): 

809 return v0_5.SoftmaxDescr(kwargs=v0_5.SoftmaxKwargs(axis=v0_5.AxisId(self.axis))) 

810 

811 

812@dataclass(frozen=True) 

813class ZeroMeanUnitVariance(SimpleOperator): 

814 """normalize to zero mean, unit variance.""" 

815 

816 mean: MeanMeasure 

817 std: StdMeasure 

818 

819 eps: float = 1e-6 

820 

821 def __post_init__(self): 

822 assert self.mean.axes == self.std.axes 

823 

824 def get_required_measures(self) -> Collection[Measure]: 

825 return {self.mean, self.std} 

826 

827 def get_output_shape( 

828 self, input_shape: Mapping[AxisId, int] 

829 ) -> Mapping[AxisId, int]: 

830 return input_shape 

831 

832 @classmethod 

833 def from_proc_descr( 

834 cls, 

835 descr: v0_4.ZeroMeanUnitVarianceDescr | v0_5.ZeroMeanUnitVarianceDescr, 

836 member_id: MemberId, 

837 preceding_ops: Sequence[Operator] = (), 

838 ): 

839 dataset_mode, axes = _get_axes(descr.kwargs) 

840 if dataset_mode: 

841 Mean = DatasetMean 

842 Std = DatasetStd 

843 else: 

844 Mean = SampleMean 

845 Std = SampleStd 

846 

847 relevant_ops = get_relevant_preceding_ops(member_id, preceding_ops) 

848 return cls( 

849 input=member_id, 

850 output=member_id, 

851 mean=Mean(axes=axes, member_id=member_id, preceding_ops=relevant_ops), 

852 std=Std(axes=axes, member_id=member_id, preceding_ops=relevant_ops), 

853 ) 

854 

855 def _apply(self, x: Tensor, stat: Stat) -> Tensor: 

856 mean = stat[self.mean] 

857 std = stat[self.std] 

858 return (x - mean) / (std + self.eps) 

859 

860 def get_descr(self): 

861 return v0_5.ZeroMeanUnitVarianceDescr( 

862 kwargs=v0_5.ZeroMeanUnitVarianceKwargs(axes=self.mean.axes, eps=self.eps) 

863 ) 

864 

865 

866@dataclass(frozen=True) 

867class FixedZeroMeanUnitVariance(SimpleOperator): 

868 """normalize to zero mean, unit variance with precomputed values.""" 

869 

870 mean: float | tuple[float, ...] 

871 std: float | tuple[float, ...] 

872 

873 eps: float = 1e-6 

874 axis: AxisId | None = None 

875 """Axis for tuple-valued mean and std.""" 

876 

877 def __post_init__(self): 

878 if not isinstance(self.mean, (int, float, tuple)) or not isinstance( 

879 self.std, (int, float, tuple) 

880 ): 

881 raise TypeError("mean and std must be scalars or tuples") 

882 if self.axis is None: 

883 if isinstance(self.mean, tuple) or isinstance(self.std, tuple): 883 ↛ 884line 883 didn't jump to line 884 because the condition on line 883 was never true

884 raise ValueError("mean and std must be scalars without an axis") 

885 elif not isinstance(self.mean, tuple) or not isinstance(self.std, tuple): 885 ↛ 886line 885 didn't jump to line 886 because the condition on line 885 was never true

886 raise ValueError("mean and std must be tuples when an axis is specified") 

887 

888 def get_output_shape( 

889 self, input_shape: Mapping[AxisId, int] 

890 ) -> Mapping[AxisId, int]: 

891 return input_shape 

892 

893 @classmethod 

894 def from_proc_descr( 

895 cls, 

896 descr: v0_5.FixedZeroMeanUnitVarianceDescr, 

897 member_id: MemberId, 

898 ) -> Self: 

899 if isinstance(descr.kwargs, v0_5.FixedZeroMeanUnitVarianceKwargs): 

900 mean, std, axis = descr.kwargs.mean, descr.kwargs.std, None 

901 elif isinstance(descr.kwargs, v0_5.FixedZeroMeanUnitVarianceAlongAxisKwargs): 

902 mean, std, axis = ( 

903 tuple(descr.kwargs.mean), 

904 tuple(descr.kwargs.std), 

905 AxisId(descr.kwargs.axis), 

906 ) 

907 else: 

908 assert_never(descr.kwargs) 

909 

910 return cls( 

911 input=member_id, 

912 output=member_id, 

913 mean=mean, 

914 std=std, 

915 axis=axis, 

916 ) 

917 

918 def get_descr(self): 

919 if self.axis is None: 

920 assert isinstance(self.mean, (int, float)) 

921 assert isinstance(self.std, (int, float)) 

922 kwargs = v0_5.FixedZeroMeanUnitVarianceKwargs(mean=self.mean, std=self.std) 

923 else: 

924 assert isinstance(self.mean, tuple) and isinstance(self.std, tuple) 

925 kwargs = v0_5.FixedZeroMeanUnitVarianceAlongAxisKwargs( 

926 axis=self.axis, 

927 mean=list(self.mean), 

928 std=list(self.std), 

929 ) 

930 

931 return v0_5.FixedZeroMeanUnitVarianceDescr(kwargs=kwargs) 

932 

933 def _apply(self, x: Tensor, stat: Stat) -> Tensor: 

934 if self.axis is None: 

935 assert isinstance(self.mean, (int, float)) and isinstance( 

936 self.std, (int, float) 

937 ) 

938 mean = self.mean 

939 std = self.std 

940 else: 

941 assert isinstance(self.mean, tuple) and isinstance(self.std, tuple) 

942 mean = xr.DataArray(list(self.mean), dims=self.axis) 

943 std = xr.DataArray(list(self.std), dims=self.axis) 

944 

945 return cast_and_assert(Tensor, (x - mean) / (std + self.eps)) 

946 

947 

948@dataclass(frozen=True) 

949class CustomProcessing(SimpleOperator): 

950 """Execute a user-supplied custom processing callable. 

951 

952 Two styles are supported — callable class and factory function:: 

953 

954 # Callable class style 

955 class my_factory: 

956 def __init__(self, threshold=0.5): 

957 self.threshold = threshold 

958 def __call__(self, *arrays): 

959 return (arrays[0] > self.threshold).astype(np.uint8) 

960 

961 # Factory function style 

962 def my_factory(threshold=0.5): 

963 def run(*arrays): 

964 return (arrays[0] > threshold).astype(np.uint8) 

965 return run 

966 

967 Runtime protocol: ``custom_callable = my_factory(**kwargs)`` once at construction; 

968 ``result = custom_callable(tensor)`` once per sample. 

969 

970 Note: The custom callable may not change the shape of the input tensor. 

971 """ 

972 

973 custom_factory: Callable[..., Callable[[NDArray[Any]], NDArray[Any]]] 

974 

975 kwargs: Mapping[str, Any] = field(compare=False) 

976 """Keyword arguments forwarded to the custom factory.""" 

977 

978 _kwargs_key: str = field(init=False, repr=False) 

979 

980 # Initialised in __post_init__ 

981 _custom_callable: Any = field(init=False, repr=False, compare=False) 

982 

983 def __post_init__(self) -> None: 

984 kwargs = dict(self.kwargs) 

985 object.__setattr__(self, "kwargs", MappingProxyType(kwargs)) 

986 object.__setattr__( 

987 self, 

988 "_kwargs_key", 

989 json.dumps(kwargs, sort_keys=True, separators=(",", ":")), 

990 ) 

991 object.__setattr__(self, "_custom_callable", self.custom_factory(**kwargs)) 

992 

993 def _apply(self, x: Tensor, stat: Stat) -> Tensor: 

994 return Tensor.from_xarray( 

995 cast_and_assert( 

996 xr.DataArray, x.data.copy(data=self._custom_callable(x.to_numpy())) 

997 ) 

998 ) 

999 

1000 def get_output_shape( 

1001 self, input_shape: Mapping[AxisId, int] 

1002 ) -> Mapping[AxisId, int]: 

1003 return input_shape 

1004 

1005 @classmethod 

1006 def from_proc_descr( 

1007 cls, 

1008 descr: v0_5.CustomProcessingDescr, 

1009 member_id: MemberId, 

1010 ) -> Self: 

1011 factory = import_callable(descr) 

1012 

1013 return cls( 

1014 input=member_id, 

1015 output=member_id, 

1016 custom_factory=factory, 

1017 kwargs=dict(descr.kwargs), 

1018 ) 

1019 

1020 

1021ProcDescr = ( 

1022 v0_4.PreprocessingDescr 

1023 | v0_4.PostprocessingDescr 

1024 | v0_5.PreprocessingDescr 

1025 | v0_5.PostprocessingDescr 

1026) 

1027Processing = ( 

1028 AddKnownDatasetStats 

1029 | Binarize 

1030 | Clip 

1031 | CellposeFlowDynamics 

1032 | CustomProcessing 

1033 | EnsureDtype 

1034 | FixedZeroMeanUnitVariance 

1035 | ScaleLinear 

1036 | ScaleMeanVariance 

1037 | ScaleRange 

1038 | Sigmoid 

1039 | StardistPostprocessing2D 

1040 | StardistPostprocessing3D 

1041 | Softmax 

1042 | UpdateStats 

1043 | ZeroMeanUnitVariance 

1044) 

1045 

1046 

1047def get_proc( 

1048 proc_descr: ProcDescr, 

1049 tensor_descr: v0_4.InputTensorDescr 

1050 | v0_4.OutputTensorDescr 

1051 | v0_5.InputTensorDescr 

1052 | v0_5.OutputTensorDescr 

1053 | MemberId, 

1054 *, 

1055 tensor_axes: str | None = None, 

1056 preceding_ops: Sequence[Operator] = (), 

1057) -> Processing: 

1058 """Construct an operator from a processing description. 

1059 

1060 `tensor_axes` is only needed if `proc_descr` is a v0.4.ZeroMeanUnitVarianceDescr with mode "fixed". 

1061 """ 

1062 if isinstance( 

1063 tensor_descr, 

1064 ( 

1065 v0_4.InputTensorDescr, 

1066 v0_4.OutputTensorDescr, 

1067 v0_5.InputTensorDescr, 

1068 v0_5.OutputTensorDescr, 

1069 ), 

1070 ): 

1071 member_id = get_member_id(tensor_descr) 

1072 if isinstance(tensor_descr, (v0_4.InputTensorDescr, v0_4.OutputTensorDescr)): 

1073 tensor_axes = tensor_descr.axes 

1074 else: 

1075 member_id = tensor_descr 

1076 

1077 if isinstance(proc_descr, (v0_4.BinarizeDescr, v0_5.BinarizeDescr)): 1077 ↛ 1078line 1077 didn't jump to line 1078 because the condition on line 1077 was never true

1078 return Binarize.from_proc_descr(proc_descr, member_id) 

1079 elif isinstance(proc_descr, v0_5.CellposeFlowDynamicsDescr): 

1080 return CellposeFlowDynamics.from_proc_descr(proc_descr, member_id) 

1081 elif isinstance(proc_descr, (v0_4.ClipDescr, v0_5.ClipDescr)): 

1082 return Clip.from_proc_descr(proc_descr, member_id, preceding_ops) 

1083 elif isinstance(proc_descr, v0_5.CustomProcessingDescr): 1083 ↛ 1084line 1083 didn't jump to line 1084 because the condition on line 1083 was never true

1084 return CustomProcessing.from_proc_descr(proc_descr, member_id) 

1085 elif isinstance(proc_descr, v0_5.EnsureDtypeDescr): 

1086 return EnsureDtype.from_proc_descr(proc_descr, member_id) 

1087 elif isinstance(proc_descr, v0_5.FixedZeroMeanUnitVarianceDescr): 1087 ↛ 1088line 1087 didn't jump to line 1088 because the condition on line 1087 was never true

1088 return FixedZeroMeanUnitVariance.from_proc_descr(proc_descr, member_id) 

1089 elif isinstance(proc_descr, (v0_4.ScaleLinearDescr, v0_5.ScaleLinearDescr)): 1089 ↛ 1090line 1089 didn't jump to line 1090 because the condition on line 1089 was never true

1090 return ScaleLinear.from_proc_descr(proc_descr, member_id) 

1091 elif isinstance( 

1092 proc_descr, (v0_4.ScaleMeanVarianceDescr, v0_5.ScaleMeanVarianceDescr) 

1093 ): 

1094 return ScaleMeanVariance.from_proc_descr(proc_descr, member_id, preceding_ops) 

1095 elif isinstance(proc_descr, (v0_4.ScaleRangeDescr, v0_5.ScaleRangeDescr)): 

1096 return ScaleRange.from_proc_descr(proc_descr, member_id, preceding_ops) 

1097 elif isinstance(proc_descr, (v0_4.SigmoidDescr, v0_5.SigmoidDescr)): 

1098 return Sigmoid.from_proc_descr(proc_descr, member_id) 

1099 elif ( 

1100 isinstance(proc_descr, v0_4.ZeroMeanUnitVarianceDescr) 

1101 and proc_descr.kwargs.mode == "fixed" 

1102 ): 

1103 if tensor_axes is None: 1103 ↛ 1104line 1103 didn't jump to line 1104 because the condition on line 1103 was never true

1104 raise TypeError( 

1105 "Expected v0.4 tensor axes for fixed v0.4 processing description" 

1106 ) 

1107 

1108 v5_proc_descr = _convert_proc(proc_descr, tensor_axes) 

1109 assert isinstance(v5_proc_descr, v0_5.FixedZeroMeanUnitVarianceDescr) 

1110 return FixedZeroMeanUnitVariance.from_proc_descr(v5_proc_descr, member_id) 

1111 elif isinstance(proc_descr, v0_5.SoftmaxDescr): 

1112 return Softmax.from_proc_descr(proc_descr, member_id) 

1113 elif isinstance(proc_descr, v0_5.StardistPostprocessingDescr): 1113 ↛ 1114line 1113 didn't jump to line 1114 because the condition on line 1113 was never true

1114 if len(proc_descr.kwargs.grid) == 2: 

1115 return StardistPostprocessing2D.from_proc_descr(proc_descr, member_id) 

1116 elif len(proc_descr.kwargs.grid) == 3: 

1117 return StardistPostprocessing3D.from_proc_descr(proc_descr, member_id) 

1118 else: 

1119 assert_never(proc_descr.kwargs.grid) 

1120 elif isinstance( 

1121 proc_descr, 

1122 (v0_4.ZeroMeanUnitVarianceDescr, v0_5.ZeroMeanUnitVarianceDescr), 

1123 ): 

1124 return ZeroMeanUnitVariance.from_proc_descr( 

1125 proc_descr, member_id, preceding_ops 

1126 ) 

1127 else: 

1128 assert_never(proc_descr) 

1129 

1130 

1131def _processing_tag(value: object) -> str | None: 

1132 if isinstance(value, Mapping): 

1133 tag = cast(Mapping[str, object], value).get("type") 

1134 return tag if isinstance(tag, str) else None 

1135 return type(value).__name__ 

1136 

1137 

1138SerializableProcessing = Annotated[ 

1139 Annotated[Binarize, Tag("Binarize")] 

1140 | Annotated[Clip, Tag("Clip")] 

1141 | Annotated[CellposeFlowDynamics, Tag("CellposeFlowDynamics")] 

1142 | Annotated[EnsureDtype, Tag("EnsureDtype")] 

1143 | Annotated[FixedZeroMeanUnitVariance, Tag("FixedZeroMeanUnitVariance")] 

1144 | Annotated[ScaleLinear, Tag("ScaleLinear")] 

1145 | Annotated[ScaleMeanVariance, Tag("ScaleMeanVariance")] 

1146 | Annotated[ScaleRange, Tag("ScaleRange")] 

1147 | Annotated[Sigmoid, Tag("Sigmoid")] 

1148 | Annotated[StardistPostprocessing2D, Tag("StardistPostprocessing2D")] 

1149 | Annotated[StardistPostprocessing3D, Tag("StardistPostprocessing3D")] 

1150 | Annotated[Softmax, Tag("Softmax")] 

1151 | Annotated[ZeroMeanUnitVariance, Tag("ZeroMeanUnitVariance")], 

1152 Discriminator(_processing_tag), 

1153] 

1154_PROCESSING_ADAPTER: TypeAdapter[SerializableProcessing] = TypeAdapter( 

1155 SerializableProcessing 

1156) 

1157_SERIALIZABLE_PROCESSING_TYPES = ( 

1158 Binarize, 

1159 Clip, 

1160 CellposeFlowDynamics, 

1161 EnsureDtype, 

1162 FixedZeroMeanUnitVariance, 

1163 ScaleLinear, 

1164 ScaleMeanVariance, 

1165 ScaleRange, 

1166 Sigmoid, 

1167 StardistPostprocessing2D, 

1168 StardistPostprocessing3D, 

1169 Softmax, 

1170 ZeroMeanUnitVariance, 

1171) 

1172 

1173 

1174class _SerializedUpdateStats(BaseModel): 

1175 model_config = ConfigDict(extra="forbid") 

1176 

1177 type: Literal["UpdateStats"] 

1178 stats_calculator: _StatsCalculatorReference 

1179 keep_updating_initial_dataset_stats: bool 

1180 

1181 

1182def serialize_processing(proc: Operator) -> dict[str, Any]: 

1183 """Serialize an operator, omitting mutable statistics calculator state.""" 

1184 if type(proc) is AddKnownDatasetStats: 

1185 from .io import serialize_stat 

1186 

1187 assert isinstance(proc, AddKnownDatasetStats) 

1188 return { 

1189 "type": "AddKnownDatasetStats", 

1190 "dataset_stats": serialize_stat(proc.dataset_stats), 

1191 } 

1192 if type(proc) is UpdateStats: 

1193 assert isinstance(proc, UpdateStats) 

1194 return _SerializedUpdateStats( 

1195 type="UpdateStats", 

1196 stats_calculator=proc.calculator_reference, 

1197 keep_updating_initial_dataset_stats=proc.keep_updating_initial_dataset_stats, 

1198 ).model_dump(mode="json") 

1199 if ( 

1200 not isinstance(proc, _SERIALIZABLE_PROCESSING_TYPES) 

1201 or type(proc) not in _SERIALIZABLE_PROCESSING_TYPES 

1202 ): 

1203 raise TypeError(f"{type(proc).__name__} cannot be serialized") 

1204 

1205 return { 

1206 "type": type(proc).__name__, 

1207 **_PROCESSING_ADAPTER.dump_python(proc, mode="json", warnings="error"), 

1208 } 

1209 

1210 

1211def deserialize_processing( 

1212 data: Mapping[str, Any], 

1213) -> SerializableProcessing | AddKnownDatasetStats | UpdateStats: 

1214 """Restore a tagged operator, rejecting incomplete or unexpected fields.""" 

1215 if data.get("type") == "AddKnownDatasetStats": 

1216 from .io import JsonValue, load_stat 

1217 

1218 if set(data) != {"type", "dataset_stats"} or not isinstance( 1218 ↛ 1221line 1218 didn't jump to line 1221 because the condition on line 1218 was never true

1219 data["dataset_stats"], list 

1220 ): 

1221 raise ValueError("Invalid serialized AddKnownDatasetStats fields") 

1222 stats = load_stat(cast(list[JsonValue], data["dataset_stats"])) 

1223 if any(not isinstance(measure, DatasetMeasureBase) for measure in stats): 

1224 raise ValueError("AddKnownDatasetStats requires dataset measures") 

1225 return AddKnownDatasetStats(stats) 

1226 if data.get("type") == "UpdateStats": 

1227 payload = _SerializedUpdateStats.model_validate(data) 

1228 return UpdateStats( 

1229 stats_calculator=payload.stats_calculator, 

1230 keep_updating_initial_dataset_stats=payload.keep_updating_initial_dataset_stats, 

1231 ) 

1232 proc = _PROCESSING_ADAPTER.validate_python(data) 

1233 expected = {"type", *(f.name for f in fields(proc) if f.init)} 

1234 if set(data) != expected: 

1235 raise ValueError( 

1236 f"Invalid serialized {type(proc).__name__} fields: missing " 

1237 + f"{expected - set(data)}, unexpected {set(data) - expected}" 

1238 ) 

1239 return proc