Coverage for python/lsst/images/_observation_summary

1# This file is part of lsst-images.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (https://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# Use of this source code is governed by a 3-clause BSD-style

10# license that can be found in the LICENSE file.

11from __future__ import annotations

13__all__ = ("ObservationSummaryStats",)

15import dataclasses

16import math

17from typing import TYPE_CHECKING, Any, ClassVar, Self, final, get_origin

19import pydantic

21from lsst.images.describe import (

22 DescribableMixin,

23 DescribeOptions,

24 FieldRole,

25 Report,

26 ReportField,

27 ReportValueGroup,

28)

29from lsst.images.serialization import ArchiveTree, InputArchive, InvalidParameterError, OutputArchive

31if TYPE_CHECKING:

32 try:

33 from lsst.afw.image import ExposureSummaryStats as LegacyExposureSummaryStats

34 except ImportError:

35 type LegacyExposureSummaryStats = Any # type: ignore[no-redef]

38def _default_corners() -> tuple[float, float, float, float]:

39 return (math.nan, math.nan, math.nan, math.nan)

42def _is_empty(value: Any) -> bool:

43 """Return whether a summary-statistic value is unset.

45 A value counts as unset if it is NaN, or an empty sequence, or a sequence

46 whose entries are all unset. Such fields carry no information and can be

47 dropped when converting to or from the legacy representation, allowing the

48 two representations to define different sets of fields as long as the

49 fields they do not share are empty.

50 """

51 if isinstance(value, (list, tuple)):

52 return all(_is_empty(item) for item in value)

53 return isinstance(value, float) and math.isnan(value)

56@final

57class ObservationSummaryStats(ArchiveTree, DescribableMixin):

58 """Various statistics obtained from a single observation."""

60 SCHEMA_NAME: ClassVar[str] = "observation_summary_stats"

61 SCHEMA_VERSION: ClassVar[str] = "1.0.0.dev0"

62 MIN_READ_VERSION: ClassVar[int] = 1

63 PUBLIC_TYPE: ClassVar[type] # Assigned after class construction.

65 psfSigma: float = pydantic.Field(math.nan, description="PSF determinant radius (pixels).")

67 psfArea: float = pydantic.Field(math.nan, description="PSF effective area (pixels**2).")

69 psfIxx: float = pydantic.Field(math.nan, description="PSF shape Ixx (pixels**2).")

71 psfIyy: float = pydantic.Field(math.nan, description="PSF shape Iyy (pixels**2).")

73 psfIxy: float = pydantic.Field(math.nan, description="PSF shape Ixy (pixels**2).")

75 ra: float = pydantic.Field(math.nan, description="Bounding box center Right Ascension (degrees).")

77 dec: float = pydantic.Field(math.nan, description="Bounding box center Declination (degrees).")

79 pixelScale: float = pydantic.Field(math.nan, description="Measured detector pixel scale (arcsec/pixel).")

81 zenithDistance: float = pydantic.Field(

82 math.nan, description="Bounding box center zenith distance (degrees)."

83 )

85 expTime: float = pydantic.Field(math.nan, description="Exposure time of the exposure (seconds).")

87 zeroPoint: float = pydantic.Field(math.nan, description="Mean zeropoint in detector (mag).")

89 skyBg: float = pydantic.Field(math.nan, description="Average sky background (ADU).")

91 skyNoise: float = pydantic.Field(math.nan, description="Average sky noise (ADU).")

93 meanVar: float = pydantic.Field(math.nan, description="Mean variance of the weight plane (ADU**2).")

95 raCorners: tuple[float, float, float, float] = pydantic.Field(

96 default_factory=_default_corners, description="Right Ascension of bounding box corners (degrees)."

97 )

99 decCorners: tuple[float, float, float, float] = pydantic.Field(

100 default_factory=_default_corners, description="Declination of bounding box corners (degrees)."

101 )

102

103 psfAdaptiveThresholdValue: float = pydantic.Field(

104 math.nan,

105 description="Threshold value used in the adaptive threshold detection pass for PSF modelling.",

106 )

107

108 psfAdaptiveIncludeThresholdMultiplier: float = pydantic.Field(

109 math.nan,

110 description="Threshold multiplier used in the adaptive threshold detection pass for PSF modelling.",

111 )

112

113 nShapeletsStar: int = pydantic.Field(

114 0,

115 description="Number of sources used in the shapelet decomposition.",

116 )

117

118 shapeletsOnlyIqScore: float = pydantic.Field(

119 math.nan,

120 description=(

121 "The dimensionless image quality score as determined from the shapelets decomposition "

122 "that includes power only from the non-atmospheric decomposition coefficients. The "

123 "score spans the range [0.0, 1.0] with lower values indicating better image quality."

124 ),

125 )

126

127 shapeletsIqScore: float = pydantic.Field(

128 math.nan,

129 description=(

130 "The dimensionless image quality score as determined from the shapelets decomposition "

131 "that includes power from the median centroid offset between those used in the decomposition "

132 "and those of the centroid slot in addition to non-atmospheric decomposition coefficients. "

133 "The score spans the range [0.0, 1.0] with lower values indicating better image quality."

134 ),

135 )

136

137 shapeletsCoeffs: tuple[float, ...] = pydantic.Field(

138 default_factory=tuple,

139 description="Coefficients from the PSF star shapelet decomposition.",

140 )

141

142 centroidDiffShapeletsVsSlotMedian: float = pydantic.Field(

143 math.nan,

144 description=(

145 "Median centroid difference (sqrt((slot_x - shapelet_x)**2 + (slot_y - shapelet_y)**2)) for "

146 "sources used in the shapelet decomposition (pixels)."

147 ),

148 )

149

150 shapeletsStarEMedian: float = pydantic.Field(

151 math.nan,

152 description=(

153 "Median ellipticity (sqrt(starE1**2.0 + starE2**2.0)) of the sources used in the "

154 "shapelet decomposition."

155 ),

156 )

157

158 shapeletsStarUnNormalizedEMedian: float = pydantic.Field(

159 math.nan,

160 description=(

161 "Median un-normalized ellipticity (sqrt((starXX - starYY)**2.0 + (2.0*starXY)**2.0)) "

162 "of the sources used in the shapelet decomposition (pixel**2)."

163 ),

164 )

165

166 refCatSourceDensity: float = pydantic.Field(

167 math.nan,

168 description=(

169 "Source density for the detector region as computed from the loaded reference catalog "

170 "(number per degrees**2)."

171 ),

172 )

173

174 astromOffsetMean: float = pydantic.Field(math.nan, description="Astrometry match offset mean.")

175

176 astromOffsetStd: float = pydantic.Field(math.nan, description="Astrometry match offset stddev.")

177

178 nPsfStar: int = pydantic.Field(0, description="Number of stars used for psf model.")

179

180 psfStarDeltaE1Median: float = pydantic.Field(

181 math.nan, description="Psf stars median E1 residual (starE1 - psfE1)."

182 )

183

184 psfStarDeltaE2Median: float = pydantic.Field(

185 math.nan, description="Psf stars median E2 residual (starE2 - psfE2)."

186 )

187

188 psfStarDeltaE1Scatter: float = pydantic.Field(

189 math.nan, description="Psf stars MAD E1 scatter (starE1 - psfE1)."

190 )

191

192 psfStarDeltaE2Scatter: float = pydantic.Field(

193 math.nan, description="Psf stars MAD E2 scatter (starE2 - psfE2)."

194 )

195

196 psfStarDeltaSizeMedian: float = pydantic.Field(

197 math.nan, description="Psf stars median size residual (starSize - psfSize)."

198 )

199

200 psfStarDeltaSizeScatter: float = pydantic.Field(

201 math.nan, description="Psf stars MAD size scatter (starSize - psfSize)."

202 )

203

204 psfStarScaledDeltaSizeScatter: float = pydantic.Field(

205 math.nan, description="Psf stars MAD size scatter scaled by psfSize**2."

206 )

207

208 psfTraceRadiusDelta: float = pydantic.Field(

209 math.nan,

210 description=(

211 "Delta (max - min) of the model psf trace radius values evaluated on a grid of "

212 "unmasked pixels (pixels)."

213 ),

214 )

215

216 psfApFluxDelta: float = pydantic.Field(

217 math.nan,

218 description=(

219 "Delta (max - min) of the model psf aperture flux (with aperture radius of max(2, 3*psfSigma)) "

220 "values evaluated on a grid of unmasked pixels."

221 ),

222 )

223

224 psfApCorrSigmaScaledDelta: float = pydantic.Field(

225 math.nan,

226 description=(

227 "Delta (max - min) of the psf flux aperture correction factors scaled (divided) by the "

228 "psfSigma evaluated on a grid of unmasked pixels."

229 ),

230 )

231

232 maxDistToNearestPsf: float = pydantic.Field(

233 math.nan,

234 description="Maximum distance of an unmasked pixel to its nearest model psf star (pixels).",

235 )

236

237 starEMedian: float = pydantic.Field(

238 math.nan,

239 description=(

240 "Median ellipticity (sqrt(starE1**2.0 + starE2**2.0)) of the stars used in the PSF model."

241 ),

242 )

243

244 starUnNormalizedEMedian: float = pydantic.Field(

245 math.nan,

246 description=(

247 "Median un-normalized ellipticity (sqrt((starXX - starYY)**2.0 + "

248 "(2.0*starXY)**2.0)) of the stars used in the PSF model."

249 ),

250 )

251

252 starComa1Median: float = pydantic.Field(

253 math.nan,

254 description=(

255 "Coma-like higher-order moment combination: median M30 + M12 of the stars used in the PSF model."

256 ),

257 )

258

259 starComa2Median: float = pydantic.Field(

260 math.nan,

261 description=(

262 "Coma-like higher-order moment combination: median M21 + M03 of the stars used in the PSF model."

263 ),

264 )

265

266 starTrefoil1Median: float = pydantic.Field(

267 math.nan,

268 description=(

269 "Trefoil-like higher-order moment combination: median M30 - 3*M12 "

270 "of the stars used in the PSF model."

271 ),

272 )

273

274 starTrefoil2Median: float = pydantic.Field(

275 math.nan,

276 description=(

277 "Trefoil-like higher-order moment combination: median 3*M21 - M03 "

278 "of the stars used in the PSF model."

279 ),

280 )

281

282 starKurtosisMedian: float = pydantic.Field(

283 math.nan,

284 description=(

285 "Kurtosis-like higher-order moment combination: median M40 + 2*M22 + M04 "

286 "of the stars used in the PSF model."

287 ),

288 )

289

290 starE41Median: float = pydantic.Field(

291 math.nan,

292 description=(

293 "Fourth-order ellipticity-like higher-order moment combination: median M40 - M04 "

294 "of the stars used in the PSF model."

295 ),

296 )

297

298 starE42Median: float = pydantic.Field(

299 math.nan,

300 description=(

301 "Fourth-order ellipticity-like higher-order moment combination: median 2*(M31 + M13) "

302 "of the stars used in the PSF model."

303 ),

304 )

305

306 effTime: float = pydantic.Field(

307 math.nan,

308 description="Effective exposure time calculated from psfSigma, skyBg, and zeroPoint (seconds).",

309 )

310

311 effTimePsfSigmaScale: float = pydantic.Field(

312 math.nan, description="PSF scaling of the effective exposure time."

313 )

314

315 effTimeSkyBgScale: float = pydantic.Field(

316 math.nan, description="Sky background scaling of the effective exposure time."

317 )

318

319 effTimeZeroPointScale: float = pydantic.Field(

320 math.nan, description="Zeropoint scaling of the effective exposure time."

321 )

322

323 magLim: float = pydantic.Field(

324 math.nan,

325 description=(

326 "Magnitude limit at fixed SNR (default SNR=5) calculated from psfSigma, skyBg,"

327 " zeroPoint, and readNoise."

328 ),

329 )

330

331 psfTE1e1: float = pydantic.Field(

332 math.nan,

333 description=(

334 "Per-exposure TE1e1 ~ <de1 de1> of PSF residual ellipticity, averaged over theta "

335 "[0,1] arcmin via treecorr KK correlation. Dimensionless; used to form the "

336 "full-survey TE1 metric."

337 ),

338 )

339

340 psfTE1e2: float = pydantic.Field(

341 math.nan,

342 description=(

343 "Per-exposure TE1e2 ~ <de2 de2> of PSF residual ellipticity, averaged over theta "

344 "[0,1] arcmin via treecorr KK correlation. Dimensionless; used to form the "

345 "full-survey TE1 metric."

346 ),

347 )

348

349 psfTE1ex: float = pydantic.Field(

350 math.nan,

351 description=(

352 "Per-exposure TE1ex ~ <de1 de2> of PSF residual ellipticity, averaged over theta "

353 "[0,1] arcmin via treecorr KK correlation. Dimensionless; used to form the "

354 "full-survey TE1 metric."

355 ),

356 )

357

358 psfTE2e1: float = pydantic.Field(

359 math.nan,

360 description=(

361 "Per-exposure TE2e1 ~ <de1 de1> of PSF residual ellipticity, averaged over theta "

362 "[5,100] arcmin via treecorr KK correlation. Dimensionless; used to form the "

363 "full-survey TE2 metric."

364 ),

365 )

366

367 psfTE2e2: float = pydantic.Field(

368 math.nan,

369 description=(

370 "Per-exposure TE2e2 ~ <de2 de2> of PSF residual ellipticity, averaged over theta "

371 "[5,100] arcmin via treecorr KK correlation. Dimensionless; used to form the "

372 "full-survey TE2 metric."

373 ),

374 )

375

376 psfTE2ex: float = pydantic.Field(

377 math.nan,

378 description=(

379 "Per-exposure TE2ex ~ <de1 de2> of PSF residual ellipticity, averaged over theta "

380 "[5,100] arcmin via treecorr KK correlation. Dimensionless; used to form the "

381 "full-survey TE2 metric."

382 ),

383 )

384

385 psfTE3e1: float = pydantic.Field(

386 math.nan,

387 description=(

388 "Per-exposure median-over-CCDs of TE3e1 ~ <de1 de1> of PSF residual ellipticity, "

389 "where each CCD uses theta within [0,5] arcmin bins. Dimensionless; downstream "

390 "pipelines take the 85th percentile over images to evaluate TE3."

391 ),

392 )

393

394 psfTE3e2: float = pydantic.Field(

395 math.nan,

396 description=(

397 "Per-exposure median-over-CCDs of TE3e2 ~ <de2 de2> of PSF residual ellipticity, "

398 "where each CCD uses theta within [0,5] arcmin bins. Dimensionless; downstream "

399 "pipelines take the 85th percentile over images to evaluate TE3."

400 ),

401 )

402

403 psfTE3ex: float = pydantic.Field(

404 math.nan,

405 description=(

406 "Per-exposure median-over-CCDs of TE3ex ~ <de1 de2> of PSF residual ellipticity, "

407 "where each CCD uses theta within [0,5] arcmin bins. Dimensionless; downstream "

408 "pipelines take the 85th percentile over images to evaluate TE3."

409 ),

410 )

411

412 psfTE4e1: float = pydantic.Field(

413 math.nan,

414 description=(

415 "Per-exposure median-over-CCDs of TE4e1 ~ <de1 de1> of PSF residual ellipticity, "

416 "where each CCD uses theta within [5,20] arcmin bins. Dimensionless; downstream "

417 "pipelines take the 85th percentile over images to evaluate TE4."

418 ),

419 )

420

421 psfTE4e2: float = pydantic.Field(

422 math.nan,

423 description=(

424 "Per-exposure median-over-CCDs of TE4e2 ~ <de2 de2> of PSF residual ellipticity, "

425 "where each CCD uses theta within [5,20] arcmin bins. Dimensionless; downstream "

426 "pipelines take the 85th percentile over images to evaluate TE4."

427 ),

428 )

429

430 psfTE4ex: float = pydantic.Field(

431 math.nan,

432 description=(

433 "Per-exposure median-over-CCDs of TE4ex ~ <de1 de2> of PSF residual ellipticity, "

434 "where each CCD uses theta within [5,20] arcmin bins. Dimensionless; downstream "

435 "pipelines take the 85th percentile over images to evaluate TE4."

436 ),

437 )

438

439 def __eq__(self, other: object) -> bool:

440 if not isinstance(other, ObservationSummaryStats): 440 ↛ 441line 440 didn't jump to line 441 because the condition on line 440 was never true

441 return NotImplemented

442 for name in ObservationSummaryStats.model_fields:

443 if name in ArchiveTree.model_fields:

444 # Parent class fields, not summary statistics.

445 continue

446 a = getattr(self, name)

447 b = getattr(other, name)

448 if isinstance(a, tuple) and isinstance(b, tuple):

449 if len(a) != len(b): 449 ↛ 450line 449 didn't jump to line 450 because the condition on line 449 was never true

450 return False

451 for ai, bi in zip(a, b):

452 if ai != bi and not (math.isnan(ai) and math.isnan(bi)): 452 ↛ 453line 452 didn't jump to line 453 because the condition on line 452 was never true

453 return False

454 elif a != b and not (math.isnan(a) and math.isnan(b)):

455 return False

456 return True

457

458 def _describe(self, options: DescribeOptions = DescribeOptions(), /) -> Report:

459 """Return a `Report` describing these summary statistics.

460

461 Parameters

462 ----------

463 options : `DescribeOptions`, optional

464 Rendering options. `DescribeOptions.brief` reports how many

465 statistics are set instead of listing them.

466

467 Notes

468 -----

469 Which statistics an observation carries depends on which pipeline

470 steps produced it, and most of the many fields are unset in any given

471 one, so the report covers whichever of them hold a value rather than a

472 fixed selection.

473 """

474 # Everything this class adds to ArchiveTree is a statistic; what it

475 # inherits from it is serialization plumbing.

476 names = sorted(set(type(self).model_fields) - set(ArchiveTree.model_fields))

477 present = [(name, value) for name in names if not _is_empty(value := getattr(self, name))]

478 summary = f"ObservationSummaryStats({len(present)} of {len(names)} statistics set)"

479 if options.brief:

480 # The count is worth stating on its own where the values are not

481 # listed; alongside them it would only restate what is visible.

482 return Report(

483 type_name="ObservationSummaryStats",

484 summary=summary,

485 fields=[

486 ReportField(

487 label="statistics set",

488 value=f"{len(present)} of {len(names)}",

489 role=FieldRole.DERIVED,

490 )

491 ],

492 )

493 scalars: list[tuple[str, Any]] = []

494 fields: list[ReportField] = []

495 for name, value in present:

496 if isinstance(value, list | tuple):

497 # Too long to pack alongside the scalars; give it a line.

498 fields.append(ReportField(label=name, value=value, role=FieldRole.DERIVED))

499 else:

500 scalars.append((name, value))

501 return Report(

502 type_name="ObservationSummaryStats",

503 summary=summary,

504 fields=fields,

505 value_groups=[ReportValueGroup(values=scalars)] if scalars else [],

506 )

507

508 def __str__(self) -> str:

509 # pydantic's __str__ precedes the mixin's in the MRO and spells out all

510 # of the fields, nearly all of which are unset. ``repr`` keeps that

511 # exhaustive form, since it is the one that round-trips; ``str`` is the

512 # readable one, so it takes the report's summary.

513 return DescribableMixin.__str__(self)

514

515 def deserialize(self, archive: InputArchive[Any], **kwargs: Any) -> Self:

516 """Extract this object from an archive.

517

518 Parameters

519 ----------

520 archive

521 Archive to read from.

522 **kwargs

523 Optional parameters. Not supported by this class.

524 """

525 if kwargs: 525 ↛ 526line 525 didn't jump to line 526 because the condition on line 525 was never true

526 raise InvalidParameterError(

527 f"Unrecognized parameters for ObservationSummaryStats: {set(kwargs.keys())}."

528 )

529 # The model we want *is* the ArchiveTree.

530 return self

531

532 def serialize(self, archive: OutputArchive[Any]) -> Self:

533 """Write this object to an archive.

534

535 Parameters

536 ----------

537 archive

538 Archive to write to.

539 """

540 # Copy so write-time overrides (metadata, butler_info) applied

541 # by serialize_root do not mutate this object.

542 return self.model_copy(deep=True)

543

544 @classmethod

545 def from_legacy(cls, exposure_summary_stats: LegacyExposureSummaryStats) -> Self:

546 """Return an `ObservationSummaryStats` from a legacy

547 `lsst.afw.image.ExposureSummaryStats`.

548

549 Parameters

550 ----------

551 exposure_summary_stats

552 Legacy exposure summary statistics to convert.

553

554 Notes

555 -----

556 Legacy fields that are empty (NaN) are dropped, so a legacy struct that

557 carries fields unknown to this class is accepted as long as those

558 fields are empty. A legacy field that holds a real value but is

559 unknown here raises `ValueError`, since dropping it would lose data.

560 """

561 known_fields = set(cls.model_fields)

562 kwargs: dict[str, Any] = {}

563 for name, value in dataclasses.asdict(exposure_summary_stats).items():

564 if _is_empty(value):

565 continue

566 # Strip version since it carries no information (it is always 0

567 # in all our existing files) and this class uses explicit schema

568 # versioning.

569 if name == "version":

570 continue

571 if name not in known_fields:

572 raise ValueError(

573 f"Legacy field {name!r} has a value ({value!r}) but is not known to "

574 f"ObservationSummaryStats."

575 )

576 kwargs[name] = value

577 return cls.model_validate(kwargs)

578

579 def to_legacy(self) -> LegacyExposureSummaryStats:

580 """Convert to an `lsst.afw.image.ExposureSummaryStats` instance.

581

582 Notes

583 -----

584 Empty (NaN) fields are not passed to the legacy struct, so fields

585 defined here that are unknown to the installed version of

586 `~lsst.afw.image.ExposureSummaryStats` are dropped when empty. A field

587 that holds a real value but is unknown to the legacy struct raises

588 `ValueError`, since dropping it would lose data.

589 """

590 from lsst.afw.image import ExposureSummaryStats as LegacyExposureSummaryStats

591

592 legacy_fields = {field.name for field in dataclasses.fields(LegacyExposureSummaryStats)}

593 kwargs: dict[str, Any] = {}

594 for name, info in ObservationSummaryStats.model_fields.items():

595 if name in ArchiveTree.model_fields:

596 # Parent class fields, not summary statistics.

597 continue

598 value = getattr(self, name)

599 if _is_empty(value):

600 continue

601 if name not in legacy_fields: 601 ↛ 602line 601 didn't jump to line 602 because the condition on line 601 was never true

602 raise ValueError(

603 f"Field {name!r} has a value ({value!r}) but is not supported by this "

604 f"version of lsst.afw.image.ExposureSummaryStats."

605 )

606 # Doing this in general is hard, so we handle the fields that we

607 # know about and raise if somebody adds a field with a new type

608 # without updating this function.

609 if info.annotation in (float, int): 609 ↛ 611line 609 didn't jump to line 611 because the condition on line 609 was always true

610 kwargs[name] = value

611 elif get_origin(info.annotation) is tuple:

612 kwargs[name] = list(value)

613 else:

614 raise NotImplementedError(f"Unsupported field type: {info.annotation}.")

615 return LegacyExposureSummaryStats(**kwargs)

616

617

618# Can not assign to itself in construction so assign now.

619ObservationSummaryStats.PUBLIC_TYPE = ObservationSummaryStats

Coverage for python/lsst/images/_observation_summary_stats.py: 47%

155 statements