Coverage for python/lsst/images/serialization/_common.py: 54%
126 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-30 08:38 +0000
1# This file is part of lsst-images.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12from __future__ import annotations
14__all__ = (
15 "SCHEMA_URL_BASE",
16 "ArchiveAccessRequiredError",
17 "ArchiveReadError",
18 "ArchiveTree",
19 "ButlerInfo",
20 "DevelopmentSchemaWarning",
21 "InvalidComponentError",
22 "InvalidParameterError",
23 "JsonRef",
24 "MetadataValue",
25 "OpaqueArchiveMetadata",
26 "is_development_version",
27 "no_header_updates",
28 "warn_for_development_schemas",
29)
31import operator
32import warnings
33from abc import ABC, abstractmethod
34from collections.abc import Iterator
35from typing import TYPE_CHECKING, Any, ClassVar, Protocol, Self
37import astropy.table
38import astropy.units
39import pydantic
40from packaging.version import Version
42from .._geom import Box
43from ..utils import is_none
45try:
46 from lsst.daf.butler import DatasetProvenance, SerializedDatasetRef
47except ImportError:
48 type DatasetProvenance = Any # type: ignore[no-redef]
49 type SerializedDatasetRef = Any # type: ignore[no-redef]
51if TYPE_CHECKING:
52 import astropy.io.fits
54 from ._input_archive import InputArchive
57type MetadataValue = (
58 pydantic.StrictInt | pydantic.StrictFloat | pydantic.StrictStr | pydantic.StrictBool | None
59)
61SCHEMA_URL_BASE = "https://images.lsst.io/schemas"
62"""Base for the schema URLs of this package's own schemas, used as
63``{SCHEMA_URL_BASE}/{name}-{version}``.
65External packages providing their own schemas override
66`~lsst.images.serialization.ArchiveTree.SCHEMA_URL_BASE` instead, so their
67schema URLs are minted under a site they control.
68"""
71class ButlerInfo(pydantic.BaseModel):
72 """Information about a butler dataset."""
74 dataset: SerializedDatasetRef
75 provenance: DatasetProvenance = pydantic.Field(default_factory=DatasetProvenance)
78class JsonRef(pydantic.BaseModel, serialize_by_alias=True):
79 """Pydantic model for JSON Reference / Pointer (IETF RFC 6901).
81 Notes
82 -----
83 This model does not do any of the escaping or special-character
84 interpretation required by the spec; it assumes that's already been done,
85 so its job is *just* putting a ``$ref`` field inside another model.
86 """
88 ref: str = pydantic.Field(alias="$ref")
91class ArchiveTree(
92 pydantic.BaseModel, ABC, ser_json_inf_nan="constants", ser_json_bytes="base64", val_json_bytes="base64"
93):
94 """An intermediate base class of `pydantic.BaseModel` that should be used
95 for all objects that may be used as the top-level tree models written to
96 archives.
98 See :ref:`lsst.images-schema-versioning` for how the ``SCHEMA_NAME`` /
99 ``SCHEMA_VERSION`` / ``MIN_READ_VERSION`` constants and the
100 ``schema_version`` / ``min_read_version`` / ``schema_url`` fields are used.
101 """
103 SCHEMA_NAME: ClassVar[str]
104 SCHEMA_VERSION: ClassVar[str]
105 MIN_READ_VERSION: ClassVar[int]
106 SCHEMA_URL_BASE: ClassVar[str] = SCHEMA_URL_BASE
107 """Base for this schema's URL, as ``{SCHEMA_URL_BASE}/{name}-{version}``.
109 External packages providing their own schemas should override this (once,
110 on a shared intermediate base class) so their schema URLs are minted
111 under a documentation site they control rather than images.lsst.io."""
113 PUBLIC_TYPE: ClassVar[type]
114 """In-memory Python type produced by this tree's ``deserialize`` (e.g.
115 `dict` for a mapping return). Declared explicitly by each concrete
116 subclass and surfaced by
117 `~lsst.images.serialization.public_type_for_schema`."""
119 # These mirror the SCHEMA_VERSION / MIN_READ_VERSION class constants and
120 # are never passed on construction, so repr omits them; they say nothing
121 # about the object that its type does not already say.
122 schema_version: str = pydantic.Field(
123 default="1.0.0",
124 description="Data-model schema version of this tree (major.minor.patch).",
125 repr=False,
126 )
127 min_read_version: int = pydantic.Field(
128 default=1,
129 description="Smallest reader major that can interpret this tree.",
130 repr=False,
131 )
132 metadata: dict[str, MetadataValue] = pydantic.Field(
133 default_factory=dict, description="Additional unstructured metadata.", exclude_if=operator.not_
134 )
135 butler_info: ButlerInfo | None = pydantic.Field(
136 default=None,
137 description="Information about the butler dataset backed by this file.",
138 exclude_if=is_none,
139 )
140 # Populated by the archive machinery rather than by a caller, so repr
141 # omits it as well.
142 indirect: list[Any] = pydantic.Field(
143 default_factory=list,
144 description="Serialized nested objects that may be saved or read more than once.",
145 exclude_if=operator.not_,
146 repr=False,
147 )
149 @pydantic.computed_field(description="Canonical schema URL for this tree.") # type: ignore[prop-decorator]
150 @property
151 def schema_url(self) -> str:
152 """Return the schema URL of this tree's class.
154 Computed from ``SCHEMA_NAME`` and ``SCHEMA_VERSION`` ClassVars.
155 """
156 cls = type(self)
157 return f"{cls.SCHEMA_URL_BASE}/{cls.SCHEMA_NAME}-{cls.SCHEMA_VERSION}"
159 @pydantic.model_validator(mode="after")
160 def _check_and_normalize_schema_version(self) -> Self:
161 """Validate and normalise the schema version fields.
163 Compares the on-tree ``schema_version`` / ``min_read_version`` against
164 the in-code values from the subclass's ClassVars; raises if
165 incompatible, otherwise normalises the fields to the in-code values.
166 """
167 cls = type(self)
168 # ArchiveTree itself is abstract (deserialize is @abstractmethod).
169 # Subclasses that haven't yet declared SCHEMA_NAME are skipped — this
170 # matters during incremental rollout and remains a safe no-op
171 # afterwards (a class-invariants test ensures every concrete subclass
172 # has the constants).
173 if not hasattr(cls, "SCHEMA_NAME"): 173 ↛ 174line 173 didn't jump to line 174 because the condition on line 173 was never true
174 return self
175 _check_compat(
176 cls.SCHEMA_NAME,
177 self.schema_version,
178 self.min_read_version,
179 cls.SCHEMA_VERSION,
180 )
181 if self.schema_version != cls.SCHEMA_VERSION:
182 self.schema_version = cls.SCHEMA_VERSION
183 if self.min_read_version != cls.MIN_READ_VERSION: 183 ↛ 184line 183 didn't jump to line 184 because the condition on line 183 was never true
184 self.min_read_version = cls.MIN_READ_VERSION
185 return self
187 @classmethod
188 def __pydantic_init_subclass__(cls, **kwargs: Any) -> None:
189 """Inject ``$id`` and ``title`` into the subclass's JSON Schema, and
190 register the subclass in the schema-name registry.
192 Populates ``model_config['json_schema_extra']`` with values derived
193 from the subclass's ``SCHEMA_NAME`` / ``SCHEMA_VERSION`` ClassVars,
194 then registers the subclass so it can be looked up by schema name.
195 Subclasses that haven't declared the ClassVars are skipped.
196 """
197 super().__pydantic_init_subclass__(**kwargs)
198 name = cls.__dict__.get("SCHEMA_NAME")
199 version = cls.__dict__.get("SCHEMA_VERSION")
200 if name is None or version is None:
201 return
202 json_schema_extra = cls.model_config.get("json_schema_extra") or {}
203 if isinstance(json_schema_extra, dict): 203 ↛ 215line 203 didn't jump to line 215 because the condition on line 203 was always true
204 existing = dict(json_schema_extra)
205 # Always override: a subclass of a concrete schema (e.g.
206 # visit_image subclassing masked_image) inherits its parent's
207 # already-stamped values through the merged model_config, and
208 # this hook only runs when the subclass declares its own
209 # SCHEMA_NAME / SCHEMA_VERSION for these to be derived from.
210 existing["$id"] = f"{cls.SCHEMA_URL_BASE}/{name}-{version}"
211 existing["title"] = name
212 cls.model_config = {**cls.model_config, "json_schema_extra": existing}
213 # Local import to avoid the _io -> _common circular dependency at
214 # module load time.
215 from ._io import register_schema_class
217 register_schema_class(cls)
219 @abstractmethod
220 def deserialize(self, archive: InputArchive[Any], **kwargs: Any) -> Any:
221 """Return the in-memory object that was serialized to this tree.
223 Parameters
224 ----------
225 archive
226 The input archive to read from.
227 **kwargs
228 Additional keyword arguments specific to this type.
230 Raises
231 ------
232 ~lsst.images.serialization.InvalidParameterError
233 Raised for unsupported ``**kwargs``.
235 Notes
236 -----
237 Subclass implementations may take additional keyword-only arguments.
238 Callers that invoke this method without knowing what those might be
239 should catch `TypeError` and re-raise as
240 `~lsst.images.serialization.InvalidParameterError` if they pass
241 additional keyword arguments.
242 """
243 raise NotImplementedError()
245 def deserialize_component(self, component: str, archive: InputArchive[Any], **kwargs: Any) -> Any:
246 """Return a component in-memory object that was serialized to this
247 tree.
249 Parameters
250 ----------
251 component
252 Name of the component to read.
253 archive
254 The input archive to read from.
255 **kwargs
256 Additional keyword arguments specific to this type.
258 Raises
259 ------
260 ~lsst.images.serialization.InvalidComponentError
261 Raise if ``component`` is not recognized.
262 ~lsst.images.serialization.InvalidParameterError
263 Raised for unsupported ``**kwargs``.
265 Notes
266 -----
267 The default implementation for this method tries to get an attribute
268 with the component's name from ``self``, and then:
270 - returns `None` if it is `None`;
271 - calls `deserialize` on that object if it is also an
272 `~lsst.images.serialization.ArchiveTree`;
273 - returns it directly otherwise.
275 If there is no such attribute, it raises
276 `~lsst.images.serialization.InvalidComponentError`.
278 ``**kwargs`` are forwarded to component `deserialize` methods, but
279 are otherwise not checked. Subclasses are generally expected to
280 implement this method to do that checking and handle any components
281 for which the other will not work, and then delegate to `super` at
282 the end.
283 """
284 try:
285 component_model = getattr(self, component)
286 except AttributeError:
287 raise InvalidComponentError(
288 f"Component {component!r} is not recognized by {type(self).__name__}."
289 ) from None
290 if component_model is None:
291 return None
292 if isinstance(component_model, ArchiveTree):
293 return component_model.deserialize(archive, **kwargs)
294 return component_model
297class DevelopmentSchemaWarning(UserWarning):
298 """Warning that a file is being written with a development schema."""
301def is_development_version(version: str) -> bool:
302 """Return whether a schema version string is a PEP 440 development release.
304 Parameters
305 ----------
306 version
307 Schema version string, e.g. ``1.0.0`` or ``1.0.0.dev0``.
308 """
309 return Version(version).is_devrelease
312def _iter_archive_trees(obj: Any) -> Iterator[ArchiveTree]:
313 """Yield every `ArchiveTree` embedded in a serialized tree."""
314 if isinstance(obj, ArchiveTree):
315 yield obj
316 if isinstance(obj, pydantic.BaseModel):
317 for field_name in type(obj).model_fields:
318 yield from _iter_archive_trees(getattr(obj, field_name))
319 elif isinstance(obj, list | tuple):
320 for item in obj:
321 yield from _iter_archive_trees(item)
322 elif isinstance(obj, dict):
323 for value in obj.values():
324 yield from _iter_archive_trees(value)
327def warn_for_development_schemas(root: ArchiveTree) -> None:
328 """Emit a `DevelopmentSchemaWarning` if a serialized tree contains any
329 schema still in development.
331 Parameters
332 ----------
333 root
334 Top-level serialized tree about to be written.
335 """
336 developing = sorted(
337 {
338 tree.schema_url
339 for tree in _iter_archive_trees(root)
340 if is_development_version(type(tree).SCHEMA_VERSION)
341 }
342 )
343 if developing:
344 warnings.warn(
345 "Writing a file with development schema(s) "
346 f"{', '.join(developing)}; such files are not for production and "
347 "may not remain readable.",
348 DevelopmentSchemaWarning,
349 stacklevel=3,
350 )
353class ArchiveReadError(RuntimeError):
354 """Exception raised when the contents of an archive cannot be read."""
357class InvalidParameterError(ArchiveReadError):
358 """Exception raised by `ArchiveTree.deserialize` or
359 `ArchiveTree.deserialize_component` when passed an invalid keyword
360 argument.
361 """
364class InvalidComponentError(ArchiveReadError):
365 """Exception `ArchiveTree.deserialize_component` when passed an invalid
366 component name.
367 """
370class ArchiveAccessRequiredError(RuntimeError):
371 """Exception raised when a deserialization needs data from the file.
373 Raised by all data-access methods of
374 `~lsst.images.serialization.DetachedArchive`, signaling that the
375 requested object cannot be deserialized from the JSON tree alone.
377 Notes
378 -----
379 This deliberately does not inherit from `ArchiveReadError`: it signals
380 that a live archive is required rather than that an archive is corrupt,
381 and it must not be swallowed by ``except ArchiveReadError`` handlers.
382 """
385class OpaqueArchiveMetadata(Protocol):
386 """Interface for opaque archive metadata.
388 In addition to implementing the methods defined here, all implementations
389 must be pickleable.
390 """
392 def copy(self) -> Self | None:
393 """Copy, reference, or discard metadata when its holding object is
394 copied.
395 """
396 ...
398 def subset(self, bbox: Box) -> Self | None:
399 """Copy, reference, or discard metadata when a subset of its its
400 holding object is extracted.
402 Parameters
403 ----------
404 bbox
405 Bounding box of the subset being extracted.
406 """
407 ...
410def no_header_updates(header: astropy.io.fits.Header) -> None:
411 """Do not make any modifications to the given FITS header.
413 Parameters
414 ----------
415 header
416 FITS header that is left unchanged.
417 """
420def _parse_major(version: str) -> int:
421 """Return the integer major component of a major.minor.patch string.
423 Raises
424 ------
425 ArchiveReadError
426 If ``version`` is not a non-empty string of the form
427 ``major.minor.patch`` with integer components.
428 """
429 if not isinstance(version, str) or not version: 429 ↛ 430line 429 didn't jump to line 430 because the condition on line 429 was never true
430 raise ArchiveReadError(f"Schema version {version!r} is not a non-empty string.")
431 head = version.split(".", 1)[0]
432 try:
433 return int(head)
434 except ValueError as exc:
435 raise ArchiveReadError(f"Schema version {version!r} has non-integer major.") from exc
438def _check_compat(
439 name: str,
440 on_disk_version: str,
441 on_disk_min_read: int,
442 in_code_version: str,
443) -> None:
444 """Raise `ArchiveReadError` if a tree written with the given
445 schema_version/min_read_version cannot be read by the current code.
447 See :ref:`lsst.images-schema-versioning` for the compatibility rule.
448 """
449 in_code_major = _parse_major(in_code_version)
450 if on_disk_min_read > in_code_major:
451 raise ArchiveReadError(
452 f"{name}: tree requires reader major >= {on_disk_min_read}; this release is {in_code_version}."
453 )
456def _check_format_version(name: str, on_disk: int, in_code: int) -> None:
457 """Raise `ArchiveReadError` if a backend file's container layout
458 version is newer than this release knows how to read.
459 """
460 if on_disk > in_code:
461 raise ArchiveReadError(
462 f"{name}: on-disk container format version {on_disk} is "
463 f"newer than this release ({in_code}); cannot read."
464 )