Coverage for python/lsst/summit/utils/consdbClient.py: 76%
205 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-02 02:35 -0800
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import logging
23import os
24from collections.abc import Mapping
25from dataclasses import dataclass
26from typing import Any
27from urllib.parse import quote, urlparse
29import numpy as np
30import requests
31from astropy.table import Column, Table
33__all__ = ["ConsDbClient", "FlexibleMetadataInfo", "getCcdVisitTableForDay", "getWideQuicklookTableForDay"]
36logger = logging.getLogger(__name__)
38# Timeouts in seconds applied to every request made by ConsDbClient.
39# The read timeout is generous because some queries are genuinely slow.
40DEFAULT_CONNECT_TIMEOUT: float = 10.0
41DEFAULT_READ_TIMEOUT: float = 600.0
44def _urljoin(*args: str) -> str:
45 """Join parts of a URL with slashes.
47 Does not do any quoting. Mostly to remove a level of list-making.
49 Parameters
50 ----------
51 *args : `str`
52 Each parameter is a URL part.
54 Returns
55 -------
56 url : `str`
57 The joined URL.
58 """
59 return "/".join(args)
62def _check_status(r: requests.Response) -> None:
63 """Check the status of an HTTP response and raise if an error.
65 Adds additional response information to the raise_for_status exception.
67 Parameters
68 ----------
69 r : `requests.Response`
70 The response to check.
72 Raises
73 ------
74 requests.HTTPError
75 Raised if a non-successful status is returned.
76 """
77 try:
78 r.raise_for_status()
79 except requests.HTTPError as e:
80 try:
81 # Use ``r`` rather than ``e.response``: raise_for_status always
82 # attaches this response, but its type is Response | None.
83 json_data = r.json()
84 e.add_note(str(json_data))
85 if "message" in json_data: 85 ↛ 89line 85 didn't jump to line 89 because the condition on line 85 was always true
86 e.add_note(f"\n\n{json_data['message']}")
87 except requests.JSONDecodeError:
88 pass
89 raise e
92def clean_url(resp: requests.Response, *args: Any, **kwargs: Any) -> requests.Response:
93 """Parse url from response and remove netloc portion.
95 Set new url in response and return response
97 Parameters
98 ----------
99 resp : `requests.Response`
100 The response that could contain a URL with tokens
101 """
102 url = urlparse(resp.url)
103 short_user = f"{url.username[:2]}***" if url.username is not None else ""
104 if url.password is not None:
105 pwd = url.password
106 short_pass = ":gt***" if pwd.startswith("gt-") else ":xxxxxxx"
107 else:
108 short_pass = ""
109 netloc = f"{short_user}{short_pass}@{url.hostname}"
110 resp.url = url._replace(netloc=netloc).geturl()
111 return resp
114@dataclass
115class FlexibleMetadataInfo:
116 """Description of a flexible metadata value.
118 Parameters
119 ----------
120 dtype : `str`
121 Data type of the flexible metadata value.
122 One of ``bool``, ``int``, ``float``, or ``str``.
123 doc : `str`
124 Documentation string.
125 unit : `str`, optional
126 Unit of the value.
127 ucd : `str`, optional
128 IVOA Unified Content Descriptor
129 (https://www.ivoa.net/documents/UCD1+/).
130 """
132 dtype: str
133 doc: str
134 unit: str | None = None
135 ucd: str | None = None
138class ConsDbClient:
139 """A client library for accessing the Consolidated Database.
141 This library provides a basic interface for using flexible metadata
142 (key/value pairs associated with observation ids from an observation
143 type table), determining the schema of ConsDB tables, querying the
144 ConsDB using a general SQL SELECT statement, and inserting into
145 ConsDB tables.
147 Parameters
148 ----------
149 url : `str` | `None`
150 Base URL of the Web service, defaults to the value of environment
151 variable ``LSST_CONSDB_PQ_URL`` (the location of the publish/query
152 service).
153 token : `str` | `None`
154 Authentication token for the RSP. The token must begin with "gt-".
155 If `None`, the ``ACCESS_TOKEN`` environment variable will be
156 checked and used if available.
157 connect_timeout : `float` | `None`
158 Seconds to wait for the connection to be established, defaults to
159 `DEFAULT_CONNECT_TIMEOUT`.
160 read_timeout : `float` | `None`
161 Seconds to wait between bytes sent by the server, defaults to
162 `DEFAULT_READ_TIMEOUT`. Pass `None` to wait indefinitely, which is
163 rarely what you want.
165 Notes
166 -----
167 This client is a thin layer over the publish/query Web service, which
168 avoids having a dependency on database drivers.
170 It enforces the return of query results as Astropy Tables.
171 """
173 def __init__(
174 self,
175 url: str | None = None,
176 token: str | None = None,
177 connect_timeout: float | None = DEFAULT_CONNECT_TIMEOUT,
178 read_timeout: float | None = DEFAULT_READ_TIMEOUT,
179 ):
180 self.session = requests.Session()
181 self.session.hooks["response"].append(clean_url)
182 self.connect_timeout = connect_timeout
183 self.read_timeout = read_timeout
185 if token is None:
186 token = os.environ.get("ACCESS_TOKEN") or None
187 if token is not None:
188 if token.startswith("gt-"):
189 logger.debug("Using authentication token from the ACCESS_TOKEN environment variable")
190 else:
191 token = None
192 logger.warning("Ignoring ACCESS_TOKEN environment variable: it does not start with `gt-`")
194 if token is not None:
195 if not token.startswith("gt-"):
196 raise ValueError("token must start with `gt-`.")
198 self.session.headers.update({"Authorization": f"Bearer {token}"})
200 if url is None: 200 ↛ 201line 200 didn't jump to line 201 because the condition on line 200 was never true
201 self.url = os.environ["LSST_CONSDB_PQ_URL"]
202 else:
203 self.url = url
204 self.url = self.url.rstrip("/")
206 @property
207 def timeout(self) -> tuple[float | None, float | None]:
208 """The ``(connect, read)`` timeout pair passed to requests
209 (`tuple` [ `float` | `None`, `float` | `None` ], read-only).
210 """
211 return (self.connect_timeout, self.read_timeout)
213 def _handle_get(self, url: str, query: dict[str, str | list[str]] | None = None) -> Any:
214 """Submit GET requests to the server.
216 Parameters
217 ----------
218 url : `str`
219 URL to GET.
220 query : `dict` [`str`, `str` | `list` [`str`]], optional
221 Query parameters to attach to the URL.
223 Raises
224 ------
225 requests.RequestException
226 Raised if any kind of connection error occurs.
227 requests.Timeout
228 Raised if the request exceeds the client timeout.
229 requests.HTTPError
230 Raised if a non-successful status is returned.
231 requests.JSONDecodeError
232 Raised if the result does not decode as JSON.
234 Returns
235 -------
236 result : `Any`
237 Result of decoding the Web service result content as JSON.
238 """
239 logger.debug(f"GET {url}")
240 response = self.session.get(url, params=query, timeout=self.timeout)
241 _check_status(response)
242 return response.json()
244 def _handle_post(self, url: str, data: dict[str, Any]) -> requests.Response:
245 """Submit POST requests to the server.
247 Parameters
248 ----------
249 url : `str`
250 URL to POST.
251 data : `dict` [`str`, `Any`]
252 Key/value pairs of data to POST.
254 Raises
255 ------
256 requests.RequestException
257 Raised if any kind of connection error occurs.
258 requests.Timeout
259 Raised if the request exceeds the client timeout.
260 requests.HTTPError
261 Raised if a non-successful status is returned.
263 Returns
264 -------
265 result : `requests.Response`
266 The raw Web service result object.
267 """
268 logger.debug(f"POST {url}: {data}")
269 response = self.session.post(url, json=data, timeout=self.timeout)
270 _check_status(response)
271 return response
273 @staticmethod
274 def compute_flexible_metadata_table_name(instrument: str, obs_type: str) -> str:
275 """Compute the name of a flexible metadata table.
277 Each instrument and observation type made with that instrument can
278 have a flexible metadata table. This function is useful when
279 issuing SQL queries, and it avoids a round-trip to the server.
281 Parameters
282 ----------
283 instrument : `str`
284 Name of the instrument (e.g. ``LATISS``).
285 obs_type : `str`
286 Name of the observation type (e.g. ``Exposure``).
288 Returns
289 -------
290 table_name : `str`
291 Name of the appropriate flexible metadata table.
292 """
293 return f"cdb_{instrument}.{obs_type}_flexdata"
295 @staticmethod
296 def compute_fixed_metadata_namespace(instrument: str) -> str:
297 """Compute the namespace for a fixed metadata table.
299 Each instrument has its own namespace in the ConsDB.
300 This function is useful when issuing SQL queries, and it avoids a
301 round-trip to the server.
303 Parameters
304 ----------
305 instrument : `str`
306 Name of the instrument (e.g. ``LATISS``).
308 Returns
309 -------
310 namespace_name : `str`
311 Name of the appropriate namespace
312 """
313 return f"cdb_{instrument}"
315 def add_flexible_metadata_key(
316 self,
317 instrument: str,
318 obs_type: str,
319 key: str,
320 dtype: str,
321 doc: str,
322 unit: str | None = None,
323 ucd: str | None = None,
324 ) -> requests.Response:
325 """Add a key to a flexible metadata table.
327 Parameters
328 ----------
329 instrument : `str`
330 Name of the instrument (e.g. ``LATISS``).
331 obs_type : `str`
332 Name of the observation type (e.g. ``Exposure``).
333 key : `str`
334 Name of the key to be added (must not already exist).
335 dtype : `str`
336 One of ``bool``, ``int``, ``float``, or ``str``.
337 doc : `str`
338 Documentation string for the key.
339 unit : `str`, optional
340 Unit for the value. Should be from the IVOA
341 (https://www.ivoa.net/documents/VOUnits/) or astropy.
342 ucd : `str`, optional
343 IVOA Unified Content Descriptor
344 (https://www.ivoa.net/documents/UCD1+/).
346 Returns
347 -------
348 response : `requests.Response`
349 HTTP response from the server, with 200 status for success.
351 Raises
352 ------
353 requests.RequestException
354 Raised if any kind of connection error occurs.
355 requests.HTTPError
356 Raised if a non-successful status is returned.
357 """
358 data = {"key": key, "dtype": dtype, "doc": doc}
359 if unit is not None: 359 ↛ 360line 359 didn't jump to line 360 because the condition on line 359 was never true
360 data["unit"] = unit
361 if ucd is not None: 361 ↛ 362line 361 didn't jump to line 362 because the condition on line 361 was never true
362 data["ucd"] = ucd
363 url = _urljoin(self.url, "flex", quote(instrument), quote(obs_type), "addkey")
364 return self._handle_post(url, data)
366 def get_flexible_metadata_keys(self, instrument: str, obs_type: str) -> dict[str, FlexibleMetadataInfo]:
367 """Retrieve the valid keys for a flexible metadata table.
369 Parameters
370 ----------
371 instrument : `str`
372 Name of the instrument (e.g. ``LATISS``).
373 obs_type : `str`
374 Name of the observation type (e.g. ``Exposure``).
376 Returns
377 -------
378 key_info : `dict` [ `str`, `FlexibleMetadataInfo` ]
379 Dict of keys and information values.
381 Raises
382 ------
383 requests.RequestException
384 Raised if any kind of connection error occurs.
385 requests.HTTPError
386 Raised if a non-successful status is returned.
387 """
388 url = _urljoin(self.url, "flex", quote(instrument), quote(obs_type), "schema")
389 result = self._handle_get(url)
390 return {key: FlexibleMetadataInfo(*value) for key, value in result.items()}
392 def get_flexible_metadata(
393 self, instrument: str, obs_type: str, obs_id: int, keys: list[str] | None = None
394 ) -> dict[str, Any]:
395 """Get the flexible metadata for an observation.
397 Parameters
398 ----------
399 instrument : `str`
400 Name of the instrument (e.g. ``LATISS``).
401 obs_type : `str`
402 Name of the observation type (e.g. ``Exposure``).
403 obs_id : `int`
404 Unique observation id.
405 keys : `list` [ `str` ], optional
406 List of keys to be retrieved; all if not specified.
408 Returns
409 -------
410 result_dict : `dict` [ `str`, `Any` ]
411 Dictionary of key/value pairs for the observation.
413 Raises
414 ------
415 requests.RequestException
416 Raised if any kind of connection error occurs.
417 requests.HTTPError
418 Raised if a non-successful status is returned.
419 """
420 url = _urljoin(
421 self.url,
422 "flex",
423 quote(instrument),
424 quote(obs_type),
425 "obs",
426 quote(str(obs_id)),
427 )
428 return self._handle_get(url, {"k": keys} if keys else None)
430 def get_all_metadata(
431 self, instrument: str, obs_type: str, obs_id: int, flex: bool = False
432 ) -> dict[str, Any]:
433 """Get all metadata for an observation.
435 Parameters
436 ----------
437 instrument : `str`
438 Name of the instrument (e.g. ``LATISS``).
439 obs_type : `str`
440 Name of the observation type (e.g. ``Exposure``).
441 obs_id : `int`
442 Unique observation id.
443 flex : `bool`
444 Include flexible metadata.
446 Returns
447 -------
448 result_dict : `dict` [ `str`, `Any` ]
449 Dictionary of key/value pairs for the observation.
451 Raises
452 ------
453 requests.RequestException
454 Raised if any kind of connection error occurs.
455 requests.HTTPError
456 Raised if a non-successful status is returned.
457 """
458 url = _urljoin(
459 self.url,
460 "query",
461 quote(instrument),
462 quote(obs_type),
463 "obs",
464 quote(str(obs_id)),
465 )
466 return self._handle_get(url, {"flex": "1"} if flex else None)
468 def insert_flexible_metadata(
469 self,
470 instrument: str,
471 obs_type: str,
472 obs_id: int,
473 values: dict[str, Any] | None = None,
474 *,
475 allow_update: bool = False,
476 **kwargs: Any,
477 ) -> requests.Response:
478 """Set flexible metadata values for an observation.
480 Parameters
481 ----------
482 instrument : `str`
483 Name of the instrument (e.g. ``LATISS``).
484 obs_type : `str`
485 Name of the observation type (e.g. ``Exposure``).
486 obs_id : `int`
487 Unique observation id.
488 values : `dict` [ `str`, `Any` ], optional
489 Dictionary of key/value pairs to add for the observation.
490 allow_update : `bool`, optional
491 If ``True``, allow replacement of values of existing keys.
492 **kwargs : `dict`
493 Additional key/value pairs, overriding ``values``.
495 Returns
496 -------
497 response : `requests.Response`
498 HTTP response from the server, with 200 status for success.
500 Raises
501 ------
502 ValueError
503 Raised if no values are provided in ``values`` or kwargs.
504 requests.RequestException
505 Raised if any kind of connection error occurs.
506 requests.HTTPError
507 Raised if a non-successful status is returned.
508 """
509 if values:
510 values.update(kwargs)
511 else:
512 values = kwargs
513 if not values:
514 raise ValueError(f"No values to set for {instrument} {obs_type} {obs_id}")
515 data = {"values": values}
516 url = _urljoin(
517 self.url,
518 "flex",
519 quote(instrument),
520 quote(obs_type),
521 "obs",
522 quote(str(obs_id)),
523 "update",
524 )
525 if allow_update:
526 url += "?u=1"
527 return self._handle_post(url, data)
529 def insert(
530 self,
531 instrument: str,
532 table: str,
533 obs_id: tuple[int, int] | tuple[int, int, int] | int,
534 values: Mapping[str, Any],
535 *,
536 allow_update: bool = False,
537 **kwargs: Any,
538 ) -> requests.Response:
539 """Insert values into a single ConsDB fixed metadata table.
541 Parameters
542 ----------
543 instrument : `str`
544 Name of the instrument (e.g. ``LATISS``).
545 table : `str`
546 Name of the table to insert into.
547 obs_id : `tuple`[`int`, `int`] | `tuple`[`int`, `int`, `int`] | `int`
548 How to address the row to insert:
550 - `int`: the natural observation id (e.g. ``exposure_id``,
551 ``ccdexposure_id``), targeting the ``.../obs/{obs_id}``
552 endpoint.
553 - 2-tuple ``(day_obs, seq_num)``: targeting the
554 ``.../by_seq_num/{day_obs}/{seq_num}`` endpoint, for
555 exposure-level tables.
556 - 3-tuple ``(day_obs, seq_num, detector)``: targeting the
557 ``.../by_seq_num/{day_obs}/{seq_num}/{detector}`` endpoint,
558 for ccdexposure-level (per-detector) tables.
559 values : `Mapping` [ `str`, `Any` ]
560 Dictionary-like mapping of column/value pairs to add for the
561 observation.
562 allow_update : `bool`, optional
563 If ``True``, allow replacement of values of existing columns.
564 **kwargs : `dict`
565 Additional column/value pairs, overriding ``values``.
567 Returns
568 -------
569 response : `requests.Response`
570 HTTP response from the server, with 200 status for success.
572 Raises
573 ------
574 ValueError
575 Raised if no values are provided in ``values`` or kwargs.
576 requests.RequestException
577 Raised if any kind of connection error occurs.
578 requests.HTTPError
579 Raised if a non-successful status is returned.
580 """
581 # Build a new merged dict to avoid mutating the incoming Mapping.
582 merged_values: dict[str, Any] = {**(dict(values) if values else {}), **kwargs}
583 if not merged_values:
584 raise ValueError(f"No values to insert for {instrument} {table} {obs_id}")
586 data: dict[str, Any]
587 if isinstance(obs_id, tuple):
588 assert len(obs_id) in (
589 2,
590 3,
591 ), f"obs_id tuple must be (day_obs, seq_num) or (day_obs, seq_num, detector); got {obs_id!r}"
593 data = {"table": table, "values": merged_values}
594 url = _urljoin(
595 self.url,
596 "insert",
597 quote(instrument),
598 quote(table),
599 "by_seq_num",
600 # Two segments for exposure-level tables, three when a
601 # detector is supplied for ccdexposure-level tables.
602 *(quote(str(part)) for part in obs_id),
603 )
604 else:
605 data = {"table": table, "obs_id": obs_id, "values": merged_values}
606 url = _urljoin(
607 self.url,
608 "insert",
609 quote(instrument),
610 quote(table),
611 "obs",
612 quote(str(obs_id)),
613 )
614 if allow_update:
615 url += "?u=1"
616 return self._handle_post(url, data)
618 def insert_multiple(
619 self,
620 instrument: str,
621 table: str,
622 obs_dict: dict[int, dict[str, Any]],
623 *,
624 allow_update: bool = False,
625 ) -> requests.Response:
626 """Insert values into a single ConsDB fixed metadata table.
628 Parameters
629 ----------
630 instrument : `str`
631 Name of the instrument (e.g. ``LATISS``).
632 table : `str`
633 Name of the table to insert into.
634 obs_dict : `dict` [ `int`, `dict` [ `str`, `Any` ] ]
635 Dictionary of observation ids, each with a dictionary of
636 column/value pairs to add for each observation.
637 allow_update : `bool`, optional
638 If ``True``, allow replacement of values of existing columns.
640 Returns
641 -------
642 response : `requests.Response`
643 HTTP response from the server, with 200 status for success.
645 Raises
646 ------
647 ValueError
648 Raised if no values are provided in ``obs_dict``.
649 requests.RequestException
650 Raised if any kind of connection error occurs.
651 requests.HTTPError
652 Raised if a non-successful status is returned.
653 """
654 if not obs_dict:
655 raise ValueError(f"No values to insert for {instrument} {table}")
656 data = {"table": table, "obs_dict": obs_dict}
657 url = _urljoin(
658 self.url,
659 "insert",
660 quote(instrument),
661 quote(table),
662 )
663 if allow_update:
664 url += "?u=1"
665 return self._handle_post(url, data)
667 def query(self, query: str) -> Table:
668 """Query the ConsDB database.
670 Parameters
671 ----------
672 query : `str`
673 A SQL query (currently) to the database.
675 Returns
676 -------
677 result : `Table`
678 An ``astropy.Table`` containing the query results.
680 Raises
681 ------
682 requests.RequestException
683 Raised if any kind of connection error occurs.
684 requests.HTTPError
685 Raised if a non-successful status is returned.
687 Notes
688 -----
689 This is a very general query interface because it is expected that
690 a wide variety of types of queries will be needed. If some types prove
691 to be common, syntactic sugar could be added to make them simpler.
692 """
693 url = _urljoin(self.url, "query")
694 data = {"query": query}
695 result = self._handle_post(url, data).json()
697 columns = result.get("columns", [])
698 if not columns: 698 ↛ 700line 698 didn't jump to line 700 because the condition on line 698 was never true
699 # No result columns
700 return Table(rows=[])
702 rows = result.get("data", [])
703 if not rows:
704 # No result rows
705 return Table(names=columns)
707 return Table(rows=rows, names=columns)
709 def schema(
710 self, instrument: str | None = None, table: str | None = None
711 ) -> dict[str, tuple[str, str]] | list[str]:
712 """Retrieve information about ConsDB.
714 If ``instrument`` and ``table`` are given, return the schema of a
715 fixed metadata table in ConsDB.
717 If only ``instrument`` is given, return the names of all tables
718 for that instrument.
720 If no arguments are given, return the names of all instruments.
722 Parameters
723 ----------
724 instrument : `str`, optional
725 Name of the instrument (e.g. ``LATISS``).
726 table : `str`, optional
727 Name of the table to insert into.
729 Returns
730 -------
731 info : `list` [ `str` ] or `dict` [ `str`, `tuple` [ `str`, `str` ] ]
732 A list of instrument strings or table names, or else a dict of
733 columns with values that are tuples containing a data type string
734 and a documentation string.
736 Raises
737 ------
738 ValueError
739 Raised if only ``table`` is given.
740 requests.RequestException
741 Raised if any kind of connection error occurs.
742 requests.HTTPError
743 Raised if a non-successful status is returned.
745 Notes
746 -----
747 Fixed metadata data types may use the full database vocabulary,
748 unlike flexible metadata data types.
749 """
750 if instrument is None:
751 if table is not None: 751 ↛ 752line 751 didn't jump to line 752 because the condition on line 751 was never true
752 raise ValueError("Must specify instrument if table is given")
753 url = _urljoin(self.url, "schema")
754 elif table is None: 754 ↛ 755line 754 didn't jump to line 755 because the condition on line 754 was never true
755 url = _urljoin(self.url, "schema", quote(instrument))
756 else:
757 url = _urljoin(self.url, "schema", quote(instrument), quote(table))
758 result = self._handle_get(url)
759 if instrument is not None and table is not None:
760 return {key: (str(value[0]), str(value[1])) for key, value in result.items()}
761 else:
762 return [str(value) for value in result]
765def getCcdVisitTableForDay(
766 client: ConsDbClient,
767 dayObs: int,
768 visitTableItems: list[str] | None = None,
769 detectors: list[int] | None = None,
770 withZeropoint: bool = False,
771) -> Table:
772 """Get the ccdvisit1_quicklook table for a given dayObs.
774 Parameters
775 ----------
776 client : `ConsDbClient`
777 The ConsDbClient to use.
778 dayObs : `int`
779 The dayObs to query for.
780 visitTableItems : `list` of `str`, optional
781 Additional items from the visit1 table to include.
782 detectors : `list` of `int`, optional
783 If given, only return rows for these detectors.
784 withZeropoint : `bool`, optional
785 If ``True``, only return rows with a non-null zeropoint.
787 Returns
788 -------
789 table : `astropy.table.Table`
790 The resulting table.
791 """
792 # ``cvq.*`` already pulls in every column of ccdvisit1_quicklook, so any
793 # column we also name explicitly from ccdvisit1/visit1 has to be dropped
794 # from the SELECT when the quicklook table itself already defines it.
795 # ConsDB has been denormalising identity columns (visit_id, detector,
796 # seq_num, ...) onto the quicklook tables, and selecting such a column
797 # twice makes the server return duplicate column names, which astropy then
798 # refuses to build a Table from. getWideQuicklookTableForDay does the same
799 # dedup for the visit1 join.
800 cvqCols = set(client.query("SELECT * FROM cdb_LSSTCam.ccdvisit1_quicklook LIMIT 0").colnames)
802 visitItems = ["band", "exp_time", "seq_num", "day_obs", "img_type"]
803 if visitTableItems: 803 ↛ 804line 803 didn't jump to line 804 because the condition on line 803 was never true
804 visitItems += visitTableItems
806 selectClauses = ["cvq.*"]
807 claimed = set(cvqCols) # cvq.* already provides all of these names
808 for col in ("detector", "visit_id"):
809 if col not in claimed:
810 selectClauses.append(f"cv.{col}")
811 claimed.add(col)
812 for col in visitItems:
813 if col not in claimed:
814 selectClauses.append(f"v.{col}")
815 claimed.add(col)
817 query = (
818 f"SELECT {', '.join(selectClauses)} "
819 "FROM cdb_LSSTCam.ccdvisit1_quicklook as cvq, "
820 "cdb_LSSTCam.ccdvisit1 as cv, "
821 "cdb_LSSTCam.visit1 as v "
822 )
823 where = f"WHERE cvq.ccdvisit_id=cv.ccdvisit_id and cv.visit_id=v.visit_id and v.day_obs={dayObs}"
824 if detectors: 824 ↛ 825line 824 didn't jump to line 825 because the condition on line 824 was never true
825 where += f" and cv.detector in ({','.join([str(d) for d in detectors])})"
826 if withZeropoint: 826 ↛ 827line 826 didn't jump to line 827 because the condition on line 826 was never true
827 where += " and cvq.zero_point is not null"
829 table = client.query(query + where)
830 return table
833def columnsEqual(a: Column, b: Column) -> bool:
834 """Check if two columns are equal, taking masks into account.
836 Parameters
837 ----------
838 a : `Column`
839 First column to compare.
840 b : `Column`
841 Second column to compare.
843 Returns
844 -------
845 equal : `bool`
846 True if the columns are equal, False otherwise.
847 """
848 aArr = np.asanyarray(a)
849 bArr = np.asanyarray(b)
850 if aArr.shape != bArr.shape:
851 return False
853 aMask = getattr(a, "mask", None)
854 bMask = getattr(b, "mask", None)
856 if aMask is None and bMask is None:
857 return bool(np.all(aArr == bArr))
859 if aMask is None:
860 aMask = np.zeros(aArr.shape, dtype=bool)
861 if bMask is None:
862 bMask = np.zeros(bArr.shape, dtype=bool)
864 aMaskArr = np.asanyarray(aMask)
865 bMaskArr = np.asanyarray(bMask)
867 if np.any(aMaskArr ^ bMaskArr):
868 return False # one masked where the other isn't
870 present = ~aMaskArr
871 return bool(np.all(aArr[present] == bArr[present]))
874def getWideQuicklookTableForDay(client: ConsDbClient, dayObs: int) -> Table:
875 """Get a wide quicklook table for a given dayObs.
877 Joins all columns from the visit1 table to the visit1_quicklook table. Note
878 that the visit1 table already contains all the columns from the exposure
879 table, and is just keyed by the exposure_id instead of the visit_id.
881 Parameters
882 ----------
883 client : `ConsDbClient`
884 The ConsDbClient to use.
885 dayObs : `int`
886 The dayObs to query for.
888 Returns
889 -------
890 table : `astropy.table.Table`
891 The resulting wide quicklook table.
892 """
893 vqCols = set(client.query("SELECT * FROM cdb_LSSTCam.visit1_quicklook LIMIT 0").colnames)
894 vCols = set(client.query("SELECT * FROM cdb_LSSTCam.visit1 LIMIT 0").colnames)
896 vOnlyCols = vCols - vqCols # exclude visit_id and all duplicates
898 selectClauses = ["vq.*"] + [f"v.{col}" for col in sorted(vOnlyCols)]
900 query = f"""
901 SELECT {', '.join(selectClauses)}
902 FROM cdb_LSSTCam.visit1_quicklook vq
903 INNER JOIN cdb_LSSTCam.visit1 v USING (visit_id)
904 WHERE vq.day_obs = {dayObs}
905 """
906 return client.query(query)