Coverage for tests/test_consdbClient.py: 100%
199 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-28 11:20 -0700
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import json
23from typing import Any
25import pytest
26import responses
27from astropy.table import Table
28from requests import HTTPError, Response, Timeout
30from lsst.summit.utils import ConsDbClient, FlexibleMetadataInfo
31from lsst.summit.utils.consdbClient import (
32 DEFAULT_CONNECT_TIMEOUT,
33 DEFAULT_READ_TIMEOUT,
34 getCcdVisitTableForDay,
35)
38@pytest.fixture(autouse=True)
39def no_access_token(monkeypatch: pytest.MonkeyPatch) -> None:
40 """Keep an ``ACCESS_TOKEN`` in the environment from leaking into
41 the tests.
42 """
43 monkeypatch.delenv("ACCESS_TOKEN", raising=False)
46@pytest.fixture
47def client() -> ConsDbClient:
48 """Initialize client with a fake url
49 Requires mocking connection with @responses.activate decorator
50 """
51 return ConsDbClient("http://example.com/consdb")
54def test_table_name() -> None:
55 instrument = "latiss"
56 obs_type = "exposure"
57 assert (
58 ConsDbClient.compute_flexible_metadata_table_name(instrument, obs_type)
59 == "cdb_latiss.exposure_flexdata"
60 )
63@responses.activate
64def test_add_flexible_metadata_key(client: ConsDbClient) -> None:
65 instrument = "latiss"
66 obs_type = "exposure"
67 responses.post(
68 "http://example.com/consdb/flex/latiss/exposure/addkey",
69 json={
70 "message": "Key added to flexible metadata",
71 "key": "foo",
72 "instrument": "latiss",
73 "obs_type": "exposure",
74 },
75 match=[
76 responses.matchers.json_params_matcher({"key": "foo", "dtype": "bool", "doc": "bool key"}),
77 ],
78 )
79 responses.post(
80 "http://example.com/consdb/flex/latiss/exposure/addkey",
81 json={
82 "message": "Key added to flexible metadata",
83 "key": "bar",
84 "instrument": "latiss",
85 "obs_type": "exposure",
86 },
87 match=[
88 responses.matchers.json_params_matcher({"key": "bar", "dtype": "int", "doc": "int key"}),
89 ],
90 )
91 responses.post(
92 "http://example.com/consdb/flex/bad_instrument/exposure/addkey",
93 status=404,
94 json={"message": "Unknown instrument", "value": "bad_instrument", "valid": ["latiss"]},
95 )
96 responses.post(
97 "http://example.com/consdb/flex/latiss/bad_obs_type/addkey",
98 status=404,
99 json={"message": "Unknown observation type", "value": "bad_obs_type", "valid": ["exposure"]},
100 )
102 assert (
103 client.add_flexible_metadata_key(instrument, obs_type, "foo", "bool", "bool key").json()["key"]
104 == "foo"
105 )
106 assert (
107 client.add_flexible_metadata_key(instrument, obs_type, "bar", "int", "int key").json()["instrument"]
108 == "latiss"
109 )
110 with pytest.raises(HTTPError, match="404") as e:
111 client.add_flexible_metadata_key("bad_instrument", obs_type, "error", "int", "instrument error")
112 assert "Unknown instrument" in str(e.value.__notes__)
113 assert e.value.response is not None
114 json_data = e.value.response.json()
115 assert json_data["message"] == "Unknown instrument"
116 assert json_data["value"] == "bad_instrument"
117 assert json_data["valid"] == ["latiss"]
118 with pytest.raises(HTTPError, match="404"):
119 client.add_flexible_metadata_key(instrument, "bad_obs_type", "error", "int", "obs_type error")
122@responses.activate
123def test_get_flexible_metadata_keys(client: ConsDbClient) -> None:
124 description = {"foo": ["bool", "a", None, None], "bar": ["float", "b", "deg", "pos.eq.ra"]}
125 responses.get(
126 "http://example.com/consdb/flex/latiss/exposure/schema",
127 json=description,
128 )
129 instrument = "latiss"
130 obs_type = "exposure"
131 assert client.get_flexible_metadata_keys(instrument, obs_type) == {
132 "foo": FlexibleMetadataInfo("bool", "a"),
133 "bar": FlexibleMetadataInfo("float", "b", "deg", "pos.eq.ra"),
134 }
137@responses.activate
138def test_get_flexible_metadata(client: ConsDbClient) -> None:
139 results = {"bool_key": True, "int_key": 42, "float_key": 3.14159, "str_key": "foo"}
140 responses.get(
141 "http://example.com/consdb/flex/latiss/exposure/obs/271828",
142 json=results,
143 )
144 responses.get(
145 "http://example.com/consdb/flex/latiss/exposure/obs/271828?k=float_key", json={"float_key": 3.14159}
146 )
147 responses.get(
148 "http://example.com/consdb/flex/latiss/exposure/obs/271828?k=int_key&k=float_key",
149 json={"float_key": 3.14159, "int_key": 42},
150 )
151 instrument = "latiss"
152 obs_type = "exposure"
153 obs_id = 271828
154 assert client.get_flexible_metadata(instrument, obs_type, obs_id) == results
155 assert client.get_flexible_metadata(instrument, obs_type, obs_id, ["float_key"]) == {
156 "float_key": results["float_key"]
157 }
158 assert client.get_flexible_metadata(instrument, obs_type, obs_id, ["int_key", "float_key"]) == {
159 "int_key": results["int_key"],
160 "float_key": results["float_key"],
161 }
164@responses.activate
165def test_insert_flexible_metadata(client: ConsDbClient) -> None:
166 instrument = "latiss"
167 obs_type = "exposure"
168 with pytest.raises(ValueError):
169 client.insert_flexible_metadata(instrument, obs_type, 271828)
170 responses.post(
171 "http://example.com/consdb/flex/latiss/exposure/obs/271828/update",
172 json={"message": "Flexible metadata inserted"},
173 match=[responses.matchers.json_params_matcher({"values": {"int_key": 42}})],
174 )
175 responses.post(
176 "http://example.com/consdb/flex/latiss/exposure/obs/271828/update?u=1",
177 json={"message": "Flexible metadata inserted"},
178 match=[responses.matchers.query_param_matcher({"u": "1"})],
179 )
180 client.insert_flexible_metadata(instrument, obs_type, 271828, {"int_key": 42})
181 client.insert_flexible_metadata(instrument, obs_type, 271828, int_key=43, allow_update=True)
182 assert len(responses.calls) == 2
183 # TODO: more POST tests
186@responses.activate
187def test_schema(client: ConsDbClient) -> None:
188 description = {"foo": ("bool", "a"), "bar": ("int", "b")}
189 responses.get(
190 "http://example.com/consdb/schema/latiss/misc_table",
191 json=description,
192 )
193 instrument = "latiss"
194 table = "misc_table"
195 assert client.schema(instrument, table) == description
198@responses.activate
199@pytest.mark.parametrize(
200 "secret, redacted",
201 [
202 ("usdf:v987wefVMPz", "us***:xxxxxxx"),
203 ("u:v", "u***:xxxxxxx"),
204 ("ulysses", "ul***"),
205 (":alberta94", "***:xxxxxxx"),
206 ("x:gt-abc123", "x***:gt***"),
207 ],
208)
209def test_clean_token_url_response(secret: str, redacted: str) -> None:
210 """Test tokens URL is cleaned when an error is thrown from requests
211 Use with pytest raises assert an error'
212 assert that url does not contain tokens
213 """
214 domain = "@usdf-fake.slackers.stanford.edu/consdb"
215 complex_client = ConsDbClient(f"https://{secret}{domain}")
217 obs_type = "exposure"
218 responses.post(
219 f"https://{secret}{domain}/flex/bad_instrument/exposure/addkey",
220 status=404,
221 )
222 with pytest.raises(HTTPError, match="404") as error:
223 complex_client.add_flexible_metadata_key(
224 "bad_instrument", obs_type, "error", "int", "instrument error"
225 )
227 url = error.value.args[0].split()[-1]
228 sanitized = f"https://{redacted}{domain}/flex/bad_instrument/exposure/addkey"
229 assert url == sanitized
232def test_client(client: ConsDbClient) -> None:
233 """Test ConsDbClient is initialized properly"""
234 assert "clean_url" in str(client.session.hooks["response"])
235 assert client.connect_timeout == DEFAULT_CONNECT_TIMEOUT
236 assert client.read_timeout == DEFAULT_READ_TIMEOUT
237 assert client.timeout == (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT)
240def test_no_token(client: ConsDbClient) -> None:
241 """Without a token or ACCESS_TOKEN, no Authorization header is sent."""
242 assert "Authorization" not in client.session.headers
245def test_explicit_token() -> None:
246 """A valid explicit token is sent as a Bearer token."""
247 client = ConsDbClient("http://example.com/consdb", token="gt-abc123")
248 assert client.session.headers["Authorization"] == "Bearer gt-abc123"
251def test_explicit_bad_token() -> None:
252 """An explicit token with an unknown prefix is rejected."""
253 with pytest.raises(ValueError, match="gt-"):
254 ConsDbClient("http://example.com/consdb", token="abc123")
257def test_env_token(monkeypatch: pytest.MonkeyPatch) -> None:
258 """A valid ACCESS_TOKEN is used when no token is given."""
259 monkeypatch.setenv("ACCESS_TOKEN", "gt-abc123")
260 client = ConsDbClient("http://example.com/consdb")
261 assert client.session.headers["Authorization"] == "Bearer gt-abc123"
264def test_explicit_token_overrides_env(monkeypatch: pytest.MonkeyPatch) -> None:
265 """An explicit token takes precedence over ACCESS_TOKEN."""
266 monkeypatch.setenv("ACCESS_TOKEN", "gt-fromenv")
267 client = ConsDbClient("http://example.com/consdb", token="gt-explicit")
268 assert client.session.headers["Authorization"] == "Bearer gt-explicit"
271@pytest.mark.parametrize("token", ["abc123", ""])
272def test_env_bad_token_ignored(token: str, monkeypatch: pytest.MonkeyPatch) -> None:
273 """An invalid or empty ACCESS_TOKEN is ignored rather than raising."""
274 monkeypatch.setenv("ACCESS_TOKEN", token)
275 client = ConsDbClient("http://example.com/consdb")
276 assert "Authorization" not in client.session.headers
279def test_timeout_override() -> None:
280 """The timeouts are configurable, including disabling them entirely."""
281 tuned = ConsDbClient("http://example.com/consdb", connect_timeout=5, read_timeout=30)
282 assert tuned.timeout == (5, 30)
283 unbounded = ConsDbClient("http://example.com/consdb", connect_timeout=None, read_timeout=None)
284 assert unbounded.timeout == (None, None)
287def test_get_passes_timeout(client: ConsDbClient, monkeypatch: pytest.MonkeyPatch) -> None:
288 """GET requests must carry a timeout so a stalled server cannot hang."""
289 captured: dict[str, Any] = {}
291 def fake_get(url: str, **kwargs: Any) -> Response:
292 captured.update(kwargs)
293 response = Response()
294 response.status_code = 200
295 response._content = b"{}"
296 return response
298 monkeypatch.setattr(client.session, "get", fake_get)
299 client.schema()
300 assert captured["timeout"] == (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT)
303def test_post_passes_timeout(client: ConsDbClient, monkeypatch: pytest.MonkeyPatch) -> None:
304 """POST requests must carry a timeout so a stalled server cannot hang."""
305 captured: dict[str, Any] = {}
307 def fake_post(url: str, **kwargs: Any) -> Response:
308 captured.update(kwargs)
309 response = Response()
310 response.status_code = 200
311 response._content = b'{"message": "Data inserted"}'
312 return response
314 monkeypatch.setattr(client.session, "post", fake_post)
315 client.insert("latiss", "exposure", 271828, {"foo": 1})
316 assert captured["timeout"] == (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT)
319def test_timeout_propagates(client: ConsDbClient, monkeypatch: pytest.MonkeyPatch) -> None:
320 """A timed-out request surfaces as requests.Timeout to the caller."""
322 def fake_get(url: str, **kwargs: Any) -> Response:
323 raise Timeout("timed out")
325 monkeypatch.setattr(client.session, "get", fake_get)
326 with pytest.raises(Timeout):
327 client.schema()
330@responses.activate
331def test_insert_obs_id(client: ConsDbClient) -> None:
332 """An integer obs_id targets the ``.../obs/{obs_id}`` endpoint."""
333 responses.post(
334 "http://example.com/consdb/insert/latiss/exposure/obs/271828",
335 json={"message": "Data inserted"},
336 match=[
337 responses.matchers.json_params_matcher(
338 {"table": "exposure", "obs_id": 271828, "values": {"foo": 1}}
339 ),
340 ],
341 )
342 assert client.insert("latiss", "exposure", 271828, {"foo": 1}).json()["message"] == "Data inserted"
345@responses.activate
346def test_insert_by_seq_num(client: ConsDbClient) -> None:
347 """A 2-tuple targets the ``.../{day_obs}/{seq_num}`` endpoint."""
348 responses.post(
349 "http://example.com/consdb/insert/latiss/exposure/by_seq_num/20240603/123",
350 json={"message": "Data inserted"},
351 match=[
352 responses.matchers.json_params_matcher({"table": "exposure", "values": {"foo": 1}}),
353 ],
354 )
355 assert (
356 client.insert("latiss", "exposure", (20240603, 123), {"foo": 1}).json()["message"] == "Data inserted"
357 )
360@responses.activate
361def test_insert_by_seq_num_detector(client: ConsDbClient) -> None:
362 """A 3-tuple targets the ``.../{day_obs}/{seq_num}/{detector}``
363 endpoint for per-detector (ccdexposure-level) tables."""
364 responses.post(
365 "http://example.com/consdb/insert/lsstcam/ccdexposure/by_seq_num/20240603/123/94",
366 json={"message": "Data inserted"},
367 match=[
368 responses.matchers.json_params_matcher({"table": "ccdexposure", "values": {"foo": 1}}),
369 ],
370 )
371 assert (
372 client.insert("lsstcam", "ccdexposure", (20240603, 123, 94), {"foo": 1}).json()["message"]
373 == "Data inserted"
374 )
377@responses.activate
378def test_insert_allow_update(client: ConsDbClient) -> None:
379 """allow_update appends ``?u=1`` to upsert against the addressed key."""
380 responses.post(
381 "http://example.com/consdb/insert/lsstcam/ccdexposure/by_seq_num/20240603/123/94?u=1",
382 json={"message": "Data inserted"},
383 match=[responses.matchers.query_param_matcher({"u": "1"})],
384 )
385 assert (
386 client.insert("lsstcam", "ccdexposure", (20240603, 123, 94), {"foo": 1}, allow_update=True).json()[
387 "message"
388 ]
389 == "Data inserted"
390 )
393def test_insert_bad_obs_id_tuple(client: ConsDbClient) -> None:
394 """A tuple that is not length 2 or 3 is rejected before any request."""
395 with pytest.raises(AssertionError, match="obs_id tuple"):
396 client.insert("latiss", "exposure", (20240603,), {"foo": 1}) # type: ignore[arg-type]
397 with pytest.raises(AssertionError, match="obs_id tuple"):
398 client.insert("latiss", "exposure", (20240603, 123, 94, 0), {"foo": 1}) # type: ignore[arg-type]
401def test_insert_no_values(client: ConsDbClient) -> None:
402 """Inserting with no values raises before any request."""
403 with pytest.raises(ValueError, match="No values to insert"):
404 client.insert("latiss", "exposure", 271828, {})
407@responses.activate
408def test_getCcdVisitTableForDay_dedupes_overlapping_columns(client: ConsDbClient) -> None:
409 """Columns already present in ccdvisit1_quicklook must not be re-selected.
411 ``cvq.*`` pulls in every column of ccdvisit1_quicklook. As ConsDB
412 denormalises identity columns (visit_id, detector, seq_num, ...) onto the
413 quicklook table, naming those again explicitly from the joined tables makes
414 the server return duplicate column names, which astropy refuses to build a
415 Table from (DM-55152). They must be dropped from the SELECT instead.
416 """
417 url = "http://example.com/consdb/query"
419 # First query is the LIMIT 0 schema probe: pretend the quicklook table has
420 # denormalised visit_id, detector and seq_num onto itself.
421 responses.post(
422 url,
423 json={"columns": ["ccdvisit_id", "visit_id", "detector", "seq_num", "psf_sigma"], "data": []},
424 )
425 # Second query is the real data query; its response only has to build
426 # cleanly, the behaviour under test is the SELECT clause that was sent.
427 responses.post(
428 url,
429 json={
430 "columns": ["ccdvisit_id", "visit_id", "detector", "seq_num", "psf_sigma", "band"],
431 "data": [[1, 100, 7, 5, 1.2, "r"]],
432 },
433 )
435 table = getCcdVisitTableForDay(client, 20240101)
436 assert isinstance(table, Table)
438 # Only inspect the SELECT clause; the WHERE clause legitimately references
439 # cv.visit_id etc. in its join conditions.
440 requestBody = responses.calls[1].request.body
441 assert isinstance(requestBody, (str, bytes))
442 sentQuery = json.loads(requestBody)["query"]
443 selectClause = sentQuery.split(" FROM ")[0]
444 # Columns already provided by cvq.* must not be re-selected explicitly...
445 assert "cvq.*" in selectClause
446 assert "cv.visit_id" not in selectClause
447 assert "cv.detector" not in selectClause
448 assert "v.seq_num" not in selectClause
449 # ...but columns the quicklook table lacks must still be pulled from visit1
450 for col in ("v.band", "v.exp_time", "v.day_obs", "v.img_type"):
451 assert col in selectClause
454@responses.activate
455def test_getCcdVisitTableForDay_keeps_columns_when_no_overlap(client: ConsDbClient) -> None:
456 """When the quicklook table shares no names, every extra is selected."""
457 url = "http://example.com/consdb/query"
458 responses.post(url, json={"columns": ["ccdvisit_id", "psf_sigma"], "data": []})
459 responses.post(url, json={"columns": ["ccdvisit_id", "psf_sigma"], "data": [[1, 1.2]]})
461 getCcdVisitTableForDay(client, 20240101)
463 requestBody = responses.calls[1].request.body
464 assert isinstance(requestBody, (str, bytes))
465 sentQuery = json.loads(requestBody)["query"]
466 selectClause = sentQuery.split(" FROM ")[0]
467 for col in ("cv.detector", "cv.visit_id", "v.band", "v.exp_time", "v.seq_num", "v.day_obs", "v.img_type"):
468 assert col in selectClause
471# TODO: more POST tests
472# client.insert_multiple(instrument, table, obs_dict, allow_update)
473# client.query(query)