Coverage for tests/test_apdbSql.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of dax_apdb.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit test for Apdb class.
23"""
25import pandas
26import random
27from typing import Iterator
28import unittest
30from lsst.daf.base import DateTime
31from lsst.dax.apdb import ApdbSql, ApdbSqlConfig
32from lsst.sphgeom import Angle, Circle, LonLat, Region, UnitVector3d
33from lsst.geom import SpherePoint
34import lsst.utils.tests
37def _makeRegion() -> Region:
38 """Generate pixel ID ranges for some envelope region"""
39 pointing_v = UnitVector3d(1., 1., -1.)
40 fov = 0.05 # radians
41 region = Circle(pointing_v, Angle(fov/2))
42 return region
45def _makeVectors(region: Region, count: int = 1) -> Iterator[SpherePoint]:
46 """Generate bunch of SpherePoints inside given region.
48 Returned vectors are random but not necessarily uniformly distributed.
49 """
50 bbox = region.getBoundingBox()
51 center = bbox.getCenter()
52 center_lon = center.getLon().asRadians()
53 center_lat = center.getLat().asRadians()
54 width = bbox.getWidth().asRadians()
55 height = bbox.getHeight().asRadians()
56 while count > 0:
57 lon = random.uniform(center_lon - width / 2, center_lon + width / 2)
58 lat = random.uniform(center_lat - height / 2, center_lat + height / 2)
59 lonlat = LonLat.fromRadians(lon, lat)
60 uv3d = UnitVector3d(lonlat)
61 if region.contains(uv3d):
62 yield SpherePoint(lonlat)
63 count -= 1
66def _makeObjectCatalogPandas(region, count: int, config: ApdbSqlConfig):
67 """Make a catalog containing a bunch of DiaObjects inside region.
69 The number of created records will be equal to the number of ranges (one
70 object per pixel range). Coordinates of the created objects are not usable.
71 """
72 data_list = []
73 # 0 id'ed DiaObjects don't exist and is used as a Null value for the id.
74 for oid, sp in enumerate(_makeVectors(region, count)):
75 tmp_dict = {"diaObjectId": oid + 1,
76 "ra": sp.getRa().asDegrees(),
77 "decl": sp.getDec().asDegrees()}
78 data_list.append(tmp_dict)
80 df = pandas.DataFrame(data=data_list)
81 return df
84def _makeSourceCatalogPandas(objects, visit_time, start_id=0):
85 """Make a catalog containing a bunch of DiaSources associated with the
86 input diaObjects.
87 """
88 # make some sources
89 catalog = []
90 midPointTai = visit_time.get(system=DateTime.MJD)
91 for index, obj in objects.iterrows():
92 catalog.append({"diaSourceId": start_id,
93 "ccdVisitId": 1,
94 "diaObjectId": obj["diaObjectId"],
95 "parentDiaSourceId": 0,
96 "ra": obj["ra"],
97 "decl": obj["decl"],
98 "midPointTai": midPointTai,
99 "flags": 0})
100 start_id += 1
101 return pandas.DataFrame(data=catalog)
104def _makeForcedSourceCatalogPandas(objects, visit_time, ccdVisitId=1):
105 """Make a catalog containing a bunch of DiaFourceSources associated with
106 the input diaObjects.
107 """
108 # make some sources
109 catalog = []
110 midPointTai = visit_time.get(system=DateTime.MJD)
111 for index, obj in objects.iterrows():
112 catalog.append({"diaObjectId": obj["diaObjectId"],
113 "ccdVisitId": ccdVisitId,
114 "midPointTai": midPointTai,
115 "flags": 0})
116 return pandas.DataFrame(data=catalog)
119class ApdbTestCase(unittest.TestCase):
120 """A test case for Apdb class
121 """
123 data_type = pandas.DataFrame
125 def _assertCatalog(self, catalog, size, type=pandas.DataFrame):
126 """Validate catalog type and size
128 Parameters
129 ----------
130 calalog : `object`
131 Expected type of this is ``type``.
132 size : int
133 Expected catalog size
134 type : `type`, optional
135 Expected catalog type
136 """
137 self.assertIsInstance(catalog, type)
138 self.assertEqual(len(catalog), size)
140 def test_makeSchema(self):
141 """Test for making an instance of Apdb using in-memory sqlite engine.
142 """
143 # sqlite does not support default READ_COMMITTED, for in-memory
144 # database have to use connection pool
145 config = ApdbSqlConfig(db_url="sqlite://")
146 apdb = ApdbSql(config)
147 # the essence of a test here is that there are no exceptions.
148 apdb.makeSchema()
150 def test_emptyGetsBaseline0months(self):
151 """Test for getting data from empty database.
153 All get() methods should return empty results, only useful for
154 checking that code is not broken.
155 """
157 # set read_sources_months to 0 so that Forced/Sources are None
158 config = ApdbSqlConfig(db_url="sqlite:///",
159 read_sources_months=0,
160 read_forced_sources_months=0)
161 apdb = ApdbSql(config)
162 apdb.makeSchema()
164 region = _makeRegion()
165 visit_time = DateTime.now()
167 # get objects by region
168 res = apdb.getDiaObjects(region)
169 self._assertCatalog(res, 0, type=self.data_type)
171 # get sources by region
172 res = apdb.getDiaSources(region, None, visit_time)
173 self.assertIs(res, None)
175 # get sources by object ID, empty object list
176 res = apdb.getDiaSources(region, [], visit_time)
177 self.assertIs(res, None)
179 # get forced sources by object ID, empty object list
180 res = apdb.getDiaForcedSources(region, [], visit_time)
181 self.assertIs(res, None)
183 def test_emptyGetsBaseline(self):
184 """Test for getting data from empty database.
186 All get() methods should return empty results, only useful for
187 checking that code is not broken.
188 """
190 # use non-zero months for Forced/Source fetching
191 config = ApdbSqlConfig(db_url="sqlite:///",
192 read_sources_months=12,
193 read_forced_sources_months=12)
194 apdb = ApdbSql(config)
195 apdb.makeSchema()
197 region = _makeRegion()
198 visit_time = DateTime.now()
200 # get objects by region
201 res = apdb.getDiaObjects(region)
202 self._assertCatalog(res, 0, type=self.data_type)
204 # get sources by region
205 res = apdb.getDiaSources(region, None, visit_time)
206 self._assertCatalog(res, 0, type=self.data_type)
208 res = apdb.getDiaSources(region, [], visit_time)
209 self._assertCatalog(res, 0, type=self.data_type)
211 # get sources by object ID, non-empty object list
212 res = apdb.getDiaSources(region, [1, 2, 3], visit_time)
213 self._assertCatalog(res, 0, type=self.data_type)
215 # get forced sources by object ID, empty object list
216 res = apdb.getDiaForcedSources(region, [], visit_time)
217 self._assertCatalog(res, 0, type=self.data_type)
219 # get sources by object ID, non-empty object list
220 res = apdb.getDiaForcedSources(region, [1, 2, 3], visit_time)
221 self._assertCatalog(res, 0, type=self.data_type)
223 # SQL implementation needs ID list
224 with self.assertRaises(NotImplementedError):
225 apdb.getDiaForcedSources(region, None, visit_time)
227 def test_emptyGetsObjectLast(self):
228 """Test for getting DiaObjects from empty database using DiaObjectLast
229 table.
231 All get() methods should return empty results, only useful for
232 checking that code is not broken.
233 """
235 # don't care about sources.
236 config = ApdbSqlConfig(db_url="sqlite:///",
237 dia_object_index="last_object_table")
238 apdb = ApdbSql(config)
239 apdb.makeSchema()
241 region = _makeRegion()
243 # get objects by region
244 res = apdb.getDiaObjects(region)
245 self._assertCatalog(res, 0, type=self.data_type)
247 def test_storeObjectsBaseline(self):
248 """Store and retrieve DiaObjects."""
250 # don't care about sources.
251 config = ApdbSqlConfig(db_url="sqlite:///",
252 dia_object_index="baseline")
253 apdb = ApdbSql(config)
254 apdb.makeSchema()
256 region = _makeRegion()
257 visit_time = DateTime.now()
259 # make catalog with Objects
260 catalog = _makeObjectCatalogPandas(region, 100, config)
262 # store catalog
263 apdb.store(visit_time, catalog)
265 # read it back and check sizes
266 res = apdb.getDiaObjects(region)
267 self._assertCatalog(res, len(catalog), type=self.data_type)
269 def test_storeObjectsLast(self):
270 """Store and retrieve DiaObjects using DiaObjectLast table."""
271 # don't care about sources.
272 config = ApdbSqlConfig(db_url="sqlite:///",
273 dia_object_index="last_object_table",
274 object_last_replace=True)
275 apdb = ApdbSql(config)
276 apdb.makeSchema()
278 region = _makeRegion()
279 visit_time = DateTime.now()
281 # make catalog with Objects
282 catalog = _makeObjectCatalogPandas(region, 100, config)
284 # store catalog
285 apdb.store(visit_time, catalog)
287 # read it back and check sizes
288 res = apdb.getDiaObjects(region)
289 self._assertCatalog(res, len(catalog), type=self.data_type)
291 def test_storeSources(self):
292 """Store and retrieve DiaSources."""
293 config = ApdbSqlConfig(db_url="sqlite:///",
294 read_sources_months=12,
295 read_forced_sources_months=12)
296 apdb = ApdbSql(config)
297 apdb.makeSchema()
299 region = _makeRegion()
300 visit_time = DateTime.now()
302 # have to store Objects first
303 objects = _makeObjectCatalogPandas(region, 100, config)
304 oids = list(objects["diaObjectId"])
305 sources = _makeSourceCatalogPandas(objects, visit_time)
307 # save the objects and sources
308 apdb.store(visit_time, objects, sources)
310 # read it back, no ID filtering
311 res = apdb.getDiaSources(region, None, visit_time)
312 self._assertCatalog(res, len(sources), type=self.data_type)
314 # read it back and filter by ID
315 res = apdb.getDiaSources(region, oids, visit_time)
316 self._assertCatalog(res, len(sources), type=self.data_type)
318 # read it back to get schema
319 res = apdb.getDiaSources(region, [], visit_time)
320 self._assertCatalog(res, 0, type=self.data_type)
322 def test_storeForcedSources(self):
323 """Store and retrieve DiaForcedSources."""
325 config = ApdbSqlConfig(db_url="sqlite:///",
326 read_sources_months=12,
327 read_forced_sources_months=12)
328 apdb = ApdbSql(config)
329 apdb.makeSchema()
331 region = _makeRegion()
332 visit_time = DateTime.now()
334 # have to store Objects first
335 objects = _makeObjectCatalogPandas(region, 100, config)
336 oids = list(objects["diaObjectId"])
337 catalog = _makeForcedSourceCatalogPandas(objects, visit_time)
339 apdb.store(visit_time, objects, forced_sources=catalog)
341 # read it back and check sizes
342 res = apdb.getDiaForcedSources(region, oids, visit_time)
343 self._assertCatalog(res, len(catalog), type=self.data_type)
345 # read it back to get schema
346 res = apdb.getDiaForcedSources(region, [], visit_time)
347 self._assertCatalog(res, 0, type=self.data_type)
349 def test_midPointTai_src(self):
350 """Test for time filtering of DiaSources.
351 """
352 config = ApdbSqlConfig(db_url="sqlite:///",
353 read_sources_months=12,
354 read_forced_sources_months=12)
355 apdb = ApdbSql(config)
356 apdb.makeSchema()
358 region = _makeRegion()
359 # 2021-01-01 plus 360 days is 2021-12-27
360 src_time1 = DateTime(2021, 1, 1, 0, 0, 0, DateTime.TAI)
361 src_time2 = DateTime(2021, 1, 1, 0, 0, 2, DateTime.TAI)
362 visit_time0 = DateTime(2021, 12, 26, 23, 59, 59, DateTime.TAI)
363 visit_time1 = DateTime(2021, 12, 27, 0, 0, 1, DateTime.TAI)
364 visit_time2 = DateTime(2021, 12, 27, 0, 0, 3, DateTime.TAI)
366 objects = _makeObjectCatalogPandas(region, 100, config)
367 oids = list(objects["diaObjectId"])
368 sources = _makeSourceCatalogPandas(objects, src_time1, 0)
369 apdb.store(src_time1, objects, sources)
371 sources = _makeSourceCatalogPandas(objects, src_time2, 100)
372 apdb.store(src_time2, objects, sources)
374 # reading at time of last save should read all
375 res = apdb.getDiaSources(region, oids, src_time2)
376 self._assertCatalog(res, 200, type=self.data_type)
378 # one second before 12 months
379 res = apdb.getDiaSources(region, oids, visit_time0)
380 self._assertCatalog(res, 200, type=self.data_type)
382 # reading at later time of last save should only read a subset
383 res = apdb.getDiaSources(region, oids, visit_time1)
384 self._assertCatalog(res, 100, type=self.data_type)
386 # reading at later time of last save should only read a subset
387 res = apdb.getDiaSources(region, oids, visit_time2)
388 self._assertCatalog(res, 0, type=self.data_type)
390 def test_midPointTai_fsrc(self):
391 """Test for time filtering of DiaForcedSources.
392 """
393 config = ApdbSqlConfig(db_url="sqlite:///",
394 read_sources_months=12,
395 read_forced_sources_months=12)
396 apdb = ApdbSql(config)
397 apdb.makeSchema()
399 region = _makeRegion()
400 src_time1 = DateTime(2021, 1, 1, 0, 0, 0, DateTime.TAI)
401 src_time2 = DateTime(2021, 1, 1, 0, 0, 2, DateTime.TAI)
402 visit_time0 = DateTime(2021, 12, 26, 23, 59, 59, DateTime.TAI)
403 visit_time1 = DateTime(2021, 12, 27, 0, 0, 1, DateTime.TAI)
404 visit_time2 = DateTime(2021, 12, 27, 0, 0, 3, DateTime.TAI)
406 objects = _makeObjectCatalogPandas(region, 100, config)
407 oids = list(objects["diaObjectId"])
408 sources = _makeForcedSourceCatalogPandas(objects, src_time1, 1)
409 apdb.store(src_time1, objects, forced_sources=sources)
411 sources = _makeForcedSourceCatalogPandas(objects, src_time2, 2)
412 apdb.store(src_time2, objects, forced_sources=sources)
414 # reading at time of last save should read all
415 res = apdb.getDiaForcedSources(region, oids, src_time2)
416 self._assertCatalog(res, 200, type=self.data_type)
418 # one second before 12 months
419 res = apdb.getDiaForcedSources(region, oids, visit_time0)
420 self._assertCatalog(res, 200, type=self.data_type)
422 # reading at later time of last save should only read a subset
423 res = apdb.getDiaForcedSources(region, oids, visit_time1)
424 self._assertCatalog(res, 100, type=self.data_type)
426 # reading at later time of last save should only read a subset
427 res = apdb.getDiaForcedSources(region, oids, visit_time2)
428 self._assertCatalog(res, 0, type=self.data_type)
431class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
432 pass
435def setup_module(module):
436 lsst.utils.tests.init()
439if __name__ == "__main__": 439 ↛ 440line 439 didn't jump to line 440, because the condition on line 439 was never true
440 lsst.utils.tests.init()
441 unittest.main()