Coverage for tests/test_apdb.py: 17%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of dax_apdb.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit test for Apdb class.
23"""
25import datetime
26import pandas
27import unittest
29import lsst.afw.table as afwTable
30from lsst.dax.apdb import (Apdb, ApdbConfig, make_minimal_dia_object_schema,
31 make_minimal_dia_source_schema)
32from lsst.sphgeom import Angle, Circle, HtmPixelization, Vector3d, UnitVector3d
33from lsst.geom import SpherePoint
34import lsst.utils.tests
37# HTM indexing level used in the unit tests
38HTM_LEVEL = 20
41def _makePixelRanges():
42 """Generate pixel ID ranges for some envelope region"""
43 pointing_v = UnitVector3d(1., 1., -1.)
44 fov = 0.05 # radians
45 region = Circle(pointing_v, Angle(fov/2))
46 pixelator = HtmPixelization(HTM_LEVEL)
47 indices = pixelator.envelope(region, 128)
48 return indices.ranges()
51def _makeObjectCatalog(pixel_ranges):
52 """Make a catalog containing a bunch of DiaObjects inside pixel envelope.
54 The number of created records will be equal number of ranges (one object
55 per pixel range). Coordinates of the created objects are not usable.
56 """
57 # make afw catalog
58 schema = make_minimal_dia_object_schema()
59 catalog = afwTable.SourceCatalog(schema)
61 # make small bunch of records, one entry per one pixel range,
62 # we do not care about coordinates here, in current implementation
63 # they are not used in any query
64 v3d = Vector3d(1., 1., -1.)
65 sp = SpherePoint(v3d)
66 for oid, (start, end) in enumerate(pixel_ranges):
67 record = catalog.addNew()
68 record.set("id", oid)
69 record.set("pixelId", start)
70 record.set("coord_ra", sp.getRa())
71 record.set("coord_dec", sp.getDec())
73 return catalog
76def _makeObjectCatalogPandas(pixel_ranges):
77 """Make a catalog containing a bunch of DiaObjects inside pixel envelope.
79 The number of created records will be equal to the number of ranges (one
80 object per pixel range). Coordinates of the created objects are not usable.
81 """
82 v3d = Vector3d(1., 1., -1.)
83 sp = SpherePoint(v3d)
84 data_list = []
85 for oid, (start, end) in enumerate(pixel_ranges):
86 tmp_dict = {"diaObjectId": oid,
87 "pixelId": start,
88 "ra": sp.getRa().asDegrees(),
89 "decl": sp.getDec().asDegrees()}
90 data_list.append(tmp_dict)
92 df = pandas.DataFrame(data=data_list)
93 return df
96def _makeSourceCatalog(objects):
97 """Make a catalog containing a bunch of DiaSources associated with the
98 input diaObjects.
99 """
100 # make some sources
101 schema = make_minimal_dia_source_schema()
102 catalog = afwTable.BaseCatalog(schema)
103 oids = []
104 for sid, obj in enumerate(objects):
105 record = catalog.addNew()
106 record.set("id", sid)
107 record.set("ccdVisitId", 1)
108 record.set("diaObjectId", obj["id"])
109 record.set("parent", 0)
110 record.set("coord_ra", obj["coord_ra"])
111 record.set("coord_dec", obj["coord_dec"])
112 record.set("flags", 0)
113 record.set("pixelId", obj["pixelId"])
114 oids.append(obj["id"])
116 return catalog, oids
119def _makeSourceCatalogPandas(objects):
120 """Make a catalog containing a bunch of DiaSources associated with the
121 input diaObjects.
122 """
123 # make some sources
124 catalog = []
125 oids = []
126 for sid, (index, obj) in enumerate(objects.iterrows()):
127 catalog.append({"diaSourceId": sid,
128 "ccdVisitId": 1,
129 "diaObjectId": obj["diaObjectId"],
130 "parentDiaSourceId": 0,
131 "ra": obj["ra"],
132 "decl": obj["decl"],
133 "flags": 0,
134 "pixelId": obj["pixelId"]})
135 oids.append(obj["diaObjectId"])
136 return pandas.DataFrame(data=catalog), oids
139def _makeForcedSourceCatalog(objects):
140 """Make a catalog containing a bunch of DiaFourceSources associated with
141 the input diaObjects.
142 """
143 # make some sources
144 schema = afwTable.Schema()
145 schema.addField("diaObjectId", "L")
146 schema.addField("ccdVisitId", "L")
147 schema.addField("flags", "L")
148 catalog = afwTable.BaseCatalog(schema)
149 oids = []
150 for obj in objects:
151 record = catalog.addNew()
152 record.set("diaObjectId", obj["id"])
153 record.set("ccdVisitId", 1)
154 record.set("flags", 0)
155 oids.append(obj["id"])
157 return catalog, oids
160def _makeForcedSourceCatalogPandas(objects):
161 """Make a catalog containing a bunch of DiaFourceSources associated with
162 the input diaObjects.
163 """
164 # make some sources
165 catalog = []
166 oids = []
167 for index, obj in objects.iterrows():
168 catalog.append({"diaObjectId": obj["diaObjectId"],
169 "ccdVisitId": 1,
170 "flags": 0})
171 oids.append(obj["diaObjectId"])
172 return pandas.DataFrame(data=catalog), oids
175class ApdbTestCase(unittest.TestCase):
176 """A test case for Apdb class
177 """
179 use_pandas = False
180 data_type = afwTable.SourceCatalog
182 def _assertCatalog(self, catalog, size, type=afwTable.SourceCatalog):
183 """Validate catalog type and size
185 Parameters
186 ----------
187 calalog : `lsst.afw.table.SourceCatalog`
188 size : int
189 Expected catalog size
190 type : `type`, optional
191 Expected catalog type
192 """
193 self.assertIsInstance(catalog, type)
194 self.assertEqual(len(catalog), size)
196 def test_makeSchema(self):
197 """Test for making an instance of Apdb using in-memory sqlite engine.
198 """
199 # sqlite does not support default READ_COMMITTED, for in-memory
200 # database have to use connection pool
201 config = ApdbConfig(db_url="sqlite://",
202 isolation_level="READ_UNCOMMITTED")
203 apdb = Apdb(config)
204 # the essence of a test here is that there are no exceptions.
205 apdb.makeSchema()
207 def test_emptyGetsBaseline0months(self):
208 """Test for getting data from empty database.
210 All get() methods should return empty results, only useful for
211 checking that code is not broken.
212 """
214 # set read_sources_months to 0 so that Forced/Sources are None
215 config = ApdbConfig(db_url="sqlite:///",
216 isolation_level="READ_UNCOMMITTED",
217 read_sources_months=0,
218 read_forced_sources_months=0)
219 apdb = Apdb(config)
220 apdb.makeSchema()
222 pixel_ranges = _makePixelRanges()
223 visit_time = datetime.datetime.now()
225 # get objects by region
226 res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas)
227 self._assertCatalog(res, 0, type=self.data_type)
229 # get sources by region
230 res = apdb.getDiaSourcesInRegion(pixel_ranges, visit_time, return_pandas=self.use_pandas)
231 self.assertIs(res, None)
233 # get sources by object ID, empty object list
234 res = apdb.getDiaSources([], visit_time, return_pandas=self.use_pandas)
236 # get forced sources by object ID, empty object list
237 res = apdb.getDiaForcedSources([], visit_time, return_pandas=self.use_pandas)
238 self.assertIs(res, None)
240 def test_emptyGetsBaseline(self):
241 """Test for getting data from empty database.
243 All get() methods should return empty results, only useful for
244 checking that code is not broken.
245 """
247 # use non-zero months for Forced/Source fetching
248 config = ApdbConfig(db_url="sqlite:///",
249 isolation_level="READ_UNCOMMITTED",
250 read_sources_months=12,
251 read_forced_sources_months=12)
252 apdb = Apdb(config)
253 apdb.makeSchema()
255 pixel_ranges = _makePixelRanges()
256 visit_time = datetime.datetime.now()
258 # get objects by region
259 res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas)
260 self._assertCatalog(res, 0, type=self.data_type)
262 # get sources by region
263 res = apdb.getDiaSourcesInRegion(pixel_ranges, visit_time, return_pandas=self.use_pandas)
264 self._assertCatalog(res, 0, type=self.data_type)
266 # get sources by object ID, empty object list, should return None
267 res = apdb.getDiaSources([], visit_time, return_pandas=self.use_pandas)
268 self.assertIs(res, None)
270 # get sources by object ID, non-empty object list
271 res = apdb.getDiaSources([1, 2, 3], visit_time, return_pandas=self.use_pandas)
272 self._assertCatalog(res, 0, type=self.data_type)
274 # get forced sources by object ID, empty object list
275 res = apdb.getDiaForcedSources([], visit_time, return_pandas=self.use_pandas)
276 self.assertIs(res, None)
278 # get sources by object ID, non-empty object list
279 res = apdb.getDiaForcedSources([1, 2, 3], visit_time, return_pandas=self.use_pandas)
280 self._assertCatalog(res, 0, type=self.data_type)
282 def test_emptyGetsObjectLast(self):
283 """Test for getting DiaObjects from empty database using DiaObjectLast
284 table.
286 All get() methods should return empty results, only useful for
287 checking that code is not broken.
288 """
290 # don't care about sources.
291 config = ApdbConfig(db_url="sqlite:///",
292 isolation_level="READ_UNCOMMITTED",
293 dia_object_index="last_object_table")
294 apdb = Apdb(config)
295 apdb.makeSchema()
297 pixel_ranges = _makePixelRanges()
299 # get objects by region
300 res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas)
301 self._assertCatalog(res, 0, type=self.data_type)
303 def test_storeObjectsBaseline(self):
304 """Store and retrieve DiaObjects."""
306 # don't care about sources.
307 config = ApdbConfig(db_url="sqlite:///",
308 isolation_level="READ_UNCOMMITTED",
309 dia_object_index="baseline")
310 apdb = Apdb(config)
311 apdb.makeSchema()
313 pixel_ranges = _makePixelRanges()
314 visit_time = datetime.datetime.now()
316 # make afw catalog with Objects
317 if self.use_pandas:
318 catalog = _makeObjectCatalogPandas(pixel_ranges)
319 else:
320 catalog = _makeObjectCatalog(pixel_ranges)
322 # store catalog
323 apdb.storeDiaObjects(catalog, visit_time)
325 # read it back and check sizes
326 res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas)
327 self._assertCatalog(res, len(catalog), type=self.data_type)
329 def test_storeObjectsLast(self):
330 """Store and retrieve DiaObjects using DiaObjectLast table."""
331 # don't care about sources.
332 config = ApdbConfig(db_url="sqlite:///",
333 isolation_level="READ_UNCOMMITTED",
334 dia_object_index="last_object_table",
335 object_last_replace=True)
336 apdb = Apdb(config)
337 apdb.makeSchema()
339 pixel_ranges = _makePixelRanges()
340 visit_time = datetime.datetime.now()
342 # make afw catalog with Objects
343 if self.use_pandas:
344 catalog = _makeObjectCatalogPandas(pixel_ranges)
345 else:
346 catalog = _makeObjectCatalog(pixel_ranges)
348 # store catalog
349 apdb.storeDiaObjects(catalog, visit_time)
351 # read it back and check sizes
352 res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas)
353 self._assertCatalog(res, len(catalog), type=self.data_type)
355 def test_storeSources(self):
356 """Store and retrieve DiaSources."""
357 config = ApdbConfig(db_url="sqlite:///",
358 isolation_level="READ_UNCOMMITTED",
359 read_sources_months=12,
360 read_forced_sources_months=12)
361 apdb = Apdb(config)
362 apdb.makeSchema()
364 pixel_ranges = _makePixelRanges()
365 visit_time = datetime.datetime.now()
367 # have to store Objects first
368 if self.use_pandas:
369 objects = _makeObjectCatalogPandas(pixel_ranges)
370 catalog, oids = _makeSourceCatalogPandas(objects)
371 else:
372 objects = _makeObjectCatalog(pixel_ranges)
373 catalog, oids = _makeSourceCatalog(objects)
375 # save the objects
376 apdb.storeDiaObjects(objects, visit_time)
378 # save the sources
379 apdb.storeDiaSources(catalog)
381 # read it back and check sizes
382 res = apdb.getDiaSourcesInRegion(pixel_ranges, visit_time, self.use_pandas)
383 self._assertCatalog(res, len(catalog), type=self.data_type)
385 # read it back using different method
386 res = apdb.getDiaSources(oids, visit_time, self.use_pandas)
387 self._assertCatalog(res, len(catalog), type=self.data_type)
389 def test_storeForcedSources(self):
390 """Store and retrieve DiaForcedSources."""
392 config = ApdbConfig(db_url="sqlite:///",
393 isolation_level="READ_UNCOMMITTED",
394 read_sources_months=12,
395 read_forced_sources_months=12)
396 apdb = Apdb(config)
397 apdb.makeSchema()
399 pixel_ranges = _makePixelRanges()
400 visit_time = datetime.datetime.now()
402 # have to store Objects first
403 if self.use_pandas:
404 objects = _makeObjectCatalogPandas(pixel_ranges)
405 catalog, oids = _makeForcedSourceCatalogPandas(objects)
406 else:
407 objects = _makeObjectCatalog(pixel_ranges)
408 catalog, oids = _makeForcedSourceCatalog(objects)
410 apdb.storeDiaObjects(objects, visit_time)
412 # save them
413 apdb.storeDiaForcedSources(catalog)
415 # read it back and check sizes
416 res = apdb.getDiaForcedSources(oids, visit_time, return_pandas=self.use_pandas)
417 self._assertCatalog(res, len(catalog), type=self.data_type)
420class ApdbPandasTestCase(ApdbTestCase):
421 """A test case for Apdb using Pandas as the input/output"""
423 use_pandas = True
424 data_type = pandas.DataFrame
427class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
428 pass
431def setup_module(module):
432 lsst.utils.tests.init()
435if __name__ == "__main__": 435 ↛ 436line 435 didn't jump to line 436, because the condition on line 435 was never true
436 lsst.utils.tests.init()
437 unittest.main()