Coverage for tests/test_loadDiaCatalogs.py : 20%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ap_association.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os
23import numpy as np
24import pandas as pd
25import tempfile
26import unittest
28from lsst.afw.cameraGeom.testUtils import DetectorWrapper
29import lsst.afw.geom as afwGeom
30import lsst.afw.image as afwImage
31from lsst.ap.association import (LoadDiaCatalogsTask,
32 LoadDiaCatalogsConfig,
33 make_dia_source_schema,
34 make_dia_object_schema)
35import lsst.daf.base as dafBase
36from lsst.dax.apdb import Apdb, ApdbConfig
37import lsst.geom as geom
38import lsst.sphgeom as sphgeom
39from lsst.utils import getPackageDir
40import lsst.utils.tests
43def _data_file_name(basename, module_name):
44 """Return path name of a data file.
46 Parameters
47 ----------
48 basename : `str`
49 Name of the file to add to the path string.
50 module_name : `str`
51 Name of lsst stack package environment variable.
53 Returns
54 -------
55 data_file_path : `str`
56 Full path of the file to load from the "data" directory in a given
57 repository.
58 """
59 return os.path.join(getPackageDir(module_name), "data", basename)
62def makeExposure(flipX=False, flipY=False):
63 """Create an exposure and flip the x or y (or both) coordinates.
65 Returns bounding boxes that are right or left handed around the bounding
66 polygon.
68 Parameters
69 ----------
70 flipX : `bool`
71 Flip the x coordinate in the WCS.
72 flipY : `bool`
73 Flip the y coordinate in the WCS.
75 Returns
76 -------
77 exposure : `lsst.afw.image.Exposure`
78 Exposure with a valid bounding box and wcs.
79 """
80 metadata = dafBase.PropertySet()
82 metadata.set("SIMPLE", "T")
83 metadata.set("BITPIX", -32)
84 metadata.set("NAXIS", 2)
85 metadata.set("NAXIS1", 1024)
86 metadata.set("NAXIS2", 1153)
87 metadata.set("RADECSYS", 'FK5')
88 metadata.set("EQUINOX", 2000.)
90 metadata.setDouble("CRVAL1", 215.604025685476)
91 metadata.setDouble("CRVAL2", 53.1595451514076)
92 metadata.setDouble("CRPIX1", 1109.99981456774)
93 metadata.setDouble("CRPIX2", 560.018167811613)
94 metadata.set("CTYPE1", 'RA---SIN')
95 metadata.set("CTYPE2", 'DEC--SIN')
97 xFlip = 1
98 if flipX:
99 xFlip = -1
100 yFlip = 1
101 if flipY:
102 yFlip = -1
103 metadata.setDouble("CD1_1", xFlip * 5.10808596133527E-05)
104 metadata.setDouble("CD1_2", yFlip * 1.85579539217196E-07)
105 metadata.setDouble("CD2_2", yFlip * -5.10281493481982E-05)
106 metadata.setDouble("CD2_1", xFlip * -8.27440751733828E-07)
108 wcs = afwGeom.makeSkyWcs(metadata)
109 exposure = afwImage.makeExposure(
110 afwImage.makeMaskedImageFromArrays(np.ones((1024, 1153))), wcs)
111 detector = DetectorWrapper(id=23, bbox=exposure.getBBox()).detector
112 visit = afwImage.VisitInfo(
113 exposureId=1234,
114 exposureTime=200.,
115 date=dafBase.DateTime("2014-05-13T17:00:00.000000000",
116 dafBase.DateTime.Timescale.TAI))
117 exposure.setDetector(detector)
118 exposure.getInfo().setVisitInfo(visit)
119 exposure.setFilterLabel(afwImage.FilterLabel(band='g'))
121 return exposure
124def makeDiaObjects(nObjects, exposure, pixelator):
125 """Make a test set of DiaObjects.
127 Parameters
128 ----------
129 nObjects : `int`
130 Number of objects to create.
131 exposure : `lsst.afw.image.Exposure`
132 Exposure to create objects over.
133 pixelator : `lsst.sphgeom.HtmPixelization`
134 Object to compute spatial indicies from.
136 Returns
137 -------
138 diaObjects : `pandas.DataFrame`
139 DiaObjects generated across the exposure.
140 """
141 bbox = geom.Box2D(exposure.getBBox())
142 rand_x = np.random.uniform(bbox.getMinX(), bbox.getMaxX(), size=nObjects)
143 rand_y = np.random.uniform(bbox.getMinY(), bbox.getMaxY(), size=nObjects)
145 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get(
146 system=dafBase.DateTime.MJD)
148 wcs = exposure.getWcs()
150 data = []
151 for idx, (x, y) in enumerate(zip(rand_x, rand_y)):
152 coord = wcs.pixelToSky(x, y)
153 htmIdx = pixelator.index(coord.getVector())
154 newObject = {"ra": coord.getRa().asDegrees(),
155 "decl": coord.getRa().asDegrees(),
156 "radecTai": midPointTaiMJD,
157 "diaObjectId": idx,
158 "pixelId": htmIdx,
159 "pmParallaxNdata": 0,
160 "nearbyObj1": 0,
161 "nearbyObj2": 0,
162 "nearbyObj3": 0}
163 for f in ["u", "g", "r", "i", "z", "y"]:
164 newObject["%sPSFluxNdata" % f] = 0
165 data.append(newObject)
167 return pd.DataFrame(data=data)
170def makeDiaSources(nSources, diaObjectIds, exposure, pixelator):
171 """Make a test set of DiaSources.
173 Parameters
174 ----------
175 nSources : `int`
176 Number of sources to create.
177 diaObjectIds : `numpy.ndarray`
178 Integer Ids of diaobjects to "associate" with the DiaSources.
179 exposure : `lsst.afw.image.Exposure`
180 Exposure to create sources over.
181 pixelator : `lsst.sphgeom.HtmPixelization`
182 Object to compute spatial indicies from.
184 Returns
185 -------
186 diaSources : `pandas.DataFrame`
187 DiaSources generated across the exposure.
188 """
189 bbox = geom.Box2D(exposure.getBBox())
190 rand_x = np.random.uniform(bbox.getMinX(), bbox.getMaxX(), size=nSources)
191 rand_y = np.random.uniform(bbox.getMinY(), bbox.getMaxY(), size=nSources)
192 rand_ids = diaObjectIds[np.random.randint(len(diaObjectIds), size=nSources)]
194 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get(
195 system=dafBase.DateTime.MJD)
197 wcs = exposure.getWcs()
199 data = []
200 for idx, (x, y, objId) in enumerate(zip(rand_x, rand_y, rand_ids)):
201 coord = wcs.pixelToSky(x, y)
202 htmIdx = pixelator.index(coord.getVector())
203 data.append({"ra": coord.getRa().asDegrees(),
204 "decl": coord.getRa().asDegrees(),
205 "diaObjectId": objId,
206 "diaSourceId": idx,
207 "pixelId": htmIdx,
208 "midPointTai": midPointTaiMJD})
210 return pd.DataFrame(data=data)
213def makeDiaForcedSources(nForcedSources, diaObjectIds, exposure):
214 """Make a test set of DiaForcedSources.
216 Parameters
217 ----------
218 nForcedSources : `int`
219 Number of sources to create.
220 diaObjectIds : `numpy.ndarray`
221 Integer Ids of diaobjects to "associate" with the DiaSources.
222 exposure : `lsst.afw.image.Exposure`
223 Exposure to create sources over.
225 Returns
226 -------
227 diaForcedSources : `pandas.DataFrame`
228 DiaForcedSources generated across the exposure.
229 """
230 rand_ids = diaObjectIds[
231 np.random.randint(len(diaObjectIds), size=nForcedSources)]
233 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get(
234 system=dafBase.DateTime.MJD)
236 data = []
237 for idx, objId in enumerate(rand_ids):
238 data.append({"diaObjectId": objId,
239 "diaForcedSourceId": idx,
240 "ccdVisitId": idx,
241 "midPointTai": midPointTaiMJD})
243 return pd.DataFrame(data=data)
246class TestLoadDiaCatalogs(unittest.TestCase):
248 def setUp(self):
249 np.random.seed(1234)
251 self.db_file_fd, self.db_file = tempfile.mkstemp(
252 dir=os.path.dirname(__file__))
254 self.apdbConfig = ApdbConfig()
255 self.apdbConfig.db_url = "sqlite:///" + self.db_file
256 self.apdbConfig.isolation_level = "READ_UNCOMMITTED"
257 self.apdbConfig.dia_object_index = "baseline"
258 self.apdbConfig.dia_object_columns = []
259 self.apdbConfig.schema_file = _data_file_name(
260 "apdb-schema.yaml", "dax_apdb")
261 self.apdbConfig.column_map = _data_file_name(
262 "apdb-ap-pipe-afw-map.yaml", "ap_association")
263 self.apdbConfig.extra_schema_file = _data_file_name(
264 "apdb-ap-pipe-schema-extra.yaml", "ap_association")
266 self.apdb = Apdb(config=self.apdbConfig,
267 afw_schemas=dict(DiaObject=make_dia_object_schema(),
268 DiaSource=make_dia_source_schema()))
269 self.apdb.makeSchema()
271 # Expected HTM pixel ranges for max range=4 and level = 20. This
272 # set of pixels should be same for the WCS created by default in
273 # makeExposure and for one with a flipped y axis.
274 self.ranges = np.sort(np.array([15154776375296, 15154779521024,
275 15154788958208, 15154792103936]))
277 self.pixelator = sphgeom.HtmPixelization(20)
278 self.exposure = makeExposure(False, False)
280 self.diaObjects = makeDiaObjects(20, self.exposure, self.pixelator)
281 self.diaSources = makeDiaSources(
282 100,
283 self.diaObjects["diaObjectId"].to_numpy(),
284 self.exposure,
285 self.pixelator)
286 self.diaForcedSources = makeDiaForcedSources(
287 200,
288 self.diaObjects["diaObjectId"].to_numpy(),
289 self.exposure)
291 self.apdb.storeDiaSources(self.diaSources)
292 self.apdb.storeDiaForcedSources(self.diaForcedSources)
293 self.dateTime = \
294 self.exposure.getInfo().getVisitInfo().getDate().toPython()
295 self.apdb.storeDiaObjects(self.diaObjects,
296 self.dateTime)
298 def tearDown(self):
299 os.close(self.db_file_fd)
300 os.remove(self.db_file)
302 def testRun(self):
303 """Test the full run method for the loader.
304 """
305 diaLoader = LoadDiaCatalogsTask()
306 result = diaLoader.run(self.exposure, self.apdb)
308 self.assertEqual(len(result.diaObjects), len(self.diaObjects))
309 self.assertEqual(len(result.diaSources), len(self.diaSources))
310 self.assertEqual(len(result.diaForcedSources),
311 len(self.diaForcedSources))
313 def testLoadDiaObjects(self):
314 """Test that the correct number of diaObjects are loaded.
315 """
316 diaLoader = LoadDiaCatalogsTask()
317 normPixels = diaLoader._getPixelRanges(self.exposure)
318 diaObjects = diaLoader.loadDiaObjects(normPixels,
319 self.apdb)
320 self.assertEqual(len(diaObjects), len(self.diaObjects))
322 def testLoadDiaForcedSources(self):
323 """Test that the correct number of diaForcedSources are loaded.
324 """
325 diaLoader = LoadDiaCatalogsTask()
326 diaForcedSources = diaLoader.loadDiaForcedSources(
327 self.diaObjects,
328 self.dateTime,
329 self.apdb)
330 self.assertEqual(len(diaForcedSources), len(self.diaForcedSources))
332 def testLoadDiaSourcesByPixelId(self):
333 """Test that the correct number of diaSources are loaded.
335 Also check that they can be properly loaded both by location and
336 ``diaObjectId``.
337 """
338 self._testLoadDiaSources(True)
340 def testLoadDiaSourcesByDiaObjectId(self):
341 """Test that the correct number of diaSources are loaded.
343 Also check that they can be properly loaded both by location and
344 ``diaObjectId``.
345 """
346 self._testLoadDiaSources(False)
348 def _testLoadDiaSources(self, loadByPixelId):
349 """Test that DiaSources are loaded correctly.
351 Parameters
352 ----------
353 loadByPixelId : `bool`
354 Load DiaSources by ``pixelId`` if ``True`` and by ``diaObjectId``
355 if ``False``.
356 """
357 diaConfig = LoadDiaCatalogsConfig()
358 diaConfig.loadDiaSourcesByPixelId = loadByPixelId
359 diaLoader = LoadDiaCatalogsTask(config=diaConfig)
361 normPixels = diaLoader._getPixelRanges(self.exposure)
362 diaSources = diaLoader.loadDiaSources(self.diaObjects,
363 normPixels,
364 self.dateTime,
365 self.apdb)
366 self.assertEqual(len(diaSources), len(self.diaSources))
368 def testGetPixelRanges(self):
369 """Test the same pixels are returned for flips/ordering changes in
370 the WCS.
371 """
372 diaConfig = LoadDiaCatalogsConfig()
373 diaConfig.pixelMargin = 300 # overriding to value tests were conditioned on
374 diaConfig.htmMaxRanges = 4
375 diaLoader = LoadDiaCatalogsTask(config=diaConfig)
377 # Make two exposures, one with a flipped y axis to get left vs. right
378 # handed.
379 exposure = makeExposure(False, False)
380 exposureFlip = makeExposure(False, True)
382 normPixels = diaLoader._getPixelRanges(exposure)
383 flipPixels = diaLoader._getPixelRanges(exposureFlip)
385 for normPix, flipPix, testPix in zip(
386 np.sort(np.array(normPixels).flatten()),
387 np.sort(np.array(flipPixels).flatten()),
388 self.ranges):
389 self.assertEqual(normPix, flipPix)
390 self.assertEqual(normPix, testPix)
391 self.assertEqual(flipPix, testPix)
394class MemoryTester(lsst.utils.tests.MemoryTestCase):
395 pass
398def setup_module(module):
399 lsst.utils.tests.init()
402if __name__ == "__main__": 402 ↛ 403line 402 didn't jump to line 403, because the condition on line 402 was never true
403 lsst.utils.tests.init()
404 unittest.main()