Coverage for tests/test_loadDiaCatalogs.py : 20%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ap_association.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os
23import numpy as np
24import pandas as pd
25import tempfile
26import unittest
28from lsst.afw.cameraGeom.testUtils import DetectorWrapper
29import lsst.afw.geom as afwGeom
30import lsst.afw.image as afwImage
31import lsst.afw.image.utils as afwImageUtils
32from lsst.ap.association import (LoadDiaCatalogsTask,
33 LoadDiaCatalogsConfig,
34 make_dia_source_schema,
35 make_dia_object_schema)
36import lsst.daf.base as dafBase
37from lsst.dax.apdb import Apdb, ApdbConfig
38import lsst.geom as geom
39import lsst.sphgeom as sphgeom
40from lsst.utils import getPackageDir
41import lsst.utils.tests
44def _data_file_name(basename, module_name):
45 """Return path name of a data file.
47 Parameters
48 ----------
49 basename : `str`
50 Name of the file to add to the path string.
51 module_name : `str`
52 Name of lsst stack package environment variable.
54 Returns
55 -------
56 data_file_path : `str`
57 Full path of the file to load from the "data" directory in a given
58 repository.
59 """
60 return os.path.join(getPackageDir(module_name), "data", basename)
63def makeExposure(flipX=False, flipY=False):
64 """Create an exposure and flip the x or y (or both) coordinates.
66 Returns bounding boxes that are right or left handed around the bounding
67 polygon.
69 Parameters
70 ----------
71 flipX : `bool`
72 Flip the x coordinate in the WCS.
73 flipY : `bool`
74 Flip the y coordinate in the WCS.
76 Returns
77 -------
78 exposure : `lsst.afw.image.Exposure`
79 Exposure with a valid bounding box and wcs.
80 """
81 metadata = dafBase.PropertySet()
83 metadata.set("SIMPLE", "T")
84 metadata.set("BITPIX", -32)
85 metadata.set("NAXIS", 2)
86 metadata.set("NAXIS1", 1024)
87 metadata.set("NAXIS2", 1153)
88 metadata.set("RADECSYS", 'FK5')
89 metadata.set("EQUINOX", 2000.)
91 metadata.setDouble("CRVAL1", 215.604025685476)
92 metadata.setDouble("CRVAL2", 53.1595451514076)
93 metadata.setDouble("CRPIX1", 1109.99981456774)
94 metadata.setDouble("CRPIX2", 560.018167811613)
95 metadata.set("CTYPE1", 'RA---SIN')
96 metadata.set("CTYPE2", 'DEC--SIN')
98 xFlip = 1
99 if flipX:
100 xFlip = -1
101 yFlip = 1
102 if flipY:
103 yFlip = -1
104 metadata.setDouble("CD1_1", xFlip * 5.10808596133527E-05)
105 metadata.setDouble("CD1_2", yFlip * 1.85579539217196E-07)
106 metadata.setDouble("CD2_2", yFlip * -5.10281493481982E-05)
107 metadata.setDouble("CD2_1", xFlip * -8.27440751733828E-07)
109 wcs = afwGeom.makeSkyWcs(metadata)
110 exposure = afwImage.makeExposure(
111 afwImage.makeMaskedImageFromArrays(np.ones((1024, 1153))), wcs)
112 detector = DetectorWrapper(id=23, bbox=exposure.getBBox()).detector
113 visit = afwImage.VisitInfo(
114 exposureId=1234,
115 exposureTime=200.,
116 date=dafBase.DateTime("2014-05-13T17:00:00.000000000",
117 dafBase.DateTime.Timescale.TAI))
118 exposure.setDetector(detector)
119 exposure.getInfo().setVisitInfo(visit)
120 exposure.setFilter(afwImage.Filter('g'))
122 return exposure
125def makeDiaObjects(nObjects, exposure, pixelator):
126 """Make a test set of DiaObjects.
128 Parameters
129 ----------
130 nObjects : `int`
131 Number of objects to create.
132 exposure : `lsst.afw.image.Exposure`
133 Exposure to create objects over.
134 pixelator : `lsst.sphgeom.HtmPixelization`
135 Object to compute spatial indicies from.
137 Returns
138 -------
139 diaObjects : `pandas.DataFrame`
140 DiaObjects generated across the exposure.
141 """
142 bbox = geom.Box2D(exposure.getBBox())
143 rand_x = np.random.uniform(bbox.getMinX(), bbox.getMaxX(), size=nObjects)
144 rand_y = np.random.uniform(bbox.getMinY(), bbox.getMaxY(), size=nObjects)
146 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get(
147 system=dafBase.DateTime.MJD)
149 wcs = exposure.getWcs()
151 data = []
152 for idx, (x, y) in enumerate(zip(rand_x, rand_y)):
153 coord = wcs.pixelToSky(x, y)
154 htmIdx = pixelator.index(coord.getVector())
155 newObject = {"ra": coord.getRa().asDegrees(),
156 "decl": coord.getRa().asDegrees(),
157 "radecTai": midPointTaiMJD,
158 "diaObjectId": idx,
159 "pixelId": htmIdx,
160 "pmParallaxNdata": 0,
161 "nearbyObj1": 0,
162 "nearbyObj2": 0,
163 "nearbyObj3": 0}
164 for f in ["u", "g", "r", "i", "z", "y"]:
165 newObject["%sPSFluxNdata" % f] = 0
166 data.append(newObject)
168 return pd.DataFrame(data=data)
171def makeDiaSources(nSources, diaObjectIds, exposure, pixelator):
172 """Make a test set of DiaSources.
174 Parameters
175 ----------
176 nSources : `int`
177 Number of sources to create.
178 diaObjectIds : `numpy.ndarray`
179 Integer Ids of diaobjects to "associate" with the DiaSources.
180 exposure : `lsst.afw.image.Exposure`
181 Exposure to create sources over.
182 pixelator : `lsst.sphgeom.HtmPixelization`
183 Object to compute spatial indicies from.
185 Returns
186 -------
187 diaSources : `pandas.DataFrame`
188 DiaSources generated across the exposure.
189 """
190 bbox = geom.Box2D(exposure.getBBox())
191 rand_x = np.random.uniform(bbox.getMinX(), bbox.getMaxX(), size=nSources)
192 rand_y = np.random.uniform(bbox.getMinY(), bbox.getMaxY(), size=nSources)
193 rand_ids = diaObjectIds[np.random.randint(len(diaObjectIds), size=nSources)]
195 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get(
196 system=dafBase.DateTime.MJD)
198 wcs = exposure.getWcs()
200 data = []
201 for idx, (x, y, objId) in enumerate(zip(rand_x, rand_y, rand_ids)):
202 coord = wcs.pixelToSky(x, y)
203 htmIdx = pixelator.index(coord.getVector())
204 data.append({"ra": coord.getRa().asDegrees(),
205 "decl": coord.getRa().asDegrees(),
206 "diaObjectId": objId,
207 "diaSourceId": idx,
208 "pixelId": htmIdx,
209 "midPointTai": midPointTaiMJD})
211 return pd.DataFrame(data=data)
214def makeDiaForcedSources(nForcedSources, diaObjectIds, exposure):
215 """Make a test set of DiaForcedSources.
217 Parameters
218 ----------
219 nForcedSources : `int`
220 Number of sources to create.
221 diaObjectIds : `numpy.ndarray`
222 Integer Ids of diaobjects to "associate" with the DiaSources.
223 exposure : `lsst.afw.image.Exposure`
224 Exposure to create sources over.
226 Returns
227 -------
228 diaForcedSources : `pandas.DataFrame`
229 DiaForcedSources generated across the exposure.
230 """
231 rand_ids = diaObjectIds[
232 np.random.randint(len(diaObjectIds), size=nForcedSources)]
234 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get(
235 system=dafBase.DateTime.MJD)
237 data = []
238 for idx, objId in enumerate(rand_ids):
239 data.append({"diaObjectId": objId,
240 "diaForcedSourceId": idx,
241 "ccdVisitId": idx,
242 "midPointTai": midPointTaiMJD})
244 return pd.DataFrame(data=data)
247class TestLoadDiaCatalogs(unittest.TestCase):
249 def setUp(self):
250 np.random.seed(1234)
252 # CFHT Filters from the camera mapper.
253 self.filter_names = ["g"]
254 afwImageUtils.resetFilters()
255 afwImageUtils.defineFilter('g', lambdaEff=487, alias="g.MP9401")
257 self.db_file_fd, self.db_file = tempfile.mkstemp(
258 dir=os.path.dirname(__file__))
260 self.apdbConfig = ApdbConfig()
261 self.apdbConfig.db_url = "sqlite:///" + self.db_file
262 self.apdbConfig.isolation_level = "READ_UNCOMMITTED"
263 self.apdbConfig.dia_object_index = "baseline"
264 self.apdbConfig.dia_object_columns = []
265 self.apdbConfig.schema_file = _data_file_name(
266 "apdb-schema.yaml", "dax_apdb")
267 self.apdbConfig.column_map = _data_file_name(
268 "apdb-ap-pipe-afw-map.yaml", "ap_association")
269 self.apdbConfig.extra_schema_file = _data_file_name(
270 "apdb-ap-pipe-schema-extra.yaml", "ap_association")
272 self.apdb = Apdb(config=self.apdbConfig,
273 afw_schemas=dict(DiaObject=make_dia_object_schema(),
274 DiaSource=make_dia_source_schema()))
275 self.apdb.makeSchema()
277 # Expected HTM pixel ranges for max range=4 and level = 20. This
278 # set of pixels should be same for the WCS created by default in
279 # makeExposure and for one with a flipped y axis.
280 self.ranges = np.sort(np.array([15154776375296, 15154779521024,
281 15154788958208, 15154792103936]))
283 self.pixelator = sphgeom.HtmPixelization(20)
284 self.exposure = makeExposure(False, False)
286 self.diaObjects = makeDiaObjects(20, self.exposure, self.pixelator)
287 self.diaSources = makeDiaSources(
288 100,
289 self.diaObjects["diaObjectId"].to_numpy(),
290 self.exposure,
291 self.pixelator)
292 self.diaForcedSources = makeDiaForcedSources(
293 200,
294 self.diaObjects["diaObjectId"].to_numpy(),
295 self.exposure)
297 self.apdb.storeDiaSources(self.diaSources)
298 self.apdb.storeDiaForcedSources(self.diaForcedSources)
299 self.dateTime = \
300 self.exposure.getInfo().getVisitInfo().getDate().toPython()
301 self.apdb.storeDiaObjects(self.diaObjects,
302 self.dateTime)
304 def tearDown(self):
305 os.close(self.db_file_fd)
306 os.remove(self.db_file)
308 def testRun(self):
309 """Test the full run method for the loader.
310 """
311 diaLoader = LoadDiaCatalogsTask()
312 result = diaLoader.run(self.exposure, self.apdb)
314 self.assertEqual(len(result.diaObjects), len(self.diaObjects))
315 self.assertEqual(len(result.diaSources), len(self.diaSources))
316 self.assertEqual(len(result.diaForcedSources),
317 len(self.diaForcedSources))
319 def testLoadDiaObjects(self):
320 """Test that the correct number of diaObjects are loaded.
321 """
322 diaLoader = LoadDiaCatalogsTask()
323 normPixels = diaLoader._getPixelRanges(self.exposure)
324 diaObjects = diaLoader.loadDiaObjects(normPixels,
325 self.apdb)
326 self.assertEqual(len(diaObjects), len(self.diaObjects))
328 def testLoadDiaForcedSources(self):
329 """Test that the correct number of diaForcedSources are loaded.
330 """
331 diaLoader = LoadDiaCatalogsTask()
332 diaForcedSources = diaLoader.loadDiaForcedSources(
333 self.diaObjects,
334 self.dateTime,
335 self.apdb)
336 self.assertEqual(len(diaForcedSources), len(self.diaForcedSources))
338 def testLoadDiaSourcesByPixelId(self):
339 """Test that the correct number of diaSources are loaded.
341 Also check that they can be properly loaded both by location and
342 ``diaObjectId``.
343 """
344 self._testLoadDiaSources(True)
346 def testLoadDiaSourcesByDiaObjectId(self):
347 """Test that the correct number of diaSources are loaded.
349 Also check that they can be properly loaded both by location and
350 ``diaObjectId``.
351 """
352 self._testLoadDiaSources(False)
354 def _testLoadDiaSources(self, loadByPixelId):
355 """Test that DiaSources are loaded correctly.
357 Parameters
358 ----------
359 loadByPixelId : `bool`
360 Load DiaSources by ``pixelId`` if ``True`` and by ``diaObjectId``
361 if ``False``.
362 """
363 diaConfig = LoadDiaCatalogsConfig()
364 diaConfig.loadDiaSourcesByPixelId = loadByPixelId
365 diaLoader = LoadDiaCatalogsTask(config=diaConfig)
367 normPixels = diaLoader._getPixelRanges(self.exposure)
368 diaSources = diaLoader.loadDiaSources(self.diaObjects,
369 normPixels,
370 self.dateTime,
371 self.apdb)
372 self.assertEqual(len(diaSources), len(self.diaSources))
374 def testGetPixelRanges(self):
375 """Test the same pixels are returned for flips/ordering changes in
376 the WCS.
377 """
378 diaConfig = LoadDiaCatalogsConfig()
379 diaConfig.htmMaxRanges = 4
380 diaLoader = LoadDiaCatalogsTask(config=diaConfig)
382 # Make two exposures, one with a flipped y axis to get left vs. right
383 # handed.
384 exposure = makeExposure(False, False)
385 exposureFlip = makeExposure(False, True)
387 normPixels = diaLoader._getPixelRanges(exposure)
388 flipPixels = diaLoader._getPixelRanges(exposureFlip)
390 for normPix, flipPix, testPix in zip(
391 np.sort(np.array(normPixels).flatten()),
392 np.sort(np.array(flipPixels).flatten()),
393 self.ranges):
394 self.assertEqual(normPix, flipPix)
395 self.assertEqual(normPix, testPix)
396 self.assertEqual(flipPix, testPix)
399class MemoryTester(lsst.utils.tests.MemoryTestCase):
400 pass
403def setup_module(module):
404 lsst.utils.tests.init()
407if __name__ == "__main__": 407 ↛ 408line 407 didn't jump to line 408, because the condition on line 407 was never true
408 lsst.utils.tests.init()
409 unittest.main()