Coverage for tests/test_loadDiaCatalogs.py: 22%
142 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-14 16:55 -0700
1# This file is part of ap_association.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os
23import numpy as np
24import pandas as pd
25import tempfile
26import unittest
27import yaml
29from lsst.afw.cameraGeom.testUtils import DetectorWrapper
30import lsst.afw.geom as afwGeom
31import lsst.afw.image as afwImage
32from lsst.ap.association import LoadDiaCatalogsTask, LoadDiaCatalogsConfig
33import lsst.daf.base as dafBase
34from lsst.dax.apdb import ApdbSql, ApdbSqlConfig, ApdbTables
35import lsst.geom as geom
36from lsst.utils import getPackageDir
37import lsst.utils.tests
40def _data_file_name(basename, module_name):
41 """Return path name of a data file.
43 Parameters
44 ----------
45 basename : `str`
46 Name of the file to add to the path string.
47 module_name : `str`
48 Name of lsst stack package environment variable.
50 Returns
51 -------
52 data_file_path : `str`
53 Full path of the file to load from the "data" directory in a given
54 repository.
55 """
56 return os.path.join(getPackageDir(module_name), "data", basename)
59def makeExposure(flipX=False, flipY=False):
60 """Create an exposure and flip the x or y (or both) coordinates.
62 Returns bounding boxes that are right or left handed around the bounding
63 polygon.
65 Parameters
66 ----------
67 flipX : `bool`
68 Flip the x coordinate in the WCS.
69 flipY : `bool`
70 Flip the y coordinate in the WCS.
72 Returns
73 -------
74 exposure : `lsst.afw.image.Exposure`
75 Exposure with a valid bounding box and wcs.
76 """
77 metadata = dafBase.PropertySet()
79 metadata.set("SIMPLE", "T")
80 metadata.set("BITPIX", -32)
81 metadata.set("NAXIS", 2)
82 metadata.set("NAXIS1", 1024)
83 metadata.set("NAXIS2", 1153)
84 metadata.set("RADECSYS", 'FK5')
85 metadata.set("EQUINOX", 2000.)
87 metadata.setDouble("CRVAL1", 215.604025685476)
88 metadata.setDouble("CRVAL2", 53.1595451514076)
89 metadata.setDouble("CRPIX1", 1109.99981456774)
90 metadata.setDouble("CRPIX2", 560.018167811613)
91 metadata.set("CTYPE1", 'RA---SIN')
92 metadata.set("CTYPE2", 'DEC--SIN')
94 xFlip = 1
95 if flipX:
96 xFlip = -1
97 yFlip = 1
98 if flipY:
99 yFlip = -1
100 metadata.setDouble("CD1_1", xFlip * 5.10808596133527E-05)
101 metadata.setDouble("CD1_2", yFlip * 1.85579539217196E-07)
102 metadata.setDouble("CD2_2", yFlip * -5.10281493481982E-05)
103 metadata.setDouble("CD2_1", xFlip * -8.27440751733828E-07)
105 wcs = afwGeom.makeSkyWcs(metadata)
106 exposure = afwImage.makeExposure(
107 afwImage.makeMaskedImageFromArrays(np.ones((1024, 1153))), wcs)
108 detector = DetectorWrapper(id=23, bbox=exposure.getBBox()).detector
109 visit = afwImage.VisitInfo(
110 exposureId=1234,
111 exposureTime=200.,
112 date=dafBase.DateTime("2014-05-13T17:00:00.000000000",
113 dafBase.DateTime.Timescale.TAI))
114 exposure.info.id = 1234
115 exposure.setDetector(detector)
116 exposure.getInfo().setVisitInfo(visit)
117 exposure.setFilter(afwImage.FilterLabel(band='g'))
119 return exposure
122def makeDiaObjects(nObjects, exposure):
123 """Make a test set of DiaObjects.
125 Parameters
126 ----------
127 nObjects : `int`
128 Number of objects to create.
129 exposure : `lsst.afw.image.Exposure`
130 Exposure to create objects over.
132 Returns
133 -------
134 diaObjects : `pandas.DataFrame`
135 DiaObjects generated across the exposure.
136 """
137 bbox = geom.Box2D(exposure.getBBox())
138 rand_x = np.random.uniform(bbox.getMinX(), bbox.getMaxX(), size=nObjects)
139 rand_y = np.random.uniform(bbox.getMinY(), bbox.getMaxY(), size=nObjects)
141 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get(
142 system=dafBase.DateTime.MJD)
144 wcs = exposure.getWcs()
146 data = []
147 for idx, (x, y) in enumerate(zip(rand_x, rand_y)):
148 coord = wcs.pixelToSky(x, y)
149 newObject = {"ra": coord.getRa().asDegrees(),
150 "decl": coord.getDec().asDegrees(),
151 "radecTai": midPointTaiMJD,
152 "diaObjectId": idx,
153 "pmParallaxNdata": 0,
154 "nearbyObj1": 0,
155 "nearbyObj2": 0,
156 "nearbyObj3": 0,
157 "nDiaSources": 1}
158 for f in ["u", "g", "r", "i", "z", "y"]:
159 newObject["%sPSFluxNdata" % f] = 0
160 data.append(newObject)
162 return pd.DataFrame(data=data)
165def makeDiaSources(nSources, diaObjectIds, exposure):
166 """Make a test set of DiaSources.
168 Parameters
169 ----------
170 nSources : `int`
171 Number of sources to create.
172 diaObjectIds : `numpy.ndarray`
173 Integer Ids of diaobjects to "associate" with the DiaSources.
174 exposure : `lsst.afw.image.Exposure`
175 Exposure to create sources over.
177 Returns
178 -------
179 diaSources : `pandas.DataFrame`
180 DiaSources generated across the exposure.
181 """
182 bbox = geom.Box2D(exposure.getBBox())
183 rand_x = np.random.uniform(bbox.getMinX(), bbox.getMaxX(), size=nSources)
184 rand_y = np.random.uniform(bbox.getMinY(), bbox.getMaxY(), size=nSources)
185 rand_ids = diaObjectIds[np.random.randint(len(diaObjectIds), size=nSources)]
187 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get(
188 system=dafBase.DateTime.MJD)
190 wcs = exposure.getWcs()
192 data = []
193 for idx, (x, y, objId) in enumerate(zip(rand_x, rand_y, rand_ids)):
194 coord = wcs.pixelToSky(x, y)
195 data.append({"ra": coord.getRa().asDegrees(),
196 "decl": coord.getDec().asDegrees(),
197 "diaObjectId": objId,
198 "diaSourceId": idx,
199 "midPointTai": midPointTaiMJD,
200 "ccdVisitId": 0,
201 "x": 0.5,
202 "y": 0.5})
204 return pd.DataFrame(data=data)
207def makeDiaForcedSources(nForcedSources, diaObjectIds, exposure):
208 """Make a test set of DiaForcedSources.
210 Parameters
211 ----------
212 nForcedSources : `int`
213 Number of sources to create.
214 diaObjectIds : `numpy.ndarray`
215 Integer Ids of diaobjects to "associate" with the DiaSources.
216 exposure : `lsst.afw.image.Exposure`
217 Exposure to create sources over.
219 Returns
220 -------
221 diaForcedSources : `pandas.DataFrame`
222 DiaForcedSources generated across the exposure.
223 """
224 rand_ids = diaObjectIds[
225 np.random.randint(len(diaObjectIds), size=nForcedSources)]
227 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get(
228 system=dafBase.DateTime.MJD)
230 data = []
231 for idx, objId in enumerate(rand_ids):
232 data.append({"diaObjectId": objId,
233 "diaForcedSourceId": idx,
234 "ccdVisitId": idx,
235 "midPointTai": midPointTaiMJD})
237 return pd.DataFrame(data=data)
240class TestLoadDiaCatalogs(unittest.TestCase):
242 def setUp(self):
243 np.random.seed(1234)
245 self.db_file_fd, self.db_file = tempfile.mkstemp(
246 dir=os.path.dirname(__file__))
248 self.apdbConfig = ApdbSqlConfig()
249 self.apdbConfig.db_url = "sqlite:///" + self.db_file
250 self.apdbConfig.dia_object_index = "baseline"
251 self.apdbConfig.dia_object_columns = []
253 self.apdb = ApdbSql(config=self.apdbConfig)
254 self.apdb.makeSchema()
256 self.exposure = makeExposure(False, False)
258 self.diaObjects = makeDiaObjects(20, self.exposure)
259 self.diaSources = makeDiaSources(
260 100,
261 self.diaObjects["diaObjectId"].to_numpy(),
262 self.exposure)
263 self.diaForcedSources = makeDiaForcedSources(
264 200,
265 self.diaObjects["diaObjectId"].to_numpy(),
266 self.exposure)
268 self.dateTime = self.exposure.getInfo().getVisitInfo().getDate()
269 self.apdb.store(self.dateTime,
270 self.diaObjects,
271 self.diaSources,
272 self.diaForcedSources)
274 # These columns are not in the DPDD, yet do appear in DiaSource.yaml.
275 # We don't need to check them against the default APDB schema.
276 self.ignoreColumns = ["filterName", "bboxSize", "isDipole"]
278 def tearDown(self):
279 os.close(self.db_file_fd)
280 os.remove(self.db_file)
282 def testRun(self):
283 """Test the full run method for the loader.
284 """
285 diaLoader = LoadDiaCatalogsTask()
286 result = diaLoader.run(self.exposure, self.apdb)
288 self.assertEqual(len(result.diaObjects), len(self.diaObjects))
289 self.assertEqual(len(result.diaSources), len(self.diaSources))
290 self.assertEqual(len(result.diaForcedSources),
291 len(self.diaForcedSources))
293 def testLoadDiaObjects(self):
294 """Test that the correct number of diaObjects are loaded.
295 """
296 diaLoader = LoadDiaCatalogsTask()
297 region = diaLoader._getRegion(self.exposure)
298 diaObjects = diaLoader.loadDiaObjects(region,
299 self.apdb)
300 self.assertEqual(len(diaObjects), len(self.diaObjects))
302 def testLoadDiaForcedSources(self):
303 """Test that the correct number of diaForcedSources are loaded.
304 """
305 diaLoader = LoadDiaCatalogsTask()
306 region = diaLoader._getRegion(self.exposure)
307 diaForcedSources = diaLoader.loadDiaForcedSources(
308 self.diaObjects,
309 region,
310 self.dateTime,
311 self.apdb)
312 self.assertEqual(len(diaForcedSources), len(self.diaForcedSources))
314 def testLoadDiaSources(self):
315 """Test that the correct number of diaSources are loaded.
317 Also check that they can be properly loaded both by location and
318 ``diaObjectId``.
319 """
320 diaConfig = LoadDiaCatalogsConfig()
321 diaLoader = LoadDiaCatalogsTask(config=diaConfig)
323 region = diaLoader._getRegion(self.exposure)
324 diaSources = diaLoader.loadDiaSources(self.diaObjects,
325 region,
326 self.dateTime,
327 self.apdb)
328 self.assertEqual(len(diaSources), len(self.diaSources))
330 def test_apdbSchema(self):
331 """Test that the default DiaSource schema from dax_apdb agrees with the
332 column names defined here in ap_association/data/DiaSource.yaml.
333 """
334 tableDef = self.apdb.tableDef(ApdbTables.DiaSource)
335 apdbSchemaColumns = [column.name for column in tableDef.columns]
337 functorFile = _data_file_name("DiaSource.yaml", "ap_association")
338 with open(functorFile) as yaml_stream:
339 diaSourceFunctor = yaml.safe_load_all(yaml_stream)
340 for functor in diaSourceFunctor:
341 diaSourceColumns = [column for column in list(functor['funcs'].keys())
342 if column not in self.ignoreColumns]
343 self.assertLess(set(diaSourceColumns), set(apdbSchemaColumns))
346class MemoryTester(lsst.utils.tests.MemoryTestCase):
347 pass
350def setup_module(module):
351 lsst.utils.tests.init()
354if __name__ == "__main__": 354 ↛ 355line 354 didn't jump to line 355, because the condition on line 354 was never true
355 lsst.utils.tests.init()
356 unittest.main()