Coverage for tests/test_loadDiaCatalogs.py: 18%

1# This file is part of ap_association. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import numpy as np 

24import pandas as pd 

25import tempfile 

26import unittest 

27import yaml 

28 

29from lsst.afw.cameraGeom.testUtils import DetectorWrapper 

30import lsst.afw.geom as afwGeom 

31import lsst.afw.image as afwImage 

32from lsst.ap.association import (LoadDiaCatalogsTask, 

33 LoadDiaCatalogsConfig, 

34 make_dia_source_schema, 

35 make_dia_object_schema) 

36import lsst.daf.base as dafBase 

37from lsst.dax.apdb import Apdb, ApdbConfig 

38import lsst.geom as geom 

39import lsst.sphgeom as sphgeom 

40from lsst.utils import getPackageDir 

41import lsst.utils.tests 

42 

43 

44def _data_file_name(basename, module_name): 

45 """Return path name of a data file. 

46 

47 Parameters 

48 ---------- 

49 basename : `str` 

50 Name of the file to add to the path string. 

51 module_name : `str` 

52 Name of lsst stack package environment variable. 

53 

54 Returns 

55 ------- 

56 data_file_path : `str` 

57 Full path of the file to load from the "data" directory in a given 

58 repository. 

59 """ 

60 return os.path.join(getPackageDir(module_name), "data", basename) 

61 

62 

63def makeExposure(flipX=False, flipY=False): 

64 """Create an exposure and flip the x or y (or both) coordinates. 

65 

66 Returns bounding boxes that are right or left handed around the bounding 

67 polygon. 

68 

69 Parameters 

70 ---------- 

71 flipX : `bool` 

72 Flip the x coordinate in the WCS. 

73 flipY : `bool` 

74 Flip the y coordinate in the WCS. 

75 

76 Returns 

77 ------- 

78 exposure : `lsst.afw.image.Exposure` 

79 Exposure with a valid bounding box and wcs. 

80 """ 

81 metadata = dafBase.PropertySet() 

82 

83 metadata.set("SIMPLE", "T") 

84 metadata.set("BITPIX", -32) 

85 metadata.set("NAXIS", 2) 

86 metadata.set("NAXIS1", 1024) 

87 metadata.set("NAXIS2", 1153) 

88 metadata.set("RADECSYS", 'FK5') 

89 metadata.set("EQUINOX", 2000.) 

90 

91 metadata.setDouble("CRVAL1", 215.604025685476) 

92 metadata.setDouble("CRVAL2", 53.1595451514076) 

93 metadata.setDouble("CRPIX1", 1109.99981456774) 

94 metadata.setDouble("CRPIX2", 560.018167811613) 

95 metadata.set("CTYPE1", 'RA---SIN') 

96 metadata.set("CTYPE2", 'DEC--SIN') 

97 

98 xFlip = 1 

99 if flipX: 

100 xFlip = -1 

101 yFlip = 1 

102 if flipY: 

103 yFlip = -1 

104 metadata.setDouble("CD1_1", xFlip * 5.10808596133527E-05) 

105 metadata.setDouble("CD1_2", yFlip * 1.85579539217196E-07) 

106 metadata.setDouble("CD2_2", yFlip * -5.10281493481982E-05) 

107 metadata.setDouble("CD2_1", xFlip * -8.27440751733828E-07) 

108 

109 wcs = afwGeom.makeSkyWcs(metadata) 

110 exposure = afwImage.makeExposure( 

111 afwImage.makeMaskedImageFromArrays(np.ones((1024, 1153))), wcs) 

112 detector = DetectorWrapper(id=23, bbox=exposure.getBBox()).detector 

113 visit = afwImage.VisitInfo( 

114 exposureId=1234, 

115 exposureTime=200., 

116 date=dafBase.DateTime("2014-05-13T17:00:00.000000000", 

117 dafBase.DateTime.Timescale.TAI)) 

118 exposure.setDetector(detector) 

119 exposure.getInfo().setVisitInfo(visit) 

120 exposure.setFilterLabel(afwImage.FilterLabel(band='g')) 

121 

122 return exposure 

123 

124 

125def makeDiaObjects(nObjects, exposure, pixelator): 

126 """Make a test set of DiaObjects. 

127 

128 Parameters 

129 ---------- 

130 nObjects : `int` 

131 Number of objects to create. 

132 exposure : `lsst.afw.image.Exposure` 

133 Exposure to create objects over. 

134 pixelator : `lsst.sphgeom.HtmPixelization` 

135 Object to compute spatial indicies from. 

136 

137 Returns 

138 ------- 

139 diaObjects : `pandas.DataFrame` 

140 DiaObjects generated across the exposure. 

141 """ 

142 bbox = geom.Box2D(exposure.getBBox()) 

143 rand_x = np.random.uniform(bbox.getMinX(), bbox.getMaxX(), size=nObjects) 

144 rand_y = np.random.uniform(bbox.getMinY(), bbox.getMaxY(), size=nObjects) 

145 

146 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get( 

147 system=dafBase.DateTime.MJD) 

148 

149 wcs = exposure.getWcs() 

150 

151 data = [] 

152 for idx, (x, y) in enumerate(zip(rand_x, rand_y)): 

153 coord = wcs.pixelToSky(x, y) 

154 htmIdx = pixelator.index(coord.getVector()) 

155 newObject = {"ra": coord.getRa().asDegrees(), 

156 "decl": coord.getRa().asDegrees(), 

157 "radecTai": midPointTaiMJD, 

158 "diaObjectId": idx, 

159 "pixelId": htmIdx, 

160 "pmParallaxNdata": 0, 

161 "nearbyObj1": 0, 

162 "nearbyObj2": 0, 

163 "nearbyObj3": 0} 

164 for f in ["u", "g", "r", "i", "z", "y"]: 

165 newObject["%sPSFluxNdata" % f] = 0 

166 data.append(newObject) 

167 

168 return pd.DataFrame(data=data) 

169 

170 

171def makeDiaSources(nSources, diaObjectIds, exposure, pixelator): 

172 """Make a test set of DiaSources. 

173 

174 Parameters 

175 ---------- 

176 nSources : `int` 

177 Number of sources to create. 

178 diaObjectIds : `numpy.ndarray` 

179 Integer Ids of diaobjects to "associate" with the DiaSources. 

180 exposure : `lsst.afw.image.Exposure` 

181 Exposure to create sources over. 

182 pixelator : `lsst.sphgeom.HtmPixelization` 

183 Object to compute spatial indicies from. 

184 

185 Returns 

186 ------- 

187 diaSources : `pandas.DataFrame` 

188 DiaSources generated across the exposure. 

189 """ 

190 bbox = geom.Box2D(exposure.getBBox()) 

191 rand_x = np.random.uniform(bbox.getMinX(), bbox.getMaxX(), size=nSources) 

192 rand_y = np.random.uniform(bbox.getMinY(), bbox.getMaxY(), size=nSources) 

193 rand_ids = diaObjectIds[np.random.randint(len(diaObjectIds), size=nSources)] 

194 

195 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get( 

196 system=dafBase.DateTime.MJD) 

197 

198 wcs = exposure.getWcs() 

199 

200 data = [] 

201 for idx, (x, y, objId) in enumerate(zip(rand_x, rand_y, rand_ids)): 

202 coord = wcs.pixelToSky(x, y) 

203 htmIdx = pixelator.index(coord.getVector()) 

204 data.append({"ra": coord.getRa().asDegrees(), 

205 "decl": coord.getRa().asDegrees(), 

206 "diaObjectId": objId, 

207 "diaSourceId": idx, 

208 "pixelId": htmIdx, 

209 "midPointTai": midPointTaiMJD}) 

210 

211 return pd.DataFrame(data=data) 

212 

213 

214def makeDiaForcedSources(nForcedSources, diaObjectIds, exposure): 

215 """Make a test set of DiaForcedSources. 

216 

217 Parameters 

218 ---------- 

219 nForcedSources : `int` 

220 Number of sources to create. 

221 diaObjectIds : `numpy.ndarray` 

222 Integer Ids of diaobjects to "associate" with the DiaSources. 

223 exposure : `lsst.afw.image.Exposure` 

224 Exposure to create sources over. 

225 

226 Returns 

227 ------- 

228 diaForcedSources : `pandas.DataFrame` 

229 DiaForcedSources generated across the exposure. 

230 """ 

231 rand_ids = diaObjectIds[ 

232 np.random.randint(len(diaObjectIds), size=nForcedSources)] 

233 

234 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get( 

235 system=dafBase.DateTime.MJD) 

236 

237 data = [] 

238 for idx, objId in enumerate(rand_ids): 

239 data.append({"diaObjectId": objId, 

240 "diaForcedSourceId": idx, 

241 "ccdVisitId": idx, 

242 "midPointTai": midPointTaiMJD}) 

243 

244 return pd.DataFrame(data=data) 

245 

246 

247class TestLoadDiaCatalogs(unittest.TestCase): 

248 

249 def setUp(self): 

250 np.random.seed(1234) 

251 

252 self.db_file_fd, self.db_file = tempfile.mkstemp( 

253 dir=os.path.dirname(__file__)) 

254 

255 self.apdbConfig = ApdbConfig() 

256 self.apdbConfig.db_url = "sqlite:///" + self.db_file 

257 self.apdbConfig.isolation_level = "READ_UNCOMMITTED" 

258 self.apdbConfig.dia_object_index = "baseline" 

259 self.apdbConfig.dia_object_columns = [] 

260 self.apdbConfig.schema_file = _data_file_name( 

261 "apdb-schema.yaml", "dax_apdb") 

262 self.apdbConfig.column_map = _data_file_name( 

263 "apdb-ap-pipe-afw-map.yaml", "ap_association") 

264 self.apdbConfig.extra_schema_file = _data_file_name( 

265 "apdb-ap-pipe-schema-extra.yaml", "ap_association") 

266 

267 self.apdb = Apdb(config=self.apdbConfig, 

268 afw_schemas=dict(DiaObject=make_dia_object_schema(), 

269 DiaSource=make_dia_source_schema())) 

270 self.apdb.makeSchema() 

271 

272 # Expected HTM pixel ranges for max range=4 and level = 20. This 

273 # set of pixels should be same for the WCS created by default in 

274 # makeExposure and for one with a flipped y axis. 

275 self.ranges = np.sort(np.array([15154776375296, 15154779521024, 

276 15154788958208, 15154792103936])) 

277 

278 self.pixelator = sphgeom.HtmPixelization(20) 

279 self.exposure = makeExposure(False, False) 

280 

281 self.diaObjects = makeDiaObjects(20, self.exposure, self.pixelator) 

282 self.diaSources = makeDiaSources( 

283 100, 

284 self.diaObjects["diaObjectId"].to_numpy(), 

285 self.exposure, 

286 self.pixelator) 

287 self.diaForcedSources = makeDiaForcedSources( 

288 200, 

289 self.diaObjects["diaObjectId"].to_numpy(), 

290 self.exposure) 

291 

292 self.apdb.storeDiaSources(self.diaSources) 

293 self.apdb.storeDiaForcedSources(self.diaForcedSources) 

294 self.dateTime = \ 

295 self.exposure.getInfo().getVisitInfo().getDate().toPython() 

296 self.apdb.storeDiaObjects(self.diaObjects, 

297 self.dateTime) 

298 

299 # These columns are not in the DPDD, yet do appear in DiaSource.yaml. 

300 # We don't need to check them against the default APDB schema. 

301 self.ignoreColumns = ["filterName", "bboxSize", "isDipole"] 

302 

303 def tearDown(self): 

304 os.close(self.db_file_fd) 

305 os.remove(self.db_file) 

306 

307 def testRun(self): 

308 """Test the full run method for the loader. 

309 """ 

310 diaLoader = LoadDiaCatalogsTask() 

311 result = diaLoader.run(self.exposure, self.apdb) 

312 

313 self.assertEqual(len(result.diaObjects), len(self.diaObjects)) 

314 self.assertEqual(len(result.diaSources), len(self.diaSources)) 

315 self.assertEqual(len(result.diaForcedSources), 

316 len(self.diaForcedSources)) 

317 

318 def testLoadDiaObjects(self): 

319 """Test that the correct number of diaObjects are loaded. 

320 """ 

321 diaLoader = LoadDiaCatalogsTask() 

322 normPixels = diaLoader._getPixelRanges(self.exposure) 

323 diaObjects = diaLoader.loadDiaObjects(normPixels, 

324 self.apdb) 

325 self.assertEqual(len(diaObjects), len(self.diaObjects)) 

326 

327 def testLoadDiaForcedSources(self): 

328 """Test that the correct number of diaForcedSources are loaded. 

329 """ 

330 diaLoader = LoadDiaCatalogsTask() 

331 diaForcedSources = diaLoader.loadDiaForcedSources( 

332 self.diaObjects, 

333 self.dateTime, 

334 self.apdb) 

335 self.assertEqual(len(diaForcedSources), len(self.diaForcedSources)) 

336 

337 def testLoadDiaSourcesByPixelId(self): 

338 """Test that the correct number of diaSources are loaded. 

339 

340 Also check that they can be properly loaded both by location and 

341 ``diaObjectId``. 

342 """ 

343 self._testLoadDiaSources(True) 

344 

345 def testLoadDiaSourcesByDiaObjectId(self): 

346 """Test that the correct number of diaSources are loaded. 

347 

348 Also check that they can be properly loaded both by location and 

349 ``diaObjectId``. 

350 """ 

351 self._testLoadDiaSources(False) 

352 

353 def _testLoadDiaSources(self, loadByPixelId): 

354 """Test that DiaSources are loaded correctly. 

355 

356 Parameters 

357 ---------- 

358 loadByPixelId : `bool` 

359 Load DiaSources by ``pixelId`` if ``True`` and by ``diaObjectId`` 

360 if ``False``. 

361 """ 

362 diaConfig = LoadDiaCatalogsConfig() 

363 diaConfig.loadDiaSourcesByPixelId = loadByPixelId 

364 diaLoader = LoadDiaCatalogsTask(config=diaConfig) 

365 

366 normPixels = diaLoader._getPixelRanges(self.exposure) 

367 diaSources = diaLoader.loadDiaSources(self.diaObjects, 

368 normPixels, 

369 self.dateTime, 

370 self.apdb) 

371 self.assertEqual(len(diaSources), len(self.diaSources)) 

372 

373 def testGetPixelRanges(self): 

374 """Test the same pixels are returned for flips/ordering changes in 

375 the WCS. 

376 """ 

377 diaConfig = LoadDiaCatalogsConfig() 

378 diaConfig.pixelMargin = 300 # overriding to value tests were conditioned on 

379 diaConfig.htmMaxRanges = 4 

380 diaLoader = LoadDiaCatalogsTask(config=diaConfig) 

381 

382 # Make two exposures, one with a flipped y axis to get left vs. right 

383 # handed. 

384 exposure = makeExposure(False, False) 

385 exposureFlip = makeExposure(False, True) 

386 

387 normPixels = diaLoader._getPixelRanges(exposure) 

388 flipPixels = diaLoader._getPixelRanges(exposureFlip) 

389 

390 for normPix, flipPix, testPix in zip( 

391 np.sort(np.array(normPixels).flatten()), 

392 np.sort(np.array(flipPixels).flatten()), 

393 self.ranges): 

394 self.assertEqual(normPix, flipPix) 

395 self.assertEqual(normPix, testPix) 

396 self.assertEqual(flipPix, testPix) 

397 

398 def test_apdbSchema(self): 

399 """Test that the default DiaSource schema from dax_apdb agrees with the 

400 column names defined here in ap_association/data/DiaSource.yaml. 

401 """ 

402 functorFile = _data_file_name("DiaSource.yaml", "ap_association") 

403 apdbSchemaFile = _data_file_name("apdb-schema.yaml", "dax_apdb") 

404 with open(apdbSchemaFile) as yaml_stream: 

405 table_list = list(yaml.safe_load_all(yaml_stream)) 

406 for table in table_list: 

407 if table['table'] == 'DiaSource': 

408 apdbSchemaColumns = [column['name'] for column in table['columns']] 

409 break 

410 with open(functorFile) as yaml_stream: 

411 diaSourceFunctor = yaml.safe_load_all(yaml_stream) 

412 for functor in diaSourceFunctor: 

413 diaSourceColumns = [column for column in list(functor['funcs'].keys()) 

414 if column not in self.ignoreColumns] 

415 self.assertLess(set(diaSourceColumns), set(apdbSchemaColumns)) 

416 

417 

418class MemoryTester(lsst.utils.tests.MemoryTestCase): 

419 pass 

420 

421 

422def setup_module(module): 

423 lsst.utils.tests.init() 

424 

425 

426if __name__ == "__main__": 426 ↛ 427line 426 didn't jump to line 427, because the condition on line 426 was never true

427 lsst.utils.tests.init() 

428 unittest.main()