Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ap_association. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import numpy as np 

24import pandas as pd 

25import tempfile 

26import unittest 

27 

28from lsst.afw.cameraGeom.testUtils import DetectorWrapper 

29import lsst.afw.geom as afwGeom 

30import lsst.afw.image as afwImage 

31import lsst.afw.image.utils as afwImageUtils 

32from lsst.ap.association import (LoadDiaCatalogsTask, 

33 LoadDiaCatalogsConfig, 

34 make_dia_source_schema, 

35 make_dia_object_schema) 

36import lsst.daf.base as dafBase 

37from lsst.dax.apdb import Apdb, ApdbConfig 

38import lsst.geom as geom 

39import lsst.sphgeom as sphgeom 

40from lsst.utils import getPackageDir 

41import lsst.utils.tests 

42 

43 

44def _data_file_name(basename, module_name): 

45 """Return path name of a data file. 

46 

47 Parameters 

48 ---------- 

49 basename : `str` 

50 Name of the file to add to the path string. 

51 module_name : `str` 

52 Name of lsst stack package environment variable. 

53 

54 Returns 

55 ------- 

56 data_file_path : `str` 

57 Full path of the file to load from the "data" directory in a given 

58 repository. 

59 """ 

60 return os.path.join(getPackageDir(module_name), "data", basename) 

61 

62 

63def makeExposure(flipX=False, flipY=False): 

64 """Create an exposure and flip the x or y (or both) coordinates. 

65 

66 Returns bounding boxes that are right or left handed around the bounding 

67 polygon. 

68 

69 Parameters 

70 ---------- 

71 flipX : `bool` 

72 Flip the x coordinate in the WCS. 

73 flipY : `bool` 

74 Flip the y coordinate in the WCS. 

75 

76 Returns 

77 ------- 

78 exposure : `lsst.afw.image.Exposure` 

79 Exposure with a valid bounding box and wcs. 

80 """ 

81 metadata = dafBase.PropertySet() 

82 

83 metadata.set("SIMPLE", "T") 

84 metadata.set("BITPIX", -32) 

85 metadata.set("NAXIS", 2) 

86 metadata.set("NAXIS1", 1024) 

87 metadata.set("NAXIS2", 1153) 

88 metadata.set("RADECSYS", 'FK5') 

89 metadata.set("EQUINOX", 2000.) 

90 

91 metadata.setDouble("CRVAL1", 215.604025685476) 

92 metadata.setDouble("CRVAL2", 53.1595451514076) 

93 metadata.setDouble("CRPIX1", 1109.99981456774) 

94 metadata.setDouble("CRPIX2", 560.018167811613) 

95 metadata.set("CTYPE1", 'RA---SIN') 

96 metadata.set("CTYPE2", 'DEC--SIN') 

97 

98 xFlip = 1 

99 if flipX: 

100 xFlip = -1 

101 yFlip = 1 

102 if flipY: 

103 yFlip = -1 

104 metadata.setDouble("CD1_1", xFlip * 5.10808596133527E-05) 

105 metadata.setDouble("CD1_2", yFlip * 1.85579539217196E-07) 

106 metadata.setDouble("CD2_2", yFlip * -5.10281493481982E-05) 

107 metadata.setDouble("CD2_1", xFlip * -8.27440751733828E-07) 

108 

109 wcs = afwGeom.makeSkyWcs(metadata) 

110 exposure = afwImage.makeExposure( 

111 afwImage.makeMaskedImageFromArrays(np.ones((1024, 1153))), wcs) 

112 detector = DetectorWrapper(id=23, bbox=exposure.getBBox()).detector 

113 visit = afwImage.VisitInfo( 

114 exposureId=1234, 

115 exposureTime=200., 

116 date=dafBase.DateTime("2014-05-13T17:00:00.000000000", 

117 dafBase.DateTime.Timescale.TAI)) 

118 exposure.setDetector(detector) 

119 exposure.getInfo().setVisitInfo(visit) 

120 exposure.setFilter(afwImage.Filter('g')) 

121 

122 return exposure 

123 

124 

125def makeDiaObjects(nObjects, exposure, pixelator): 

126 """Make a test set of DiaObjects. 

127 

128 Parameters 

129 ---------- 

130 nObjects : `int` 

131 Number of objects to create. 

132 exposure : `lsst.afw.image.Exposure` 

133 Exposure to create objects over. 

134 pixelator : `lsst.sphgeom.HtmPixelization` 

135 Object to compute spatial indicies from. 

136 

137 Returns 

138 ------- 

139 diaObjects : `pandas.DataFrame` 

140 DiaObjects generated across the exposure. 

141 """ 

142 bbox = geom.Box2D(exposure.getBBox()) 

143 rand_x = np.random.uniform(bbox.getMinX(), bbox.getMaxX(), size=nObjects) 

144 rand_y = np.random.uniform(bbox.getMinY(), bbox.getMaxY(), size=nObjects) 

145 

146 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get( 

147 system=dafBase.DateTime.MJD) 

148 

149 wcs = exposure.getWcs() 

150 

151 data = [] 

152 for idx, (x, y) in enumerate(zip(rand_x, rand_y)): 

153 coord = wcs.pixelToSky(x, y) 

154 htmIdx = pixelator.index(coord.getVector()) 

155 newObject = {"ra": coord.getRa().asDegrees(), 

156 "decl": coord.getRa().asDegrees(), 

157 "radecTai": midPointTaiMJD, 

158 "diaObjectId": idx, 

159 "pixelId": htmIdx, 

160 "pmParallaxNdata": 0, 

161 "nearbyObj1": 0, 

162 "nearbyObj2": 0, 

163 "nearbyObj3": 0} 

164 for f in ["u", "g", "r", "i", "z", "y"]: 

165 newObject["%sPSFluxNdata" % f] = 0 

166 data.append(newObject) 

167 

168 return pd.DataFrame(data=data) 

169 

170 

171def makeDiaSources(nSources, diaObjectIds, exposure, pixelator): 

172 """Make a test set of DiaSources. 

173 

174 Parameters 

175 ---------- 

176 nSources : `int` 

177 Number of sources to create. 

178 diaObjectIds : `numpy.ndarray` 

179 Integer Ids of diaobjects to "associate" with the DiaSources. 

180 exposure : `lsst.afw.image.Exposure` 

181 Exposure to create sources over. 

182 pixelator : `lsst.sphgeom.HtmPixelization` 

183 Object to compute spatial indicies from. 

184 

185 Returns 

186 ------- 

187 diaSources : `pandas.DataFrame` 

188 DiaSources generated across the exposure. 

189 """ 

190 bbox = geom.Box2D(exposure.getBBox()) 

191 rand_x = np.random.uniform(bbox.getMinX(), bbox.getMaxX(), size=nSources) 

192 rand_y = np.random.uniform(bbox.getMinY(), bbox.getMaxY(), size=nSources) 

193 rand_ids = diaObjectIds[np.random.randint(len(diaObjectIds), size=nSources)] 

194 

195 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get( 

196 system=dafBase.DateTime.MJD) 

197 

198 wcs = exposure.getWcs() 

199 

200 data = [] 

201 for idx, (x, y, objId) in enumerate(zip(rand_x, rand_y, rand_ids)): 

202 coord = wcs.pixelToSky(x, y) 

203 htmIdx = pixelator.index(coord.getVector()) 

204 data.append({"ra": coord.getRa().asDegrees(), 

205 "decl": coord.getRa().asDegrees(), 

206 "diaObjectId": objId, 

207 "diaSourceId": idx, 

208 "pixelId": htmIdx, 

209 "midPointTai": midPointTaiMJD}) 

210 

211 return pd.DataFrame(data=data) 

212 

213 

214class TestLoadDiaCatalogs(unittest.TestCase): 

215 

216 def setUp(self): 

217 np.random.seed(1234) 

218 

219 # CFHT Filters from the camera mapper. 

220 self.filter_names = ["g"] 

221 afwImageUtils.resetFilters() 

222 afwImageUtils.defineFilter('g', lambdaEff=487, alias="g.MP9401") 

223 

224 self.tmp_file, self.db_file = tempfile.mkstemp( 

225 dir=os.path.dirname(__file__)) 

226 

227 self.apdbConfig = ApdbConfig() 

228 self.apdbConfig.db_url = "sqlite:///" + self.db_file 

229 self.apdbConfig.isolation_level = "READ_UNCOMMITTED" 

230 self.apdbConfig.dia_object_index = "baseline" 

231 self.apdbConfig.dia_object_columns = [] 

232 self.apdbConfig.schema_file = _data_file_name( 

233 "apdb-schema.yaml", "dax_apdb") 

234 self.apdbConfig.column_map = _data_file_name( 

235 "apdb-ap-pipe-afw-map.yaml", "ap_association") 

236 self.apdbConfig.extra_schema_file = _data_file_name( 

237 "apdb-ap-pipe-schema-extra.yaml", "ap_association") 

238 

239 self.apdb = Apdb(config=self.apdbConfig, 

240 afw_schemas=dict(DiaObject=make_dia_object_schema(), 

241 DiaSource=make_dia_source_schema())) 

242 self.apdb.makeSchema() 

243 

244 # Expected HTM pixel ranges for max range=4 and level = 20. This 

245 # set of pixels should be same for the WCS created by default in 

246 # makeExposure and for one with a flipped y axis. 

247 self.ranges = np.sort(np.array([15154776375296, 15154779521024, 

248 15154788958208, 15154792103936])) 

249 

250 self.pixelator = sphgeom.HtmPixelization(20) 

251 self.exposure = makeExposure(False, False) 

252 

253 self.diaObjects = makeDiaObjects(20, self.exposure, self.pixelator) 

254 self.diaSources = makeDiaSources( 

255 100, 

256 self.diaObjects["diaObjectId"].to_numpy(), 

257 self.exposure, 

258 self.pixelator) 

259 

260 self.apdb.storeDiaSources(self.diaSources) 

261 self.dateTime = \ 

262 self.exposure.getInfo().getVisitInfo().getDate().toPython() 

263 self.apdb.storeDiaObjects(self.diaObjects, 

264 self.dateTime) 

265 

266 def tearDown(self): 

267 del self.tmp_file 

268 os.remove(self.db_file) 

269 del self.db_file 

270 

271 def testRun(self): 

272 """Test the full run method for the loader. 

273 """ 

274 diaLoader = LoadDiaCatalogsTask() 

275 result = diaLoader.run(self.exposure, self.apdb) 

276 

277 self.assertEqual(len(result.diaObjects), len(self.diaObjects)) 

278 self.assertEqual(len(result.diaSources), len(self.diaSources)) 

279 

280 def testLoadDiaObjects(self): 

281 """Test that the correct number of diaObjects are loaded. 

282 """ 

283 diaLoader = LoadDiaCatalogsTask() 

284 normPixels = diaLoader._getPixelRanges(self.exposure) 

285 diaObjects = diaLoader.loadDiaObjects(normPixels, 

286 self.apdb) 

287 self.assertEqual(len(diaObjects), len(self.diaObjects)) 

288 

289 def testLoadDiaSourcesByPixelId(self): 

290 """Test that the correct number of diaSources are loaded. 

291 

292 Also check that they can be properly loaded both by location and 

293 ``diaObjectId``. 

294 """ 

295 self._testLoadDiaSources(True) 

296 

297 def testLoadDiaSourcesByDiaObjectId(self): 

298 """Test that the correct number of diaSources are loaded. 

299 

300 Also check that they can be properly loaded both by location and 

301 ``diaObjectId``. 

302 """ 

303 self._testLoadDiaSources(False) 

304 

305 def _testLoadDiaSources(self, loadByPixelId): 

306 """Test that DiaSources are loaded correctly. 

307 

308 Parameters 

309 ---------- 

310 loadByPixelId : `bool` 

311 Load DiaSources by ``pixelId`` if ``True`` and by ``diaObjectId`` 

312 if ``False``. 

313 """ 

314 diaConfig = LoadDiaCatalogsConfig() 

315 diaConfig.loadDiaSourcesByPixelId = loadByPixelId 

316 diaLoader = LoadDiaCatalogsTask(config=diaConfig) 

317 

318 normPixels = diaLoader._getPixelRanges(self.exposure) 

319 diaSources = diaLoader.loadDiaSources(self.diaObjects, 

320 normPixels, 

321 self.dateTime, 

322 self.apdb) 

323 self.assertEqual(len(diaSources), len(self.diaSources)) 

324 

325 def testGetPixelRanges(self): 

326 """Test the same pixels are returned for flips/ordering changes in 

327 the WCS. 

328 """ 

329 diaConfig = LoadDiaCatalogsConfig() 

330 diaConfig.htmMaxRanges = 4 

331 diaLoader = LoadDiaCatalogsTask(config=diaConfig) 

332 

333 # Make two exposures, one with a flipped y axis to get left vs. right 

334 # handed. 

335 exposure = makeExposure(False, False) 

336 exposureFlip = makeExposure(False, True) 

337 

338 normPixels = diaLoader._getPixelRanges(exposure) 

339 flipPixels = diaLoader._getPixelRanges(exposureFlip) 

340 

341 for normPix, flipPix, testPix in zip( 

342 np.sort(np.array(normPixels).flatten()), 

343 np.sort(np.array(flipPixels).flatten()), 

344 self.ranges): 

345 self.assertEqual(normPix, flipPix) 

346 self.assertEqual(normPix, testPix) 

347 self.assertEqual(flipPix, testPix) 

348 

349 

350class MemoryTester(lsst.utils.tests.MemoryTestCase): 

351 pass 

352 

353 

354def setup_module(module): 

355 lsst.utils.tests.init() 

356 

357 

358if __name__ == "__main__": 358 ↛ 359line 358 didn't jump to line 359, because the condition on line 358 was never true

359 lsst.utils.tests.init() 

360 unittest.main()