Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import collections 

23 

24import lsst.pex.config 

25import lsst.pex.exceptions 

26from lsst.log import Log 

27import lsst.pipe.base 

28import lsst.geom 

29import lsst.afw.geom 

30import lsst.afw.image 

31import lsst.afw.table 

32import lsst.sphgeom 

33 

34from lsst.pipe.base import PipelineTaskConnections 

35import lsst.pipe.base.connectionTypes as cT 

36from .forcedPhotImage import ForcedPhotImageTask, ForcedPhotImageConfig 

37 

38try: 

39 from lsst.meas.mosaic import applyMosaicResults 

40except ImportError: 

41 applyMosaicResults = None 

42 

43__all__ = ("PerTractCcdDataIdContainer", "ForcedPhotCcdConfig", "ForcedPhotCcdTask", "imageOverlapsTract") 

44 

45 

46class PerTractCcdDataIdContainer(lsst.pipe.base.DataIdContainer): 

47 """A data ID container which combines raw data IDs with a tract. 

48 

49 Notes 

50 ----- 

51 Required because we need to add "tract" to the raw data ID keys (defined as 

52 whatever we use for ``src``) when no tract is provided (so that the user is 

53 not required to know which tracts are spanned by the raw data ID). 

54 

55 This subclass of `~lsst.pipe.base.DataIdContainer` assumes that a calexp is 

56 being measured using the detection information, a set of reference 

57 catalogs, from the set of coadds which intersect with the calexp. It needs 

58 the calexp id (e.g. visit, raft, sensor), but is also uses the tract to 

59 decide what set of coadds to use. The references from the tract whose 

60 patches intersect with the calexp are used. 

61 """ 

62 

63 def makeDataRefList(self, namespace): 

64 """Make self.refList from self.idList 

65 """ 

66 if self.datasetType is None: 

67 raise RuntimeError("Must call setDatasetType first") 

68 log = Log.getLogger("meas.base.forcedPhotCcd.PerTractCcdDataIdContainer") 

69 skymap = None 

70 visitTract = collections.defaultdict(set) # Set of tracts for each visit 

71 visitRefs = collections.defaultdict(list) # List of data references for each visit 

72 for dataId in self.idList: 

73 if "tract" not in dataId: 

74 # Discover which tracts the data overlaps 

75 log.info("Reading WCS for components of dataId=%s to determine tracts", dict(dataId)) 

76 if skymap is None: 

77 skymap = namespace.butler.get(namespace.config.coaddName + "Coadd_skyMap") 

78 

79 for ref in namespace.butler.subset("calexp", dataId=dataId): 

80 if not ref.datasetExists("calexp"): 

81 continue 

82 

83 visit = ref.dataId["visit"] 

84 visitRefs[visit].append(ref) 

85 

86 md = ref.get("calexp_md", immediate=True) 

87 wcs = lsst.afw.geom.makeSkyWcs(md) 

88 box = lsst.geom.Box2D(lsst.afw.image.bboxFromMetadata(md)) 

89 # Going with just the nearest tract. Since we're throwing all tracts for the visit 

90 # together, this shouldn't be a problem unless the tracts are much smaller than a CCD. 

91 tract = skymap.findTract(wcs.pixelToSky(box.getCenter())) 

92 if imageOverlapsTract(tract, wcs, box): 

93 visitTract[visit].add(tract.getId()) 

94 else: 

95 self.refList.extend(ref for ref in namespace.butler.subset(self.datasetType, dataId=dataId)) 

96 

97 # Ensure all components of a visit are kept together by putting them all in the same set of tracts 

98 for visit, tractSet in visitTract.items(): 

99 for ref in visitRefs[visit]: 

100 for tract in tractSet: 

101 self.refList.append(namespace.butler.dataRef(datasetType=self.datasetType, 

102 dataId=ref.dataId, tract=tract)) 

103 if visitTract: 

104 tractCounter = collections.Counter() 

105 for tractSet in visitTract.values(): 

106 tractCounter.update(tractSet) 

107 log.info("Number of visits for each tract: %s", dict(tractCounter)) 

108 

109 

110def imageOverlapsTract(tract, imageWcs, imageBox): 

111 """Return whether the given bounding box overlaps the tract given a WCS. 

112 

113 Parameters 

114 ---------- 

115 tract : `lsst.skymap.TractInfo` 

116 TractInfo specifying a tract. 

117 imageWcs : `lsst.afw.geom.SkyWcs` 

118 World coordinate system for the image. 

119 imageBox : `lsst.geom.Box2I` 

120 Bounding box for the image. 

121 

122 Returns 

123 ------- 

124 overlap : `bool` 

125 `True` if the bounding box overlaps the tract; `False` otherwise. 

126 """ 

127 tractPoly = tract.getOuterSkyPolygon() 

128 

129 imagePixelCorners = lsst.geom.Box2D(imageBox).getCorners() 

130 try: 

131 imageSkyCorners = imageWcs.pixelToSky(imagePixelCorners) 

132 except lsst.pex.exceptions.LsstCppException as e: 

133 # Protecting ourselves from awful Wcs solutions in input images 

134 if (not isinstance(e.message, lsst.pex.exceptions.DomainErrorException) 

135 and not isinstance(e.message, lsst.pex.exceptions.RuntimeErrorException)): 

136 raise 

137 return False 

138 

139 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageSkyCorners]) 

140 return tractPoly.intersects(imagePoly) # "intersects" also covers "contains" or "is contained by" 

141 

142 

143class ForcedPhotCcdConnections(PipelineTaskConnections, 

144 dimensions=("instrument", "visit", "detector", "skymap", "tract"), 

145 defaultTemplates={"inputCoaddName": "deep", 

146 "inputName": "calexp"}): 

147 inputSchema = cT.InitInput( 

148 doc="Schema for the input measurement catalogs.", 

149 name="{inputCoaddName}Coadd_ref_schema", 

150 storageClass="SourceCatalog", 

151 ) 

152 outputSchema = cT.InitOutput( 

153 doc="Schema for the output forced measurement catalogs.", 

154 name="forced_src_schema", 

155 storageClass="SourceCatalog", 

156 ) 

157 exposure = cT.Input( 

158 doc="Input exposure to perform photometry on.", 

159 name="{inputName}", 

160 storageClass="ExposureF", 

161 dimensions=["instrument", "visit", "detector"], 

162 ) 

163 refCat = cT.Input( 

164 doc="Catalog of shapes and positions at which to force photometry.", 

165 name="{inputCoaddName}Coadd_ref", 

166 storageClass="SourceCatalog", 

167 dimensions=["skymap", "tract", "patch"], 

168 ) 

169 refWcs = cT.Input( 

170 doc="Reference world coordinate system.", 

171 name="{inputCoaddName}Coadd.wcs", 

172 storageClass="Wcs", 

173 dimensions=["band", "skymap", "tract", "patch"], 

174 ) 

175 measCat = cT.Output( 

176 doc="Output forced photometry catalog.", 

177 name="forced_src", 

178 storageClass="SourceCatalog", 

179 dimensions=["instrument", "visit", "detector", "skymap", "tract"], 

180 ) 

181 

182 

183class ForcedPhotCcdConfig(ForcedPhotImageConfig, pipelineConnections=ForcedPhotCcdConnections): 

184 doApplyUberCal = lsst.pex.config.Field( 

185 dtype=bool, 

186 doc="Apply meas_mosaic ubercal results to input calexps?", 

187 default=False, 

188 deprecated="Deprecated by DM-23352; use doApplyExternalPhotoCalib and doApplyExternalSkyWcs instead", 

189 ) 

190 doApplyExternalPhotoCalib = lsst.pex.config.Field( 

191 dtype=bool, 

192 default=False, 

193 doc=("Whether to apply external photometric calibration via an " 

194 "`lsst.afw.image.PhotoCalib` object. Uses the " 

195 "``externalPhotoCalibName`` field to determine which calibration " 

196 "to load."), 

197 ) 

198 doApplyExternalSkyWcs = lsst.pex.config.Field( 

199 dtype=bool, 

200 default=False, 

201 doc=("Whether to apply external astrometric calibration via an " 

202 "`lsst.afw.geom.SkyWcs` object. Uses ``externalSkyWcsName`` " 

203 "field to determine which calibration to load."), 

204 ) 

205 doApplySkyCorr = lsst.pex.config.Field( 

206 dtype=bool, 

207 default=False, 

208 doc="Apply sky correction?", 

209 ) 

210 includePhotoCalibVar = lsst.pex.config.Field( 

211 dtype=bool, 

212 default=False, 

213 doc="Add photometric calibration variance to warp variance plane?", 

214 ) 

215 externalPhotoCalibName = lsst.pex.config.ChoiceField( 

216 dtype=str, 

217 doc=("Type of external PhotoCalib if ``doApplyExternalPhotoCalib`` is True. " 

218 "Unused for Gen3 middleware."), 

219 default="jointcal", 

220 allowed={ 

221 "jointcal": "Use jointcal_photoCalib", 

222 "fgcm": "Use fgcm_photoCalib", 

223 "fgcm_tract": "Use fgcm_tract_photoCalib" 

224 }, 

225 ) 

226 externalSkyWcsName = lsst.pex.config.ChoiceField( 

227 dtype=str, 

228 doc="Type of external SkyWcs if ``doApplyExternalSkyWcs`` is True. Unused for Gen3 middleware.", 

229 default="jointcal", 

230 allowed={ 

231 "jointcal": "Use jointcal_wcs" 

232 }, 

233 ) 

234 

235 

236class ForcedPhotCcdTask(ForcedPhotImageTask): 

237 """A command-line driver for performing forced measurement on CCD images. 

238 

239 Notes 

240 ----- 

241 This task is a subclass of 

242 :lsst-task:`lsst.meas.base.forcedPhotImage.ForcedPhotImageTask` which is 

243 specifically for doing forced measurement on a single CCD exposure, using 

244 as a reference catalog the detections which were made on overlapping 

245 coadds. 

246 

247 The `run` method (inherited from `ForcedPhotImageTask`) takes a 

248 `~lsst.daf.persistence.ButlerDataRef` argument that corresponds to a single 

249 CCD. This should contain the data ID keys that correspond to the 

250 ``forced_src`` dataset (the output dataset for this task), which are 

251 typically all those used to specify the ``calexp`` dataset (``visit``, 

252 ``raft``, ``sensor`` for LSST data) as well as a coadd tract. The tract is 

253 used to look up the appropriate coadd measurement catalogs to use as 

254 references (e.g. ``deepCoadd_src``; see 

255 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask` for more 

256 information). While the tract must be given as part of the dataRef, the 

257 patches are determined automatically from the bounding box and WCS of the 

258 calexp to be measured, and the filter used to fetch references is set via 

259 the ``filter`` option in the configuration of 

260 :lsst-task:`lsst.meas.base.references.BaseReferencesTask`). 

261 

262 In addition to the `run` method, `ForcedPhotCcdTask` overrides several 

263 methods of `ForcedPhotImageTask` to specialize it for single-CCD 

264 processing, including `~ForcedPhotImageTask.makeIdFactory`, 

265 `~ForcedPhotImageTask.fetchReferences`, and 

266 `~ForcedPhotImageTask.getExposure`. None of these should be called 

267 directly by the user, though it may be useful to override them further in 

268 subclasses. 

269 """ 

270 

271 ConfigClass = ForcedPhotCcdConfig 

272 RunnerClass = lsst.pipe.base.ButlerInitializedTaskRunner 

273 _DefaultName = "forcedPhotCcd" 

274 dataPrefix = "" 

275 

276 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

277 inputs = butlerQC.get(inputRefs) 

278 inputs['refCat'] = self.filterReferences(inputs['exposure'], inputs['refCat'], inputs['refWcs']) 

279 inputs['measCat'] = self.generateMeasCat(inputRefs.exposure.dataId, 

280 inputs['exposure'], 

281 inputs['refCat'], inputs['refWcs'], 

282 "visit_detector") 

283 # TODO: apply external calibrations (DM-17062) 

284 outputs = self.run(**inputs) 

285 butlerQC.put(outputs, outputRefs) 

286 

287 def filterReferences(self, exposure, refCat, refWcs): 

288 """Filter reference catalog so that all sources are within the 

289 boundaries of the exposure. 

290 

291 Parameters 

292 ---------- 

293 exposure : `lsst.afw.image.exposure.Exposure` 

294 Exposure to generate the catalog for. 

295 refCat : `lsst.afw.table.SourceCatalog` 

296 Catalog of shapes and positions at which to force photometry. 

297 refWcs : `lsst.afw.image.SkyWcs` 

298 Reference world coordinate system. 

299 

300 Returns 

301 ------- 

302 refSources : `lsst.afw.table.SourceCatalog` 

303 Filtered catalog of forced sources to measure. 

304 

305 Notes 

306 ----- 

307 Filtering the reference catalog is currently handled by Gen2 

308 specific methods. To function for Gen3, this method copies 

309 code segments to do the filtering and transformation. The 

310 majority of this code is based on the methods of 

311 lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader 

312 

313 """ 

314 

315 # Step 1: Determine bounds of the exposure photometry will 

316 # be performed on. 

317 expWcs = exposure.getWcs() 

318 expRegion = exposure.getBBox(lsst.afw.image.PARENT) 

319 expBBox = lsst.geom.Box2D(expRegion) 

320 expBoxCorners = expBBox.getCorners() 

321 expSkyCorners = [expWcs.pixelToSky(corner).getVector() for 

322 corner in expBoxCorners] 

323 expPolygon = lsst.sphgeom.ConvexPolygon(expSkyCorners) 

324 

325 # Step 2: Filter out reference catalog sources that are 

326 # not contained within the exposure boundaries. 

327 sources = type(refCat)(refCat.table) 

328 for record in refCat: 

329 if expPolygon.contains(record.getCoord().getVector()): 

330 sources.append(record) 

331 refCatIdDict = {ref.getId(): ref.getParent() for ref in sources} 

332 

333 # Step 3: Cull sources that do not have their parent 

334 # source in the filtered catalog. Save two copies of each 

335 # source. 

336 refSources = type(refCat)(refCat.table) 

337 for record in refCat: 

338 if expPolygon.contains(record.getCoord().getVector()): 

339 recordId = record.getId() 

340 topId = recordId 

341 while (topId > 0): 

342 if topId in refCatIdDict: 

343 topId = refCatIdDict[topId] 

344 else: 

345 break 

346 if topId == 0: 

347 refSources.append(record) 

348 

349 # Step 4: Transform source footprints from the reference 

350 # coordinates to the exposure coordinates. 

351 for refRecord in refSources: 

352 refRecord.setFootprint(refRecord.getFootprint().transform(refWcs, 

353 expWcs, expRegion)) 

354 # Step 5: Replace reference catalog with filtered source list. 

355 return refSources 

356 

357 def makeIdFactory(self, dataRef): 

358 """Create an object that generates globally unique source IDs. 

359 

360 Source IDs are created based on a per-CCD ID and the ID of the CCD 

361 itself. 

362 

363 Parameters 

364 ---------- 

365 dataRef : `lsst.daf.persistence.ButlerDataRef` 

366 Butler data reference. The ``ccdExposureId_bits`` and 

367 ``ccdExposureId`` datasets are accessed. The data ID must have the 

368 keys that correspond to ``ccdExposureId``, which are generally the 

369 same as those that correspond to ``calexp`` (``visit``, ``raft``, 

370 ``sensor`` for LSST data). 

371 """ 

372 expBits = dataRef.get("ccdExposureId_bits") 

373 expId = int(dataRef.get("ccdExposureId")) 

374 return lsst.afw.table.IdFactory.makeSource(expId, 64 - expBits) 

375 

376 def getExposureId(self, dataRef): 

377 return int(dataRef.get("ccdExposureId", immediate=True)) 

378 

379 def fetchReferences(self, dataRef, exposure): 

380 """Get sources that overlap the exposure. 

381 

382 Parameters 

383 ---------- 

384 dataRef : `lsst.daf.persistence.ButlerDataRef` 

385 Butler data reference corresponding to the image to be measured; 

386 should have ``tract``, ``patch``, and ``filter`` keys. 

387 exposure : `lsst.afw.image.Exposure` 

388 The image to be measured (used only to obtain a WCS and bounding 

389 box). 

390 

391 Returns 

392 ------- 

393 referencs : `lsst.afw.table.SourceCatalog` 

394 Catalog of sources that overlap the exposure 

395 

396 Notes 

397 ----- 

398 The returned catalog is sorted by ID and guarantees that all included 

399 children have their parent included and that all Footprints are valid. 

400 

401 All work is delegated to the references subtask; see 

402 :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask` 

403 for information about the default behavior. 

404 """ 

405 references = lsst.afw.table.SourceCatalog(self.references.schema) 

406 badParents = set() 

407 unfiltered = self.references.fetchInBox(dataRef, exposure.getBBox(), exposure.getWcs()) 

408 for record in unfiltered: 

409 if record.getFootprint() is None or record.getFootprint().getArea() == 0: 

410 if record.getParent() != 0: 

411 self.log.warn("Skipping reference %s (child of %s) with bad Footprint", 

412 record.getId(), record.getParent()) 

413 else: 

414 self.log.warn("Skipping reference parent %s with bad Footprint", record.getId()) 

415 badParents.add(record.getId()) 

416 elif record.getParent() not in badParents: 

417 references.append(record) 

418 # catalog must be sorted by parent ID for lsst.afw.table.getChildren to work 

419 references.sort(lsst.afw.table.SourceTable.getParentKey()) 

420 return references 

421 

422 def getExposure(self, dataRef): 

423 """Read input exposure for measurement. 

424 

425 Parameters 

426 ---------- 

427 dataRef : `lsst.daf.persistence.ButlerDataRef` 

428 Butler data reference. 

429 """ 

430 exposure = ForcedPhotImageTask.getExposure(self, dataRef) 

431 

432 if self.config.doApplyExternalPhotoCalib: 

433 source = f"{self.config.externalPhotoCalibName}_photoCalib" 

434 self.log.info("Applying external photoCalib from %s", source) 

435 photoCalib = dataRef.get(source) 

436 exposure.setPhotoCalib(photoCalib) # No need for calibrateImage; having the photoCalib suffices 

437 

438 if self.config.doApplyExternalSkyWcs: 

439 source = f"{self.config.externalSkyWcsName}_wcs" 

440 self.log.info("Applying external skyWcs from %s", source) 

441 skyWcs = dataRef.get(source) 

442 exposure.setWcs(skyWcs) 

443 

444 if self.config.doApplySkyCorr: 

445 self.log.info("Apply sky correction") 

446 skyCorr = dataRef.get("skyCorr") 

447 exposure.maskedImage -= skyCorr.getImage() 

448 

449 return exposure 

450 

451 def _getConfigName(self): 

452 # Documented in superclass. 

453 return self.dataPrefix + "forcedPhotCcd_config" 

454 

455 def _getMetadataName(self): 

456 # Documented in superclass 

457 return self.dataPrefix + "forcedPhotCcd_metadata" 

458 

459 @classmethod 

460 def _makeArgumentParser(cls): 

461 parser = lsst.pipe.base.ArgumentParser(name=cls._DefaultName) 

462 parser.add_id_argument("--id", "forced_src", help="data ID with raw CCD keys [+ tract optionally], " 

463 "e.g. --id visit=12345 ccd=1,2 [tract=0]", 

464 ContainerClass=PerTractCcdDataIdContainer) 

465 return parser