23 """Build star observations for input to FGCM.
25 This task finds all the visits and calexps in a repository (or a subset
26 based on command line parameters) and extract all the potential calibration
27 stars for input into fgcm. This task additionally uses fgcm to match
28 star observations into unique stars, and performs as much cleaning of
29 the input catalog as possible.
36 import lsst.pex.config
as pexConfig
37 import lsst.pipe.base
as pipeBase
38 import lsst.afw.table
as afwTable
40 from .fgcmBuildStarsBase
import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
41 from .utilities
import computeApproxPixelAreaFields
43 __all__ = [
'FgcmBuildStarsConfig',
'FgcmBuildStarsTask']
47 """Config for FgcmBuildStarsTask"""
49 referenceCCD = pexConfig.Field(
50 doc=
"Reference CCD for scanning visits",
54 checkAllCcds = pexConfig.Field(
55 doc=(
"Check repo for all CCDs for each visit specified. To be used when the "
56 "full set of ids (visit/ccd) are not specified on the command line. For "
57 "Gen2, specifying one ccd and setting checkAllCcds=True is significantly "
58 "faster than the alternatives."),
72 fluxFlagName = self.
instFluxField[0: -len(
'instFlux')] +
'flag'
73 sourceSelector.flags.bad = [
'base_PixelFlags_flag_edge',
74 'base_PixelFlags_flag_interpolatedCenter',
75 'base_PixelFlags_flag_saturatedCenter',
76 'base_PixelFlags_flag_crCenter',
77 'base_PixelFlags_flag_bad',
78 'base_PixelFlags_flag_interpolated',
79 'base_PixelFlags_flag_saturated',
85 sourceSelector.flags.bad.append(localBackgroundFlagName)
88 sourceSelector.signalToNoise.errField = self.
instFluxField +
'Err'
93 Build stars for the FGCM global calibration, using src catalogs.
95 ConfigClass = FgcmBuildStarsConfig
96 RunnerClass = FgcmBuildStarsRunner
97 _DefaultName =
"fgcmBuildStars"
100 def _makeArgumentParser(cls):
101 """Create an argument parser"""
103 parser.add_id_argument(
"--id",
"src", help=
"Data ID, e.g. --id visit=6789")
108 self.log.info(
"Grouping dataRefs by %s" % (self.config.visitDataRefName))
110 camera = butler.get(
'camera')
113 for detector
in camera:
114 ccdIds.append(detector.getId())
123 for dataRef
in dataRefs:
124 visit = dataRef.dataId[self.config.visitDataRefName]
126 if not dataRef.datasetExists(datasetType=
'src'):
129 if self.config.checkAllCcds:
130 if visit
in groupedDataRefs:
133 dataId = dataRef.dataId.copy()
136 dataId[self.config.ccdDataRefName] = ccdId
137 if butler.datasetExists(
'src', dataId=dataId):
138 goodDataRef = butler.dataRef(
'src', dataId=dataId)
139 if visit
in groupedDataRefs:
140 if (goodDataRef.dataId[self.config.ccdDataRefName]
not in
141 [d.dataId[self.config.ccdDataRefName]
for d
in groupedDataRefs[visit]]):
142 groupedDataRefs[visit].append(goodDataRef)
146 groupedDataRefs[visit] = [goodDataRef]
150 if visit
in groupedDataRefs:
151 if (dataRef.dataId[self.config.ccdDataRefName]
not in
152 [d.dataId[self.config.ccdDataRefName]
for d
in groupedDataRefs[visit]]):
153 groupedDataRefs[visit].append(dataRef)
157 groupedDataRefs[visit] = [dataRef]
159 if (nVisits % 100) == 0
and nVisits > 0:
160 self.log.info(
"Found %d unique %ss..." % (nVisits,
161 self.config.visitDataRefName))
163 self.log.info(
"Found %d unique %ss total." % (nVisits,
164 self.config.visitDataRefName))
167 def ccdSorter(dataRef):
168 ccdId = dataRef.dataId[self.config.ccdDataRefName]
169 if ccdId == self.config.referenceCCD:
175 if not self.config.checkAllCcds:
176 for visit
in groupedDataRefs:
177 groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter)
179 return groupedDataRefs
182 calibFluxApertureRadius=None,
183 visitCatDataRef=None,
186 startTime = time.time()
192 if (visitCatDataRef
is not None and starObsDataRef
is None or
193 visitCatDataRef
is None and starObsDataRef
is not None):
194 self.log.warn(
"Only one of visitCatDataRef and starObsDataRef are set, so "
195 "no checkpoint files will be persisted.")
197 if self.config.doSubtractLocalBackground
and calibFluxApertureRadius
is None:
198 raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
201 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
202 sourceSchema = dataRef.get(
'src_schema', immediate=
True).schema
205 camera = dataRef.get(
'camera')
207 for ccdIndex, detector
in enumerate(camera):
208 ccdMapping[detector.getId()] = ccdIndex
217 outputSchema = sourceMapper.getOutputSchema()
219 if inStarObsCat
is not None:
220 fullCatalog = inStarObsCat
221 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
222 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
223 if not comp1
or not comp2:
224 raise RuntimeError(
"Existing fgcmStarObservations file found with mismatched schema.")
226 fullCatalog = afwTable.BaseCatalog(outputSchema)
230 instFluxKey = sourceSchema[self.config.instFluxField].asKey()
231 instFluxErrKey = sourceSchema[self.config.instFluxField +
'Err'].asKey()
232 visitKey = outputSchema[
'visit'].asKey()
233 ccdKey = outputSchema[
'ccd'].asKey()
234 instMagKey = outputSchema[
'instMag'].asKey()
235 instMagErrKey = outputSchema[
'instMagErr'].asKey()
236 deltaMagBkgKey = outputSchema[
'deltaMagBkg'].asKey()
239 if self.config.doSubtractLocalBackground:
240 localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey()
241 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
243 aperOutputSchema = aperMapper.getOutputSchema()
245 instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey()
246 instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField +
'Err'].asKey()
247 instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey()
248 instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField +
'Err'].asKey()
249 instMagInKey = aperOutputSchema[
'instMag_aper_inner'].asKey()
250 instMagErrInKey = aperOutputSchema[
'instMagErr_aper_inner'].asKey()
251 instMagOutKey = aperOutputSchema[
'instMag_aper_outer'].asKey()
252 instMagErrOutKey = aperOutputSchema[
'instMagErr_aper_outer'].asKey()
257 for ctr, visit
in enumerate(visitCat):
258 if visit[
'sources_read']:
261 expTime = visit[
'exptime']
266 aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
268 for dataRef
in groupedDataRefs[visit[
'visit']]:
270 ccdId = dataRef.dataId[self.config.ccdDataRefName]
272 sources = dataRef.get(datasetType=
'src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
273 goodSrc = self.sourceSelector.selectSources(sources)
275 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
276 tempCat.reserve(goodSrc.selected.sum())
277 tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
278 tempCat[visitKey][:] = visit[
'visit']
279 tempCat[ccdKey][:] = ccdId
282 scaledInstFlux = (sources[instFluxKey][goodSrc.selected] *
283 visit[
'scaling'][ccdMapping[ccdId]])
284 tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
287 if self.config.doSubtractLocalBackground:
299 localBackground = localBackgroundArea*sources[localBackgroundFluxKey]
303 tempCat[deltaMagBkgKey][:] = (-2.5*np.log10(sources[instFluxKey][goodSrc.selected] -
304 localBackground[goodSrc.selected]) -
305 -2.5*np.log10(sources[instFluxKey][goodSrc.selected]))
307 tempCat[deltaMagBkgKey][:] = 0.0
312 tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected] /
313 sources[instFluxKey][goodSrc.selected])
316 tempCat[
'jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat[
'x'],
320 if self.config.doApplyWcsJacobian:
321 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat[
'jacobian'][:])
323 fullCatalog.extend(tempCat)
327 tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
328 tempAperCat.reserve(goodSrc.selected.sum())
329 tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper)
331 with np.warnings.catch_warnings():
334 np.warnings.simplefilter(
"ignore")
336 tempAperCat[instMagInKey][:] = -2.5*np.log10(
337 sources[instFluxAperInKey][goodSrc.selected])
338 tempAperCat[instMagErrInKey][:] = k*(
339 sources[instFluxErrAperInKey][goodSrc.selected] /
340 sources[instFluxAperInKey][goodSrc.selected])
341 tempAperCat[instMagOutKey][:] = -2.5*np.log10(
342 sources[instFluxAperOutKey][goodSrc.selected])
343 tempAperCat[instMagErrOutKey][:] = k*(
344 sources[instFluxErrAperOutKey][goodSrc.selected] /
345 sources[instFluxAperOutKey][goodSrc.selected])
347 aperVisitCatalog.extend(tempAperCat)
349 nStarInVisit += len(tempCat)
352 if not aperVisitCatalog.isContiguous():
353 aperVisitCatalog = aperVisitCatalog.copy(deep=
True)
355 instMagIn = aperVisitCatalog[instMagInKey]
356 instMagErrIn = aperVisitCatalog[instMagErrInKey]
357 instMagOut = aperVisitCatalog[instMagOutKey]
358 instMagErrOut = aperVisitCatalog[instMagErrOutKey]
360 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
361 np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
363 visit[
'deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
364 visit[
'sources_read'] =
True
366 self.log.info(
" Found %d good stars in visit %d (deltaAper = %.3f)" %
367 (nStarInVisit, visit[
'visit'], visit[
'deltaAper']))
369 if ((ctr % self.config.nVisitsPerCheckpoint) == 0
and
370 starObsDataRef
is not None and visitCatDataRef
is not None):
373 starObsDataRef.put(fullCatalog)
374 visitCatDataRef.put(visitCat)
376 self.log.info(
"Found all good star observations in %.2f s" %
377 (time.time() - startTime))
381 def _makeAperMapper(self, sourceSchema):
383 Make a schema mapper for fgcm aperture measurements
387 sourceSchema: `afwTable.Schema`
388 Default source schema from the butler
392 aperMapper: `afwTable.schemaMapper`
393 Mapper to the FGCM aperture schema
396 aperMapper = afwTable.SchemaMapper(sourceSchema)
397 aperMapper.addMapping(sourceSchema[
'coord_ra'].asKey(),
'ra')
398 aperMapper.addMapping(sourceSchema[
'coord_dec'].asKey(),
'dec')
399 aperMapper.editOutputSchema().addField(
'instMag_aper_inner', type=np.float64,
400 doc=
"Magnitude at inner aperture")
401 aperMapper.editOutputSchema().addField(
'instMagErr_aper_inner', type=np.float64,
402 doc=
"Magnitude error at inner aperture")
403 aperMapper.editOutputSchema().addField(
'instMag_aper_outer', type=np.float64,
404 doc=
"Magnitude at outer aperture")
405 aperMapper.editOutputSchema().addField(
'instMagErr_aper_outer', type=np.float64,
406 doc=
"Magnitude error at outer aperture")