23 """Build star observations for input to FGCM.
25 This task finds all the visits and calexps in a repository (or a subset
26 based on command line parameters) and extract all the potential calibration
27 stars for input into fgcm. This task additionally uses fgcm to match
28 star observations into unique stars, and performs as much cleaning of
29 the input catalog as possible.
36 import lsst.pex.config
as pexConfig
37 import lsst.pipe.base
as pipeBase
38 import lsst.afw.table
as afwTable
40 from .fgcmBuildStarsBase
import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
41 from .utilities
import computeApproxPixelAreaFields
43 __all__ = [
'FgcmBuildStarsConfig',
'FgcmBuildStarsTask']
47 """Config for FgcmBuildStarsTask"""
49 referenceCCD = pexConfig.Field(
50 doc=
"Reference CCD for scanning visits",
54 checkAllCcds = pexConfig.Field(
55 doc=(
"Check repo for all CCDs for each visit specified. To be used when the "
56 "full set of ids (visit/ccd) are not specified on the command line. For "
57 "Gen2, specifying one ccd and setting checkAllCcds=True is significantly "
58 "faster than the alternatives."),
72 fluxFlagName = self.
instFluxField[0: -len(
'instFlux')] +
'flag'
73 sourceSelector.flags.bad = [
'base_PixelFlags_flag_edge',
74 'base_PixelFlags_flag_interpolatedCenter',
75 'base_PixelFlags_flag_saturatedCenter',
76 'base_PixelFlags_flag_crCenter',
77 'base_PixelFlags_flag_bad',
78 'base_PixelFlags_flag_interpolated',
79 'base_PixelFlags_flag_saturated',
85 sourceSelector.flags.bad.append(localBackgroundFlagName)
88 sourceSelector.signalToNoise.errField = self.
instFluxField +
'Err'
93 Build stars for the FGCM global calibration, using src catalogs.
95 ConfigClass = FgcmBuildStarsConfig
96 RunnerClass = FgcmBuildStarsRunner
97 _DefaultName =
"fgcmBuildStars"
100 def _makeArgumentParser(cls):
101 """Create an argument parser"""
103 parser.add_id_argument(
"--id",
"src", help=
"Data ID, e.g. --id visit=6789")
108 self.log.info(
"Grouping dataRefs by %s" % (self.config.visitDataRefName))
110 camera = butler.get(
'camera')
113 for detector
in camera:
114 ccdIds.append(detector.getId())
123 for dataRef
in dataRefs:
124 visit = dataRef.dataId[self.config.visitDataRefName]
126 if not dataRef.datasetExists(datasetType=
'src'):
129 if self.config.checkAllCcds:
130 if visit
in groupedDataRefs:
133 dataId = dataRef.dataId.copy()
136 dataId[self.config.ccdDataRefName] = ccdId
137 if butler.datasetExists(
'src', dataId=dataId):
138 goodDataRef = butler.dataRef(
'src', dataId=dataId)
139 if visit
in groupedDataRefs:
140 if (goodDataRef.dataId[self.config.ccdDataRefName]
not in
141 [d.dataId[self.config.ccdDataRefName]
for d
in groupedDataRefs[visit]]):
142 groupedDataRefs[visit].append(goodDataRef)
146 groupedDataRefs[visit] = [goodDataRef]
150 if visit
in groupedDataRefs:
151 if (dataRef.dataId[self.config.ccdDataRefName]
not in
152 [d.dataId[self.config.ccdDataRefName]
for d
in groupedDataRefs[visit]]):
153 groupedDataRefs[visit].append(dataRef)
157 groupedDataRefs[visit] = [dataRef]
159 if (nVisits % 100) == 0
and nVisits > 0:
160 self.log.info(
"Found %d unique %ss..." % (nVisits,
161 self.config.visitDataRefName))
163 self.log.info(
"Found %d unique %ss total." % (nVisits,
164 self.config.visitDataRefName))
167 def ccdSorter(dataRef):
168 ccdId = dataRef.dataId[self.config.ccdDataRefName]
169 if ccdId == self.config.referenceCCD:
175 if not self.config.checkAllCcds:
176 for visit
in groupedDataRefs:
177 groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter)
179 return groupedDataRefs
182 calibFluxApertureRadius=None,
183 visitCatDataRef=None,
186 startTime = time.time()
192 if (visitCatDataRef
is not None and starObsDataRef
is None or
193 visitCatDataRef
is None and starObsDataRef
is not None):
194 self.log.warn(
"Only one of visitCatDataRef and starObsDataRef are set, so "
195 "no checkpoint files will be persisted.")
197 if self.config.doSubtractLocalBackground
and calibFluxApertureRadius
is None:
198 raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
201 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
202 sourceSchema = dataRef.get(
'src_schema', immediate=
True).schema
205 camera = dataRef.get(
'camera')
207 for ccdIndex, detector
in enumerate(camera):
208 ccdMapping[detector.getId()] = ccdIndex
217 outputSchema = sourceMapper.getOutputSchema()
219 if inStarObsCat
is not None:
220 fullCatalog = inStarObsCat
221 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
222 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
223 if not comp1
or not comp2:
224 raise RuntimeError(
"Existing fgcmStarObservations file found with mismatched schema.")
226 fullCatalog = afwTable.BaseCatalog(outputSchema)
230 instFluxKey = sourceSchema[self.config.instFluxField].asKey()
231 instFluxErrKey = sourceSchema[self.config.instFluxField +
'Err'].asKey()
232 visitKey = outputSchema[
'visit'].asKey()
233 ccdKey = outputSchema[
'ccd'].asKey()
234 instMagKey = outputSchema[
'instMag'].asKey()
235 instMagErrKey = outputSchema[
'instMagErr'].asKey()
238 if self.config.doSubtractLocalBackground:
239 localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey()
240 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
242 localBackground = 0.0
244 aperOutputSchema = aperMapper.getOutputSchema()
246 instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey()
247 instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField +
'Err'].asKey()
248 instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey()
249 instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField +
'Err'].asKey()
250 instMagInKey = aperOutputSchema[
'instMag_aper_inner'].asKey()
251 instMagErrInKey = aperOutputSchema[
'instMagErr_aper_inner'].asKey()
252 instMagOutKey = aperOutputSchema[
'instMag_aper_outer'].asKey()
253 instMagErrOutKey = aperOutputSchema[
'instMagErr_aper_outer'].asKey()
258 for ctr, visit
in enumerate(visitCat):
259 if visit[
'sources_read']:
262 expTime = visit[
'exptime']
267 aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
269 for dataRef
in groupedDataRefs[visit[
'visit']]:
271 ccdId = dataRef.dataId[self.config.ccdDataRefName]
273 sources = dataRef.get(datasetType=
'src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
279 if self.config.doSubtractLocalBackground:
291 localBackground = localBackgroundArea*sources[localBackgroundFluxKey]
292 sources[instFluxKey] -= localBackground
294 goodSrc = self.sourceSelector.selectSources(sources)
296 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
297 tempCat.reserve(goodSrc.selected.sum())
298 tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
299 tempCat[visitKey][:] = visit[
'visit']
300 tempCat[ccdKey][:] = ccdId
303 scaledInstFlux = (sources[instFluxKey][goodSrc.selected] *
304 visit[
'scaling'][ccdMapping[ccdId]])
305 tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
310 tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected] /
311 sources[instFluxKey][goodSrc.selected])
314 tempCat[
'jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat[
'x'],
318 if self.config.doApplyWcsJacobian:
319 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat[
'jacobian'][:])
321 fullCatalog.extend(tempCat)
325 tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
326 tempAperCat.reserve(goodSrc.selected.sum())
327 tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper)
329 with np.warnings.catch_warnings():
332 np.warnings.simplefilter(
"ignore")
334 tempAperCat[instMagInKey][:] = -2.5*np.log10(
335 sources[instFluxAperInKey][goodSrc.selected])
336 tempAperCat[instMagErrInKey][:] = k*(
337 sources[instFluxErrAperInKey][goodSrc.selected] /
338 sources[instFluxAperInKey][goodSrc.selected])
339 tempAperCat[instMagOutKey][:] = -2.5*np.log10(
340 sources[instFluxAperOutKey][goodSrc.selected])
341 tempAperCat[instMagErrOutKey][:] = k*(
342 sources[instFluxErrAperOutKey][goodSrc.selected] /
343 sources[instFluxAperOutKey][goodSrc.selected])
345 aperVisitCatalog.extend(tempAperCat)
347 nStarInVisit += len(tempCat)
350 if not aperVisitCatalog.isContiguous():
351 aperVisitCatalog = aperVisitCatalog.copy(deep=
True)
353 instMagIn = aperVisitCatalog[instMagInKey]
354 instMagErrIn = aperVisitCatalog[instMagErrInKey]
355 instMagOut = aperVisitCatalog[instMagOutKey]
356 instMagErrOut = aperVisitCatalog[instMagErrOutKey]
358 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
359 np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
361 visit[
'deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
362 visit[
'sources_read'] =
True
364 self.log.info(
" Found %d good stars in visit %d (deltaAper = %.3f)" %
365 (nStarInVisit, visit[
'visit'], visit[
'deltaAper']))
367 if ((ctr % self.config.nVisitsPerCheckpoint) == 0
and
368 starObsDataRef
is not None and visitCatDataRef
is not None):
371 starObsDataRef.put(fullCatalog)
372 visitCatDataRef.put(visitCat)
374 self.log.info(
"Found all good star observations in %.2f s" %
375 (time.time() - startTime))
379 def _makeAperMapper(self, sourceSchema):
381 Make a schema mapper for fgcm aperture measurements
385 sourceSchema: `afwTable.Schema`
386 Default source schema from the butler
390 aperMapper: `afwTable.schemaMapper`
391 Mapper to the FGCM aperture schema
394 aperMapper = afwTable.SchemaMapper(sourceSchema)
395 aperMapper.addMapping(sourceSchema[
'coord_ra'].asKey(),
'ra')
396 aperMapper.addMapping(sourceSchema[
'coord_dec'].asKey(),
'dec')
397 aperMapper.editOutputSchema().addField(
'instMag_aper_inner', type=np.float64,
398 doc=
"Magnitude at inner aperture")
399 aperMapper.editOutputSchema().addField(
'instMagErr_aper_inner', type=np.float64,
400 doc=
"Magnitude error at inner aperture")
401 aperMapper.editOutputSchema().addField(
'instMag_aper_outer', type=np.float64,
402 doc=
"Magnitude at outer aperture")
403 aperMapper.editOutputSchema().addField(
'instMagErr_aper_outer', type=np.float64,
404 doc=
"Magnitude error at outer aperture")