23 """Build star observations for input to FGCM.
25 This task finds all the visits and calexps in a repository (or a subset
26 based on command line parameters) and extract all the potential calibration
27 stars for input into fgcm. This task additionally uses fgcm to match
28 star observations into unique stars, and performs as much cleaning of
29 the input catalog as possible.
36 import lsst.pex.config
as pexConfig
37 import lsst.pipe.base
as pipeBase
38 import lsst.afw.table
as afwTable
40 from .fgcmBuildStarsBase
import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
41 from .utilities
import computeApproxPixelAreaFields
43 __all__ = [
'FgcmBuildStarsConfig',
'FgcmBuildStarsTask']
47 """Config for FgcmBuildStarsTask"""
49 referenceCCD = pexConfig.Field(
50 doc=
"Reference CCD for scanning visits",
54 checkAllCcds = pexConfig.Field(
55 doc=(
"Check repo for all CCDs for each visit specified. To be used when the "
56 "full set of ids (visit/ccd) are not specified on the command line. For "
57 "Gen2, specifying one ccd and setting checkAllCcds=True is significantly "
58 "faster than the alternatives."),
72 fluxFlagName = self.
instFluxField[0: -len(
'instFlux')] +
'flag'
73 sourceSelector.flags.bad = [
'base_PixelFlags_flag_edge',
74 'base_PixelFlags_flag_interpolatedCenter',
75 'base_PixelFlags_flag_saturatedCenter',
76 'base_PixelFlags_flag_crCenter',
77 'base_PixelFlags_flag_bad',
78 'base_PixelFlags_flag_interpolated',
79 'base_PixelFlags_flag_saturated',
85 sourceSelector.flags.bad.append(localBackgroundFlagName)
88 sourceSelector.signalToNoise.errField = self.
instFluxField +
'Err'
93 Build stars for the FGCM global calibration, using src catalogs.
95 ConfigClass = FgcmBuildStarsConfig
96 RunnerClass = FgcmBuildStarsRunner
97 _DefaultName =
"fgcmBuildStars"
100 def _makeArgumentParser(cls):
101 """Create an argument parser"""
103 parser.add_id_argument(
"--id",
"src", help=
"Data ID, e.g. --id visit=6789")
107 def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None):
109 raise RuntimeError(
"Gen2 _findAndGroupDataRefs must be called with a butler.")
110 if calexpDataRefDict
is not None:
111 self.log.warn(
"Ignoring calexpDataRefDict in gen2 _findAndGroupDataRefs")
113 self.log.info(
"Grouping dataRefs by %s" % (self.config.visitDataRefName))
116 for detector
in camera:
117 ccdIds.append(detector.getId())
126 for dataRef
in dataRefs:
127 visit = dataRef.dataId[self.config.visitDataRefName]
129 if not dataRef.datasetExists(datasetType=
'src'):
132 if self.config.checkAllCcds:
133 if visit
in groupedDataRefs:
136 dataId = dataRef.dataId.copy()
139 dataId[self.config.ccdDataRefName] = ccdId
140 if butler.datasetExists(
'src', dataId=dataId):
141 goodDataRef = butler.dataRef(
'src', dataId=dataId)
142 if visit
in groupedDataRefs:
143 if (goodDataRef.dataId[self.config.ccdDataRefName]
not in
144 [d.dataId[self.config.ccdDataRefName]
for d
in groupedDataRefs[visit]]):
145 groupedDataRefs[visit].append(goodDataRef)
149 groupedDataRefs[visit] = [goodDataRef]
153 if visit
in groupedDataRefs:
154 if (dataRef.dataId[self.config.ccdDataRefName]
not in
155 [d.dataId[self.config.ccdDataRefName]
for d
in groupedDataRefs[visit]]):
156 groupedDataRefs[visit].append(dataRef)
160 groupedDataRefs[visit] = [dataRef]
162 if (nVisits % 100) == 0
and nVisits > 0:
163 self.log.info(
"Found %d unique %ss..." % (nVisits,
164 self.config.visitDataRefName))
166 self.log.info(
"Found %d unique %ss total." % (nVisits,
167 self.config.visitDataRefName))
170 def ccdSorter(dataRef):
171 ccdId = dataRef.dataId[self.config.ccdDataRefName]
172 if ccdId == self.config.referenceCCD:
178 if not self.config.checkAllCcds:
179 for visit
in groupedDataRefs:
180 groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter)
183 return dict(sorted(groupedDataRefs.items()))
188 calibFluxApertureRadius=None,
189 visitCatDataRef=None,
192 startTime = time.time()
198 if (visitCatDataRef
is not None and starObsDataRef
is None
199 or visitCatDataRef
is None and starObsDataRef
is not None):
200 self.log.warn(
"Only one of visitCatDataRef and starObsDataRef are set, so "
201 "no checkpoint files will be persisted.")
203 if self.config.doSubtractLocalBackground
and calibFluxApertureRadius
is None:
204 raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
207 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
208 sourceSchema = dataRef.get(
'src_schema', immediate=
True).schema
212 for ccdIndex, detector
in enumerate(camera):
213 ccdMapping[detector.getId()] = ccdIndex
222 outputSchema = sourceMapper.getOutputSchema()
224 if inStarObsCat
is not None:
225 fullCatalog = inStarObsCat
226 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
227 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
228 if not comp1
or not comp2:
229 raise RuntimeError(
"Existing fgcmStarObservations file found with mismatched schema.")
231 fullCatalog = afwTable.BaseCatalog(outputSchema)
235 instFluxKey = sourceSchema[self.config.instFluxField].asKey()
236 instFluxErrKey = sourceSchema[self.config.instFluxField +
'Err'].asKey()
237 visitKey = outputSchema[
'visit'].asKey()
238 ccdKey = outputSchema[
'ccd'].asKey()
239 instMagKey = outputSchema[
'instMag'].asKey()
240 instMagErrKey = outputSchema[
'instMagErr'].asKey()
241 deltaMagBkgKey = outputSchema[
'deltaMagBkg'].asKey()
244 if self.config.doSubtractLocalBackground:
245 localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey()
246 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
248 aperOutputSchema = aperMapper.getOutputSchema()
250 instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey()
251 instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField +
'Err'].asKey()
252 instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey()
253 instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField +
'Err'].asKey()
254 instMagInKey = aperOutputSchema[
'instMag_aper_inner'].asKey()
255 instMagErrInKey = aperOutputSchema[
'instMagErr_aper_inner'].asKey()
256 instMagOutKey = aperOutputSchema[
'instMag_aper_outer'].asKey()
257 instMagErrOutKey = aperOutputSchema[
'instMagErr_aper_outer'].asKey()
262 for ctr, visit
in enumerate(visitCat):
263 if visit[
'sources_read']:
266 expTime = visit[
'exptime']
271 aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
273 for dataRef
in groupedDataRefs[visit[
'visit']]:
275 ccdId = dataRef.dataId[self.config.ccdDataRefName]
277 sources = dataRef.get(datasetType=
'src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
278 goodSrc = self.sourceSelector.selectSources(sources)
282 if self.config.doSubtractLocalBackground:
283 localBackground = localBackgroundArea*sources[localBackgroundFluxKey]
285 bad, = np.where((sources[instFluxKey] - localBackground) <= 0.0)
286 goodSrc.selected[bad] =
False
288 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
289 tempCat.reserve(goodSrc.selected.sum())
290 tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
291 tempCat[visitKey][:] = visit[
'visit']
292 tempCat[ccdKey][:] = ccdId
295 scaledInstFlux = (sources[instFluxKey][goodSrc.selected]
296 * visit[
'scaling'][ccdMapping[ccdId]])
297 tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
300 if self.config.doSubtractLocalBackground:
314 tempCat[deltaMagBkgKey][:] = (-2.5*np.log10(sources[instFluxKey][goodSrc.selected]
315 - localBackground[goodSrc.selected]) -
316 -2.5*np.log10(sources[instFluxKey][goodSrc.selected]))
318 tempCat[deltaMagBkgKey][:] = 0.0
323 tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected]
324 / sources[instFluxKey][goodSrc.selected])
327 tempCat[
'jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat[
'x'],
331 if self.config.doApplyWcsJacobian:
332 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat[
'jacobian'][:])
334 fullCatalog.extend(tempCat)
338 tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
339 tempAperCat.reserve(goodSrc.selected.sum())
340 tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper)
342 with np.warnings.catch_warnings():
345 np.warnings.simplefilter(
"ignore")
347 tempAperCat[instMagInKey][:] = -2.5*np.log10(
348 sources[instFluxAperInKey][goodSrc.selected])
349 tempAperCat[instMagErrInKey][:] = k*(
350 sources[instFluxErrAperInKey][goodSrc.selected]
351 / sources[instFluxAperInKey][goodSrc.selected])
352 tempAperCat[instMagOutKey][:] = -2.5*np.log10(
353 sources[instFluxAperOutKey][goodSrc.selected])
354 tempAperCat[instMagErrOutKey][:] = k*(
355 sources[instFluxErrAperOutKey][goodSrc.selected]
356 / sources[instFluxAperOutKey][goodSrc.selected])
358 aperVisitCatalog.extend(tempAperCat)
360 nStarInVisit += len(tempCat)
363 if not aperVisitCatalog.isContiguous():
364 aperVisitCatalog = aperVisitCatalog.copy(deep=
True)
366 instMagIn = aperVisitCatalog[instMagInKey]
367 instMagErrIn = aperVisitCatalog[instMagErrInKey]
368 instMagOut = aperVisitCatalog[instMagOutKey]
369 instMagErrOut = aperVisitCatalog[instMagErrOutKey]
371 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn)
372 & np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
374 visit[
'deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
375 visit[
'sources_read'] =
True
377 self.log.info(
" Found %d good stars in visit %d (deltaAper = %.3f)" %
378 (nStarInVisit, visit[
'visit'], visit[
'deltaAper']))
380 if ((ctr % self.config.nVisitsPerCheckpoint) == 0
381 and starObsDataRef
is not None and visitCatDataRef
is not None):
384 starObsDataRef.put(fullCatalog)
385 visitCatDataRef.put(visitCat)
387 self.log.info(
"Found all good star observations in %.2f s" %
388 (time.time() - startTime))
392 def _makeAperMapper(self, sourceSchema):
394 Make a schema mapper for fgcm aperture measurements
398 sourceSchema: `afwTable.Schema`
399 Default source schema from the butler
403 aperMapper: `afwTable.schemaMapper`
404 Mapper to the FGCM aperture schema
407 aperMapper = afwTable.SchemaMapper(sourceSchema)
408 aperMapper.addMapping(sourceSchema[
'coord_ra'].asKey(),
'ra')
409 aperMapper.addMapping(sourceSchema[
'coord_dec'].asKey(),
'dec')
410 aperMapper.editOutputSchema().addField(
'instMag_aper_inner', type=np.float64,
411 doc=
"Magnitude at inner aperture")
412 aperMapper.editOutputSchema().addField(
'instMagErr_aper_inner', type=np.float64,
413 doc=
"Magnitude error at inner aperture")
414 aperMapper.editOutputSchema().addField(
'instMag_aper_outer', type=np.float64,
415 doc=
"Magnitude at outer aperture")
416 aperMapper.editOutputSchema().addField(
'instMagErr_aper_outer', type=np.float64,
417 doc=
"Magnitude error at outer aperture")