23 """Build star observations for input to FGCM using sourceTable_visit.
25 This task finds all the visits and sourceTable_visits in a repository (or a
26 subset based on command line parameters) and extracts all the potential
27 calibration stars for input into fgcm. This task additionally uses fgcm to
28 match star observations into unique stars, and performs as much cleaning of the
29 input catalog as possible.
37 import lsst.pex.config
as pexConfig
38 import lsst.pipe.base
as pipeBase
39 import lsst.afw.table
as afwTable
41 from .fgcmBuildStarsBase
import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
42 from .utilities
import computeApproxPixelAreaFields
44 __all__ = [
'FgcmBuildStarsTableConfig',
'FgcmBuildStarsTableTask']
48 """Config for FgcmBuildStarsTableTask"""
50 referenceCCD = pexConfig.Field(
51 doc=
"Reference CCD for checking PSF and background",
71 fluxFlagName = self.
instFluxField[0: -len(
'instFlux')] +
'flag'
73 sourceSelector.flags.bad = [
'PixelFlags_edge',
74 'PixelFlags_interpolatedCenter',
75 'PixelFlags_saturatedCenter',
76 'PixelFlags_crCenter',
78 'PixelFlags_interpolated',
79 'PixelFlags_saturated',
85 sourceSelector.flags.bad.append(localBackgroundFlagName)
88 sourceSelector.signalToNoise.errField = self.
instFluxField +
'Err'
90 sourceSelector.isolated.parentName =
'parentSourceId'
91 sourceSelector.isolated.nChildName =
'Deblend_nChild'
93 sourceSelector.unresolved.name =
'extendedness'
98 Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
100 ConfigClass = FgcmBuildStarsTableConfig
101 RunnerClass = FgcmBuildStarsRunner
102 _DefaultName =
"fgcmBuildStarsTable"
105 def _makeArgumentParser(cls):
106 """Create an argument parser"""
108 parser.add_id_argument(
"--id",
"sourceTable_visit", help=
"Data ID, e.g. --id visit=6789")
113 self.log.info(
"Grouping dataRefs by %s" % (self.config.visitDataRefName))
115 camera = butler.get(
'camera')
118 for detector
in camera:
119 ccdIds.append(detector.getId())
123 ccdIds.insert(0, self.config.referenceCCD)
130 groupedDataRefs = collections.defaultdict(list)
131 for dataRef
in dataRefs:
132 visit = dataRef.dataId[self.config.visitDataRefName]
138 calexpRef = butler.dataRef(
'calexp', dataId={self.config.visitDataRefName: visit,
139 self.config.ccdDataRefName: ccdId})
145 groupedDataRefs[visit].append(calexpRef)
149 groupedDataRefs[visit].append(dataRef)
151 return groupedDataRefs
154 calibFluxApertureRadius=None,
155 visitCatDataRef=None,
158 startTime = time.time()
164 if (visitCatDataRef
is not None and starObsDataRef
is None or
165 visitCatDataRef
is None and starObsDataRef
is not None):
166 self.log.warn(
"Only one of visitCatDataRef and starObsDataRef are set, so "
167 "no checkpoint files will be persisted.")
169 if self.config.doSubtractLocalBackground
and calibFluxApertureRadius
is None:
170 raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
174 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
175 sourceSchema = dataRef.get(
'src_schema', immediate=
True).schema
177 outputSchema = sourceMapper.getOutputSchema()
180 camera = dataRef.get(
'camera')
182 for ccdIndex, detector
in enumerate(camera):
183 ccdMapping[detector.getId()] = ccdIndex
187 if inStarObsCat
is not None:
188 fullCatalog = inStarObsCat
189 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
190 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
191 if not comp1
or not comp2:
192 raise RuntimeError(
"Existing fgcmStarObservations file found with mismatched schema.")
194 fullCatalog = afwTable.BaseCatalog(outputSchema)
196 visitKey = outputSchema[
'visit'].asKey()
197 ccdKey = outputSchema[
'ccd'].asKey()
198 instMagKey = outputSchema[
'instMag'].asKey()
199 instMagErrKey = outputSchema[
'instMagErr'].asKey()
202 if self.config.doSubtractLocalBackground:
203 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
210 for counter, visit
in enumerate(visitCat):
212 if visit[
'sources_read']:
215 expTime = visit[
'exptime']
217 dataRef = groupedDataRefs[visit[
'visit']][-1]
218 srcTable = dataRef.get()
220 df = srcTable.toDataFrame(columns)
222 goodSrc = self.sourceSelector.selectSources(df)
223 use, = np.where(goodSrc.selected)
225 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
226 tempCat.resize(use.size)
228 tempCat[
'ra'][:] = np.deg2rad(df[
'ra'].values[use])
229 tempCat[
'dec'][:] = np.deg2rad(df[
'decl'].values[use])
230 tempCat[
'x'][:] = df[
'x'].values[use]
231 tempCat[
'y'][:] = df[
'y'].values[use]
232 tempCat[visitKey][:] = df[self.config.visitDataRefName].values[use]
233 tempCat[ccdKey][:] = df[self.config.ccdDataRefName].values[use]
234 tempCat[
'psf_candidate'] = df[
'Calib_psf_candidate'].values[use]
236 if self.config.doSubtractLocalBackground:
248 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
252 tempCat[
'deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] -
253 localBackground[use]) -
254 -2.5*np.log10(df[self.config.instFluxField].values[use]))
256 tempCat[
'deltaMagBkg'][:] = 0.0
259 for detector
in camera:
260 ccdId = detector.getId()
262 use2 = (tempCat[ccdKey] == ccdId)
263 tempCat[
'jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat[
'x'][use2],
265 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] *
266 visit[
'scaling'][ccdMapping[ccdId]])
267 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
271 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField +
'Err'].values[use] /
272 df[self.config.instFluxField].values[use])
275 if self.config.doApplyWcsJacobian:
276 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat[
'jacobian'][:])
278 fullCatalog.extend(tempCat)
281 instMagIn = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
282 instMagErrIn = k*(df[self.config.apertureInnerInstFluxField +
'Err'].values[use] /
283 df[self.config.apertureInnerInstFluxField].values[use])
284 instMagOut = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
285 instMagErrOut = k*(df[self.config.apertureOuterInstFluxField +
'Err'].values[use] /
286 df[self.config.apertureOuterInstFluxField].values[use])
287 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
288 np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
290 visit[
'deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
291 visit[
'sources_read'] =
True
293 self.log.info(
" Found %d good stars in visit %d (deltaAper = %0.3f)",
294 use.size, visit[
'visit'], visit[
'deltaAper'])
296 if ((counter % self.config.nVisitsPerCheckpoint) == 0
and
297 starObsDataRef
is not None and visitCatDataRef
is not None):
300 starObsDataRef.put(fullCatalog)
301 visitCatDataRef.put(visitCat)
303 self.log.info(
"Found all good star observations in %.2f s" %
304 (time.time() - startTime))
308 def _get_sourceTable_visit_columns(self):
310 Get the sourceTable_visit columns from the config.
315 List of columns to read from sourceTable_visit
317 columns = [self.config.visitDataRefName, self.config.ccdDataRefName,
318 'ra',
'decl',
'x',
'y', self.config.psfCandidateName,
319 self.config.instFluxField, self.config.instFluxField +
'Err',
320 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField +
'Err',
321 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField +
'Err']
322 if self.sourceSelector.config.doFlags:
323 columns.extend(self.sourceSelector.config.flags.bad)
324 if self.sourceSelector.config.doUnresolved:
325 columns.append(self.sourceSelector.config.unresolved.name)
326 if self.sourceSelector.config.doIsolated:
327 columns.append(self.sourceSelector.config.isolated.parentName)
328 columns.append(self.sourceSelector.config.isolated.nChildName)
329 if self.config.doSubtractLocalBackground:
330 columns.append(self.config.localBackgroundFluxField)