23 """Build star observations for input to FGCM using sourceTable_visit.
25 This task finds all the visits and sourceTable_visits in a repository (or a
26 subset based on command line parameters) and extracts all the potential
27 calibration stars for input into fgcm. This task additionally uses fgcm to
28 match star observations into unique stars, and performs as much cleaning of the
29 input catalog as possible.
37 import lsst.daf.persistence
as dafPersist
38 import lsst.pex.config
as pexConfig
39 import lsst.pipe.base
as pipeBase
40 from lsst.pipe.base
import connectionTypes
41 import lsst.afw.table
as afwTable
42 from lsst.meas.algorithms
import ReferenceObjectLoader
44 from .fgcmBuildStarsBase
import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
45 from .utilities
import computeApproxPixelAreaFields, computeApertureRadiusFromDataRef
46 from .utilities
import lookupStaticCalibrations
48 __all__ = [
'FgcmBuildStarsTableConfig',
'FgcmBuildStarsTableTask']
52 dimensions=(
"instrument",),
54 camera = connectionTypes.PrerequisiteInput(
55 doc=
"Camera instrument",
57 storageClass=
"Camera",
58 dimensions=(
"instrument",),
59 lookupFunction=lookupStaticCalibrations,
63 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
64 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
65 "chromatic corrections."),
66 name=
"fgcmLookUpTable",
67 storageClass=
"Catalog",
68 dimensions=(
"instrument",),
72 sourceSchema = connectionTypes.PrerequisiteInput(
73 doc=
"Schema for source catalogs",
75 storageClass=
"SourceCatalog",
79 refCat = connectionTypes.PrerequisiteInput(
80 doc=
"Reference catalog to use for photometric calibration",
82 storageClass=
"SimpleCatalog",
83 dimensions=(
"skypix",),
88 sourceTable_visit = connectionTypes.Input(
89 doc=
"Source table in parquet format, per visit",
90 name=
"sourceTable_visit",
91 storageClass=
"DataFrame",
92 dimensions=(
"instrument",
"visit"),
97 calexp = connectionTypes.Input(
98 doc=
"Calibrated exposures used for psf and metadata",
100 storageClass=
"ExposureF",
101 dimensions=(
"instrument",
"visit",
"detector"),
106 background = connectionTypes.Input(
107 doc=
"Calexp background model",
108 name=
"calexpBackground",
109 storageClass=
"Background",
110 dimensions=(
"instrument",
"visit",
"detector"),
115 fgcmVisitCatalog = connectionTypes.Output(
116 doc=
"Catalog of visit information for fgcm",
117 name=
"fgcmVisitCatalog",
118 storageClass=
"Catalog",
119 dimensions=(
"instrument",),
122 fgcmStarObservations = connectionTypes.Output(
123 doc=
"Catalog of star observations for fgcm",
124 name=
"fgcmStarObservations",
125 storageClass=
"Catalog",
126 dimensions=(
"instrument",),
129 fgcmStarIds = connectionTypes.Output(
130 doc=
"Catalog of fgcm calibration star IDs",
132 storageClass=
"Catalog",
133 dimensions=(
"instrument",),
136 fgcmStarIndices = connectionTypes.Output(
137 doc=
"Catalog of fgcm calibration star indices",
138 name=
"fgcmStarIndices",
139 storageClass=
"Catalog",
140 dimensions=(
"instrument",),
143 fgcmReferenceStars = connectionTypes.Output(
144 doc=
"Catalog of fgcm-matched reference stars",
145 name=
"fgcmReferenceStars",
146 storageClass=
"Catalog",
147 dimensions=(
"instrument",),
153 if not config.doReferenceMatches:
154 self.prerequisiteInputs.remove(
"refCat")
155 self.prerequisiteInputs.remove(
"fgcmLookUpTable")
157 if not config.doModelErrorsWithBackground:
158 self.inputs.remove(
"background")
160 if not config.doReferenceMatches:
161 self.outputs.remove(
"fgcmReferenceStars")
165 pipelineConnections=FgcmBuildStarsTableConnections):
166 """Config for FgcmBuildStarsTableTask"""
168 referenceCCD = pexConfig.Field(
169 doc=
"Reference CCD for checking PSF and background",
189 fluxFlagName = self.
instFluxField[0: -len(
'instFlux')] +
'flag'
191 sourceSelector.flags.bad = [
'PixelFlags_edge',
192 'PixelFlags_interpolatedCenter',
193 'PixelFlags_saturatedCenter',
194 'PixelFlags_crCenter',
196 'PixelFlags_interpolated',
197 'PixelFlags_saturated',
203 sourceSelector.flags.bad.append(localBackgroundFlagName)
206 sourceSelector.signalToNoise.errField = self.
instFluxField +
'Err'
208 sourceSelector.isolated.parentName =
'parentSourceId'
209 sourceSelector.isolated.nChildName =
'Deblend_nChild'
211 sourceSelector.unresolved.name =
'extendedness'
216 Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
218 ConfigClass = FgcmBuildStarsTableConfig
219 RunnerClass = FgcmBuildStarsRunner
220 _DefaultName =
"fgcmBuildStarsTable"
222 canMultiprocess =
False
225 inputRefDict = butlerQC.get(inputRefs)
227 dataRefs = inputRefDict[
'sourceTable_visit']
229 self.log.info(
"Running with %d sourceTable_visit dataRefs", (len(dataRefs)))
231 if self.config.doReferenceMatches:
233 lutDataRef = inputRefDict[
'fgcmLookUpTable']
236 refConfig = self.config.fgcmLoadReferenceCatalog.refObjLoader
237 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
238 for ref
in inputRefs.refCat],
239 refCats=butlerQC.get(inputRefs.refCat),
242 self.makeSubtask(
'fgcmLoadReferenceCatalog', refObjLoader=refObjLoader)
248 calibFluxApertureRadius =
None
249 if self.config.doSubtractLocalBackground:
252 self.config.instFluxField)
253 except RuntimeError
as e:
254 raise RuntimeError(
"Could not determine aperture radius from %s. "
255 "Cannot use doSubtractLocalBackground." %
256 (self.config.instFluxField))
from e
258 calexpRefs = inputRefDict[
'calexp']
259 calexpDataRefDict = {(calexpRef.dataId.byName()[
'visit'],
260 calexpRef.dataId.byName()[
'detector']): calexpRef
for
261 calexpRef
in calexpRefs}
263 camera = inputRefDict[
'camera']
265 calexpDataRefDict=calexpDataRefDict)
267 if self.config.doModelErrorsWithBackground:
268 bkgRefs = inputRefDict[
'background']
269 bkgDataRefDict = {(bkgRef.dataId.byName()[
'visit'],
270 bkgRef.dataId.byName()[
'detector']): bkgRef
for
273 bkgDataRefDict =
None
278 bkgDataRefDict=bkgDataRefDict,
279 visitCatDataRef=
None,
282 rad = calibFluxApertureRadius
283 sourceSchemaDataRef = inputRefDict[
'sourceSchema']
288 calibFluxApertureRadius=rad,
290 visitCatDataRef=
None,
293 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
294 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
296 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.
fgcmMatchStars(visitCat,
297 fgcmStarObservationCat,
298 lutDataRef=lutDataRef)
300 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
301 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
302 if fgcmRefCat
is not None:
303 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
306 def _makeArgumentParser(cls):
307 """Create an argument parser"""
309 parser.add_id_argument(
"--id",
"sourceTable_visit", help=
"Data ID, e.g. --id visit=6789")
313 def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None):
314 if (butler
is None and calexpDataRefDict
is None)
or \
315 (butler
is not None and calexpDataRefDict
is not None):
316 raise RuntimeError(
"Must either set butler (Gen2) or dataRefDict (Gen3)")
318 self.log.info(
"Grouping dataRefs by %s", (self.config.visitDataRefName))
321 for detector
in camera:
322 ccdIds.append(detector.getId())
326 ccdIds.insert(0, self.config.referenceCCD)
333 groupedDataRefs = collections.defaultdict(list)
334 for dataRef
in dataRefs:
335 visit = dataRef.dataId[self.config.visitDataRefName]
340 if butler
is not None:
343 calexpRef = butler.dataRef(
'calexp', dataId={self.config.visitDataRefName: visit,
344 self.config.ccdDataRefName: ccdId})
350 calexpRef = calexpDataRefDict.get((visit, ccdId))
351 if calexpRef
is None:
356 groupedDataRefs[visit].append(calexpRef)
360 groupedDataRefs[visit].append(dataRef)
363 return dict(sorted(groupedDataRefs.items()))
368 calibFluxApertureRadius=None,
369 visitCatDataRef=None,
372 startTime = time.time()
378 if (visitCatDataRef
is not None and starObsDataRef
is None
379 or visitCatDataRef
is None and starObsDataRef
is not None):
380 self.log.warn(
"Only one of visitCatDataRef and starObsDataRef are set, so "
381 "no checkpoint files will be persisted.")
383 if self.config.doSubtractLocalBackground
and calibFluxApertureRadius
is None:
384 raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
388 sourceSchema = sourceSchemaDataRef.get().schema
390 outputSchema = sourceMapper.getOutputSchema()
394 for ccdIndex, detector
in enumerate(camera):
395 ccdMapping[detector.getId()] = ccdIndex
399 if inStarObsCat
is not None:
400 fullCatalog = inStarObsCat
401 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
402 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
403 if not comp1
or not comp2:
404 raise RuntimeError(
"Existing fgcmStarObservations file found with mismatched schema.")
406 fullCatalog = afwTable.BaseCatalog(outputSchema)
408 visitKey = outputSchema[
'visit'].asKey()
409 ccdKey = outputSchema[
'ccd'].asKey()
410 instMagKey = outputSchema[
'instMag'].asKey()
411 instMagErrKey = outputSchema[
'instMagErr'].asKey()
414 if self.config.doSubtractLocalBackground:
415 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
422 for counter, visit
in enumerate(visitCat):
424 if visit[
'sources_read']:
427 expTime = visit[
'exptime']
429 dataRef = groupedDataRefs[visit[
'visit']][-1]
431 if isinstance(dataRef, dafPersist.ButlerDataRef):
432 srcTable = dataRef.get()
433 df = srcTable.toDataFrame(columns)
435 df = dataRef.get(parameters={
'columns': columns})
437 goodSrc = self.sourceSelector.selectSources(df)
441 if self.config.doSubtractLocalBackground:
442 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
443 use, = np.where((goodSrc.selected)
444 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
446 use, = np.where(goodSrc.selected)
448 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
449 tempCat.resize(use.size)
451 tempCat[
'ra'][:] = np.deg2rad(df[
'ra'].values[use])
452 tempCat[
'dec'][:] = np.deg2rad(df[
'decl'].values[use])
453 tempCat[
'x'][:] = df[
'x'].values[use]
454 tempCat[
'y'][:] = df[
'y'].values[use]
457 tempCat[visitKey][:] = df[
'visit'].values[use]
458 tempCat[ccdKey][:] = df[
'ccd'].values[use]
459 tempCat[
'psf_candidate'] = df[
'Calib_psf_candidate'].values[use]
461 if self.config.doSubtractLocalBackground:
475 tempCat[
'deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
476 - localBackground[use]) -
477 -2.5*np.log10(df[self.config.instFluxField].values[use]))
479 tempCat[
'deltaMagBkg'][:] = 0.0
482 for detector
in camera:
483 ccdId = detector.getId()
485 use2 = (tempCat[ccdKey] == ccdId)
486 tempCat[
'jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat[
'x'][use2],
488 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
489 * visit[
'scaling'][ccdMapping[ccdId]])
490 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
494 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField +
'Err'].values[use]
495 / df[self.config.instFluxField].values[use])
498 if self.config.doApplyWcsJacobian:
499 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat[
'jacobian'][:])
501 fullCatalog.extend(tempCat)
504 with np.warnings.catch_warnings():
506 np.warnings.simplefilter(
"ignore")
508 instMagIn = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
509 instMagErrIn = k*(df[self.config.apertureInnerInstFluxField +
'Err'].values[use]
510 / df[self.config.apertureInnerInstFluxField].values[use])
511 instMagOut = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
512 instMagErrOut = k*(df[self.config.apertureOuterInstFluxField +
'Err'].values[use]
513 / df[self.config.apertureOuterInstFluxField].values[use])
515 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn)
516 & np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
518 visit[
'deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
519 visit[
'sources_read'] =
True
521 self.log.info(
" Found %d good stars in visit %d (deltaAper = %0.3f)",
522 use.size, visit[
'visit'], visit[
'deltaAper'])
524 if ((counter % self.config.nVisitsPerCheckpoint) == 0
525 and starObsDataRef
is not None and visitCatDataRef
is not None):
528 starObsDataRef.put(fullCatalog)
529 visitCatDataRef.put(visitCat)
531 self.log.info(
"Found all good star observations in %.2f s" %
532 (time.time() - startTime))
536 def _get_sourceTable_visit_columns(self):
538 Get the sourceTable_visit columns from the config.
543 List of columns to read from sourceTable_visit
546 columns = [
'visit',
'ccd',
547 'ra',
'decl',
'x',
'y', self.config.psfCandidateName,
548 self.config.instFluxField, self.config.instFluxField +
'Err',
549 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField +
'Err',
550 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField +
'Err']
551 if self.sourceSelector.config.doFlags:
552 columns.extend(self.sourceSelector.config.flags.bad)
553 if self.sourceSelector.config.doUnresolved:
554 columns.append(self.sourceSelector.config.unresolved.name)
555 if self.sourceSelector.config.doIsolated:
556 columns.append(self.sourceSelector.config.isolated.parentName)
557 columns.append(self.sourceSelector.config.isolated.nChildName)
558 if self.config.doSubtractLocalBackground:
559 columns.append(self.config.localBackgroundFluxField)