23"""Build star observations for input to FGCM using sourceTable_visit.
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs
as much cleaning of the
29input catalog
as possible.
38import lsst.pex.config as pexConfig
39import lsst.pipe.base as pipeBase
40from lsst.pipe.base import connectionTypes
41import lsst.afw.table as afwTable
42from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
46from .utilities import lookupStaticCalibrations
48__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
51class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
52 dimensions=("instrument",),
54 camera = connectionTypes.PrerequisiteInput(
55 doc=
"Camera instrument",
57 storageClass=
"Camera",
58 dimensions=(
"instrument",),
59 lookupFunction=lookupStaticCalibrations,
63 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
64 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
65 "chromatic corrections."),
66 name=
"fgcmLookUpTable",
67 storageClass=
"Catalog",
68 dimensions=(
"instrument",),
72 sourceSchema = connectionTypes.InitInput(
73 doc=
"Schema for source catalogs",
75 storageClass=
"SourceCatalog",
78 refCat = connectionTypes.PrerequisiteInput(
79 doc=
"Reference catalog to use for photometric calibration",
81 storageClass=
"SimpleCatalog",
82 dimensions=(
"skypix",),
87 sourceTable_visit = connectionTypes.Input(
88 doc=
"Source table in parquet format, per visit",
89 name=
"sourceTable_visit",
90 storageClass=
"DataFrame",
91 dimensions=(
"instrument",
"visit"),
96 visitSummary = connectionTypes.Input(
97 doc=(
"Per-visit consolidated exposure metadata. These catalogs use "
98 "detector id for the id and must be sorted for fast lookups of a "
101 storageClass=
"ExposureCatalog",
102 dimensions=(
"instrument",
"visit"),
107 fgcmVisitCatalog = connectionTypes.Output(
108 doc=
"Catalog of visit information for fgcm",
109 name=
"fgcmVisitCatalog",
110 storageClass=
"Catalog",
111 dimensions=(
"instrument",),
114 fgcmStarObservations = connectionTypes.Output(
115 doc=
"Catalog of star observations for fgcm",
116 name=
"fgcmStarObservations",
117 storageClass=
"Catalog",
118 dimensions=(
"instrument",),
121 fgcmStarIds = connectionTypes.Output(
122 doc=
"Catalog of fgcm calibration star IDs",
124 storageClass=
"Catalog",
125 dimensions=(
"instrument",),
128 fgcmStarIndices = connectionTypes.Output(
129 doc=
"Catalog of fgcm calibration star indices",
130 name=
"fgcmStarIndices",
131 storageClass=
"Catalog",
132 dimensions=(
"instrument",),
135 fgcmReferenceStars = connectionTypes.Output(
136 doc=
"Catalog of fgcm-matched reference stars",
137 name=
"fgcmReferenceStars",
138 storageClass=
"Catalog",
139 dimensions=(
"instrument",),
145 if not config.doReferenceMatches:
146 self.prerequisiteInputs.remove(
"refCat")
147 self.prerequisiteInputs.remove(
"fgcmLookUpTable")
149 if not config.doReferenceMatches:
150 self.outputs.remove(
"fgcmReferenceStars")
154 pipelineConnections=FgcmBuildStarsTableConnections):
155 """Config for FgcmBuildStarsTableTask"""
157 referenceCCD = pexConfig.Field(
158 doc=
"Reference CCD for checking PSF and background",
180 sourceSelector.flags.bad = [
'pixelFlags_edge',
181 'pixelFlags_interpolatedCenter',
182 'pixelFlags_saturatedCenter',
183 'pixelFlags_crCenter',
185 'pixelFlags_interpolated',
186 'pixelFlags_saturated',
192 sourceSelector.flags.bad.append(localBackgroundFlagName)
197 sourceSelector.isolated.parentName =
'parentSourceId'
198 sourceSelector.isolated.nChildName =
'deblend_nChild'
200 sourceSelector.requireFiniteRaDec.raColName =
'ra'
201 sourceSelector.requireFiniteRaDec.decColName =
'dec'
203 sourceSelector.unresolved.name =
'extendedness'
205 sourceSelector.doRequirePrimary =
True
210 Build stars for the FGCM
global calibration, using sourceTable_visit catalogs.
212 ConfigClass = FgcmBuildStarsTableConfig
213 _DefaultName = "fgcmBuildStarsTable"
215 canMultiprocess =
False
218 super().
__init__(initInputs=initInputs, **kwargs)
219 if initInputs
is not None:
223 inputRefDict = butlerQC.get(inputRefs)
225 sourceTableHandles = inputRefDict[
'sourceTable_visit']
227 self.log.info(
"Running with %d sourceTable_visit handles",
228 len(sourceTableHandles))
230 sourceTableHandleDict = {sourceTableHandle.dataId[
'visit']: sourceTableHandle
for
231 sourceTableHandle
in sourceTableHandles}
233 if self.config.doReferenceMatches:
235 lutHandle = inputRefDict[
'fgcmLookUpTable']
238 refConfig = LoadReferenceObjectsConfig()
239 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
240 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
241 for ref
in inputRefs.refCat],
242 refCats=butlerQC.get(inputRefs.refCat),
243 name=self.config.connections.refCat,
246 self.makeSubtask(
'fgcmLoadReferenceCatalog',
247 refObjLoader=refObjLoader,
248 refCatName=self.config.connections.refCat)
254 calibFluxApertureRadius =
None
255 if self.config.doSubtractLocalBackground:
257 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
258 except RuntimeError
as e:
259 raise RuntimeError(
"Could not determine aperture radius from %s. "
260 "Cannot use doSubtractLocalBackground." %
261 (self.config.instFluxField))
from e
263 visitSummaryHandles = inputRefDict[
'visitSummary']
264 visitSummaryHandleDict = {visitSummaryHandle.dataId[
'visit']: visitSummaryHandle
for
265 visitSummaryHandle
in visitSummaryHandles}
267 camera = inputRefDict[
'camera']
269 visitSummaryHandleDict)
273 rad = calibFluxApertureRadius
278 calibFluxApertureRadius=rad)
280 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
281 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
283 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.
fgcmMatchStars(visitCat,
284 fgcmStarObservationCat,
287 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
288 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
289 if fgcmRefCat
is not None:
290 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
292 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict):
293 """Group sourceTable and visitSummary handles.
297 sourceTableHandleDict : `dict` [`int`, `str`]
298 Dict of source tables, keyed by visit.
299 visitSummaryHandleDict : `dict` [int, `str`]
300 Dict of visit summary catalogs, keyed by visit.
304 groupedHandles : `dict` [`int`, `list`]
305 Dictionary with sorted visit keys,
and `list`s
with
306 `lsst.daf.butler.DeferredDataSetHandle`. The first
307 item
in the list will be the visitSummary ref,
and
308 the second will be the source table ref.
310 groupedHandles = collections.defaultdict(list)
311 visits = sorted(sourceTableHandleDict.keys())
314 groupedHandles[visit] = [visitSummaryHandleDict[visit],
315 sourceTableHandleDict[visit]]
317 return groupedHandles
322 calibFluxApertureRadius=None):
323 startTime = time.time()
325 if self.config.doSubtractLocalBackground
and calibFluxApertureRadius
is None:
326 raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
331 outputSchema = sourceMapper.getOutputSchema()
335 for ccdIndex, detector
in enumerate(camera):
336 ccdMapping[detector.getId()] = ccdIndex
338 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
340 fullCatalog = afwTable.BaseCatalog(outputSchema)
342 visitKey = outputSchema[
'visit'].asKey()
343 ccdKey = outputSchema[
'ccd'].asKey()
344 instMagKey = outputSchema[
'instMag'].asKey()
345 instMagErrKey = outputSchema[
'instMagErr'].asKey()
346 deltaMagAperKey = outputSchema[
'deltaMagAper'].asKey()
349 if self.config.doSubtractLocalBackground:
350 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
356 for counter, visit
in enumerate(visitCat):
357 expTime = visit[
'exptime']
359 handle = groupedHandles[visit[
'visit']][-1]
362 inColumns = handle.get(component=
'columns')
364 df = handle.get(parameters={
'columns': columns})
366 goodSrc = self.sourceSelector.selectSources(df)
370 if self.config.doSubtractLocalBackground:
371 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
372 use, = np.where((goodSrc.selected)
373 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
375 use, = np.where(goodSrc.selected)
377 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
378 tempCat.resize(use.size)
380 tempCat[
'ra'][:] = np.deg2rad(df[
'ra'].values[use])
381 tempCat[
'dec'][:] = np.deg2rad(df[
'dec'].values[use])
382 tempCat[
'x'][:] = df[
'x'].values[use]
383 tempCat[
'y'][:] = df[
'y'].values[use]
385 tempCat[visitKey][:] = df[
'visit'].values[use]
386 tempCat[ccdKey][:] = df[
'detector'].values[use]
387 tempCat[
'psf_candidate'] = df[self.config.psfCandidateName].values[use]
389 with warnings.catch_warnings():
391 warnings.simplefilter(
"ignore")
393 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
394 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField +
'Err'].values[use]
395 / df[self.config.apertureInnerInstFluxField].values[use])
396 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
397 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField +
'Err'].values[use]
398 / df[self.config.apertureOuterInstFluxField].values[use])
399 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
401 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
403 if self.config.doSubtractLocalBackground:
417 tempCat[
'deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
418 - localBackground[use]) -
419 -2.5*np.log10(df[self.config.instFluxField].values[use]))
421 tempCat[
'deltaMagBkg'][:] = 0.0
424 for detector
in camera:
425 ccdId = detector.getId()
427 use2 = (tempCat[ccdKey] == ccdId)
428 tempCat[
'jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat[
'x'][use2],
430 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
431 * visit[
'scaling'][ccdMapping[ccdId]])
432 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
436 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField +
'Err'].values[use]
437 / df[self.config.instFluxField].values[use])
440 if self.config.doApplyWcsJacobian:
441 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat[
'jacobian'][:])
443 fullCatalog.extend(tempCat)
445 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
446 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
448 visit[
'deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
449 visit[
'sources_read'] =
True
451 self.log.info(
" Found %d good stars in visit %d (deltaAper = %0.3f)",
452 use.size, visit[
'visit'], visit[
'deltaAper'])
454 self.log.info(
"Found all good star observations in %.2f s" %
455 (time.time() - startTime))
459 def _get_sourceTable_visit_columns(self, inColumns):
461 Get the sourceTable_visit columns from the config.
466 List of columns available
in the sourceTable_visit
471 List of columns to read
from sourceTable_visit.
474 columns = [
'visit',
'detector',
475 'ra',
'dec',
'x',
'y', self.config.psfCandidateName,
476 self.config.instFluxField, self.config.instFluxField +
'Err',
477 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField +
'Err',
478 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField +
'Err']
479 if self.sourceSelector.config.doFlags:
480 columns.extend(self.sourceSelector.config.flags.bad)
481 if self.sourceSelector.config.doUnresolved:
482 columns.append(self.sourceSelector.config.unresolved.name)
483 if self.sourceSelector.config.doIsolated:
484 columns.append(self.sourceSelector.config.isolated.parentName)
485 columns.append(self.sourceSelector.config.isolated.nChildName)
486 if self.sourceSelector.config.doRequirePrimary:
487 columns.append(self.sourceSelector.config.requirePrimary.primaryColName)
488 if self.config.doSubtractLocalBackground:
489 columns.append(self.config.localBackgroundFluxField)
def fgcmMakeVisitCatalog(self, camera, groupedHandles)
def fgcmMatchStars(self, visitCat, obsCat, lutHandle=None)
def _makeSourceMapper(self, sourceSchema)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
apertureOuterInstFluxField
apertureInnerInstFluxField
doSubtractLocalBackground
apertureInnerInstFluxField
apertureOuterInstFluxField
def __init__(self, *config=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def _get_sourceTable_visit_columns(self, inColumns)
def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict)
def __init__(self, initInputs=None, **kwargs)