23"""Build star observations for input to FGCM using sourceTable_visit.
25This task finds all the visits and sourceTable_visits in a repository (or a
26subset based on command line parameters) and extracts all the potential
27calibration stars for input into fgcm. This task additionally uses fgcm to
28match star observations into unique stars, and performs
as much cleaning of the
29input catalog
as possible.
38import lsst.pex.config as pexConfig
39import lsst.pipe.base as pipeBase
40from lsst.pipe.base import connectionTypes
41import lsst.afw.table as afwTable
42from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask
45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName
46from .utilities import lookupStaticCalibrations
48__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
51class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections,
52 dimensions=("instrument",),
54 camera = connectionTypes.PrerequisiteInput(
55 doc=
"Camera instrument",
57 storageClass=
"Camera",
58 dimensions=(
"instrument",),
59 lookupFunction=lookupStaticCalibrations,
63 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
64 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
65 "chromatic corrections."),
66 name=
"fgcmLookUpTable",
67 storageClass=
"Catalog",
68 dimensions=(
"instrument",),
72 sourceSchema = connectionTypes.InitInput(
73 doc=
"Schema for source catalogs",
75 storageClass=
"SourceCatalog",
78 refCat = connectionTypes.PrerequisiteInput(
79 doc=
"Reference catalog to use for photometric calibration",
81 storageClass=
"SimpleCatalog",
82 dimensions=(
"skypix",),
87 sourceTable_visit = connectionTypes.Input(
88 doc=
"Source table in parquet format, per visit",
89 name=
"sourceTable_visit",
90 storageClass=
"DataFrame",
91 dimensions=(
"instrument",
"visit"),
96 visitSummary = connectionTypes.Input(
97 doc=(
"Per-visit consolidated exposure metadata. These catalogs use "
98 "detector id for the id and must be sorted for fast lookups of a "
101 storageClass=
"ExposureCatalog",
102 dimensions=(
"instrument",
"visit"),
107 fgcmVisitCatalog = connectionTypes.Output(
108 doc=
"Catalog of visit information for fgcm",
109 name=
"fgcmVisitCatalog",
110 storageClass=
"Catalog",
111 dimensions=(
"instrument",),
114 fgcmStarObservations = connectionTypes.Output(
115 doc=
"Catalog of star observations for fgcm",
116 name=
"fgcmStarObservations",
117 storageClass=
"Catalog",
118 dimensions=(
"instrument",),
121 fgcmStarIds = connectionTypes.Output(
122 doc=
"Catalog of fgcm calibration star IDs",
124 storageClass=
"Catalog",
125 dimensions=(
"instrument",),
128 fgcmStarIndices = connectionTypes.Output(
129 doc=
"Catalog of fgcm calibration star indices",
130 name=
"fgcmStarIndices",
131 storageClass=
"Catalog",
132 dimensions=(
"instrument",),
135 fgcmReferenceStars = connectionTypes.Output(
136 doc=
"Catalog of fgcm-matched reference stars",
137 name=
"fgcmReferenceStars",
138 storageClass=
"Catalog",
139 dimensions=(
"instrument",),
145 if not config.doReferenceMatches:
146 self.prerequisiteInputs.remove(
"refCat")
147 self.prerequisiteInputs.remove(
"fgcmLookUpTable")
149 if not config.doReferenceMatches:
150 self.outputs.remove(
"fgcmReferenceStars")
154 pipelineConnections=FgcmBuildStarsTableConnections):
155 """Config for FgcmBuildStarsTableTask"""
157 referenceCCD = pexConfig.Field(
158 doc=
"Reference CCD for checking PSF and background",
180 sourceSelector.flags.bad = [
'pixelFlags_edge',
181 'pixelFlags_interpolatedCenter',
182 'pixelFlags_saturatedCenter',
183 'pixelFlags_crCenter',
185 'pixelFlags_interpolated',
186 'pixelFlags_saturated',
192 sourceSelector.flags.bad.append(localBackgroundFlagName)
197 sourceSelector.isolated.parentName =
'parentSourceId'
198 sourceSelector.isolated.nChildName =
'deblend_nChild'
200 sourceSelector.requireFiniteRaDec.raColName =
'ra'
201 sourceSelector.requireFiniteRaDec.decColName =
'decl'
203 sourceSelector.unresolved.name =
'extendedness'
208 Build stars for the FGCM
global calibration, using sourceTable_visit catalogs.
210 ConfigClass = FgcmBuildStarsTableConfig
211 _DefaultName = "fgcmBuildStarsTable"
213 canMultiprocess =
False
216 super().
__init__(initInputs=initInputs, **kwargs)
217 if initInputs
is not None:
221 inputRefDict = butlerQC.get(inputRefs)
223 sourceTableHandles = inputRefDict[
'sourceTable_visit']
225 self.log.info(
"Running with %d sourceTable_visit handles",
226 len(sourceTableHandles))
228 sourceTableHandleDict = {sourceTableHandle.dataId[
'visit']: sourceTableHandle
for
229 sourceTableHandle
in sourceTableHandles}
231 if self.config.doReferenceMatches:
233 lutHandle = inputRefDict[
'fgcmLookUpTable']
236 refConfig = LoadReferenceObjectsConfig()
237 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
238 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
239 for ref
in inputRefs.refCat],
240 refCats=butlerQC.get(inputRefs.refCat),
241 name=self.config.connections.refCat,
244 self.makeSubtask(
'fgcmLoadReferenceCatalog',
245 refObjLoader=refObjLoader,
246 refCatName=self.config.connections.refCat)
252 calibFluxApertureRadius =
None
253 if self.config.doSubtractLocalBackground:
255 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField)
256 except RuntimeError
as e:
257 raise RuntimeError(
"Could not determine aperture radius from %s. "
258 "Cannot use doSubtractLocalBackground." %
259 (self.config.instFluxField))
from e
261 visitSummaryHandles = inputRefDict[
'visitSummary']
262 visitSummaryHandleDict = {visitSummaryHandle.dataId[
'visit']: visitSummaryHandle
for
263 visitSummaryHandle
in visitSummaryHandles}
265 camera = inputRefDict[
'camera']
267 visitSummaryHandleDict)
271 rad = calibFluxApertureRadius
276 calibFluxApertureRadius=rad)
278 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
279 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
281 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.
fgcmMatchStars(visitCat,
282 fgcmStarObservationCat,
285 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
286 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
287 if fgcmRefCat
is not None:
288 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
290 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict):
291 """Group sourceTable and visitSummary handles.
295 sourceTableHandleDict : `dict` [`int`, `str`]
296 Dict of source tables, keyed by visit.
297 visitSummaryHandleDict : `dict` [int, `str`]
298 Dict of visit summary catalogs, keyed by visit.
302 groupedHandles : `dict` [`int`, `list`]
303 Dictionary with sorted visit keys,
and `list`s
with
304 `lsst.daf.butler.DeferredDataSetHandle`. The first
305 item
in the list will be the visitSummary ref,
and
306 the second will be the source table ref.
308 groupedHandles = collections.defaultdict(list)
309 visits = sorted(sourceTableHandleDict.keys())
312 groupedHandles[visit] = [visitSummaryHandleDict[visit],
313 sourceTableHandleDict[visit]]
315 return groupedHandles
320 calibFluxApertureRadius=None):
321 startTime = time.time()
323 if self.config.doSubtractLocalBackground
and calibFluxApertureRadius
is None:
324 raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
329 outputSchema = sourceMapper.getOutputSchema()
333 for ccdIndex, detector
in enumerate(camera):
334 ccdMapping[detector.getId()] = ccdIndex
336 approxPixelAreaFields = computeApproxPixelAreaFields(camera)
338 fullCatalog = afwTable.BaseCatalog(outputSchema)
340 visitKey = outputSchema[
'visit'].asKey()
341 ccdKey = outputSchema[
'ccd'].asKey()
342 instMagKey = outputSchema[
'instMag'].asKey()
343 instMagErrKey = outputSchema[
'instMagErr'].asKey()
344 deltaMagAperKey = outputSchema[
'deltaMagAper'].asKey()
347 if self.config.doSubtractLocalBackground:
348 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
354 for counter, visit
in enumerate(visitCat):
355 expTime = visit[
'exptime']
357 handle = groupedHandles[visit[
'visit']][-1]
360 inColumns = handle.get(component=
'columns')
362 df = handle.get(parameters={
'columns': columns})
364 goodSrc = self.sourceSelector.selectSources(df)
368 if self.config.doSubtractLocalBackground:
369 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
370 use, = np.where((goodSrc.selected)
371 & ((df[self.config.instFluxField].values - localBackground) > 0.0))
373 use, = np.where(goodSrc.selected)
375 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
376 tempCat.resize(use.size)
378 tempCat[
'ra'][:] = np.deg2rad(df[
'ra'].values[use])
379 tempCat[
'dec'][:] = np.deg2rad(df[
'decl'].values[use])
380 tempCat[
'x'][:] = df[
'x'].values[use]
381 tempCat[
'y'][:] = df[
'y'].values[use]
383 tempCat[visitKey][:] = df[
'visit'].values[use]
384 tempCat[ccdKey][:] = df[
'detector'].values[use]
385 tempCat[
'psf_candidate'] = df[self.config.psfCandidateName].values[use]
387 with warnings.catch_warnings():
389 warnings.simplefilter(
"ignore")
391 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
392 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField +
'Err'].values[use]
393 / df[self.config.apertureInnerInstFluxField].values[use])
394 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
395 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField +
'Err'].values[use]
396 / df[self.config.apertureOuterInstFluxField].values[use])
397 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
399 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
401 if self.config.doSubtractLocalBackground:
415 tempCat[
'deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use]
416 - localBackground[use]) -
417 -2.5*np.log10(df[self.config.instFluxField].values[use]))
419 tempCat[
'deltaMagBkg'][:] = 0.0
422 for detector
in camera:
423 ccdId = detector.getId()
425 use2 = (tempCat[ccdKey] == ccdId)
426 tempCat[
'jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat[
'x'][use2],
428 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]]
429 * visit[
'scaling'][ccdMapping[ccdId]])
430 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
434 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField +
'Err'].values[use]
435 / df[self.config.instFluxField].values[use])
438 if self.config.doApplyWcsJacobian:
439 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat[
'jacobian'][:])
441 fullCatalog.extend(tempCat)
443 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
444 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter))
446 visit[
'deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk])
447 visit[
'sources_read'] =
True
449 self.log.info(
" Found %d good stars in visit %d (deltaAper = %0.3f)",
450 use.size, visit[
'visit'], visit[
'deltaAper'])
452 self.log.info(
"Found all good star observations in %.2f s" %
453 (time.time() - startTime))
457 def _get_sourceTable_visit_columns(self, inColumns):
459 Get the sourceTable_visit columns from the config.
464 List of columns available
in the sourceTable_visit
469 List of columns to read
from sourceTable_visit.
472 columns = [
'visit',
'detector',
473 'ra',
'decl',
'x',
'y', self.config.psfCandidateName,
474 self.config.instFluxField, self.config.instFluxField +
'Err',
475 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField +
'Err',
476 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField +
'Err']
477 if self.sourceSelector.config.doFlags:
478 columns.extend(self.sourceSelector.config.flags.bad)
479 if self.sourceSelector.config.doUnresolved:
480 columns.append(self.sourceSelector.config.unresolved.name)
481 if self.sourceSelector.config.doIsolated:
482 columns.append(self.sourceSelector.config.isolated.parentName)
483 columns.append(self.sourceSelector.config.isolated.nChildName)
484 if self.config.doSubtractLocalBackground:
485 columns.append(self.config.localBackgroundFluxField)
def fgcmMakeVisitCatalog(self, camera, groupedHandles)
def fgcmMatchStars(self, visitCat, obsCat, lutHandle=None)
def _makeSourceMapper(self, sourceSchema)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
apertureOuterInstFluxField
apertureInnerInstFluxField
doSubtractLocalBackground
apertureInnerInstFluxField
apertureOuterInstFluxField
def __init__(self, *config=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None)
def _get_sourceTable_visit_columns(self, inColumns)
def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict)
def __init__(self, initInputs=None, **kwargs)