Coverage for python/lsst/jointcal/jointcal.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import dataclasses
23import collections
24import os
26import astropy.time
27import numpy as np
28import astropy.units as u
30import lsst.geom
31import lsst.utils
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34from lsst.afw.image import fluxErrFromABMagErr
35import lsst.pex.exceptions as pexExceptions
36import lsst.afw.cameraGeom
37import lsst.afw.table
38import lsst.log
39from lsst.pipe.tasks.colorterms import ColortermLibrary
40from lsst.verify import Job, Measurement
42from lsst.meas.algorithms import (LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask,
43 ReferenceObjectLoader)
44from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
46from .dataIds import PerTractCcdDataIdContainer
48import lsst.jointcal
49from lsst.jointcal import MinimizeResult
51__all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
53Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
54Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
57# TODO: move this to MeasurementSet in lsst.verify per DM-12655.
58def add_measurement(job, name, value):
59 meas = Measurement(job.metrics[name], value)
60 job.measurements.insert(meas)
63class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
64 """Subclass of TaskRunner for jointcalTask (gen2)
66 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
67 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
68 single dataRef, are are called repeatedly). This class transforms the processed
69 arguments generated by the ArgumentParser into the arguments expected by
70 Jointcal.runDataRef().
72 See pipeBase.TaskRunner for more information.
73 """
75 @staticmethod
76 def getTargetList(parsedCmd, **kwargs):
77 """
78 Return a list of tuples per tract, each containing (dataRefs, kwargs).
80 Jointcal operates on lists of dataRefs simultaneously.
81 """
82 kwargs['butler'] = parsedCmd.butler
84 # organize data IDs by tract
85 refListDict = {}
86 for ref in parsedCmd.id.refList:
87 refListDict.setdefault(ref.dataId["tract"], []).append(ref)
88 # we call runDataRef() once with each tract
89 result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
90 return result
92 def __call__(self, args):
93 """
94 Parameters
95 ----------
96 args
97 Arguments for Task.runDataRef()
99 Returns
100 -------
101 pipe.base.Struct
102 if self.doReturnResults is False:
104 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
106 if self.doReturnResults is True:
108 - ``result``: the result of calling jointcal.runDataRef()
109 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
110 """
111 exitStatus = 0 # exit status for shell
113 # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
114 dataRefList, kwargs = args
115 butler = kwargs.pop('butler')
116 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
117 result = None
118 try:
119 result = task.runDataRef(dataRefList, **kwargs)
120 exitStatus = result.exitStatus
121 job_path = butler.get('verify_job_filename')
122 result.job.write(job_path[0])
123 except Exception as e: # catch everything, sort it out later.
124 if self.doRaise:
125 raise e
126 else:
127 exitStatus = 1
128 eName = type(e).__name__
129 tract = dataRefList[0].dataId['tract']
130 task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
132 # Put the butler back into kwargs for the other Tasks.
133 kwargs['butler'] = butler
134 if self.doReturnResults:
135 return pipeBase.Struct(result=result, exitStatus=exitStatus)
136 else:
137 return pipeBase.Struct(exitStatus=exitStatus)
140class JointcalTaskConnections(pipeBase.PipelineTaskConnections,
141 dimensions=("skymap", "tract", "instrument", "physical_filter")):
142 """Middleware input/output connections for jointcal data."""
143 inputCamera = pipeBase.connectionTypes.Input(
144 doc="The camera instrument that took these observations.",
145 name="camera",
146 storageClass="Camera",
147 dimensions=("instrument",),
148 isCalibration=True
149 )
150 inputSourceTableVisit = pipeBase.connectionTypes.Input(
151 doc="Source table in parquet format, per visit",
152 name="sourceTable_visit",
153 storageClass="DataFrame",
154 dimensions=("instrument", "visit"),
155 deferLoad=True,
156 multiple=True,
157 )
158 inputVisitSummary = pipeBase.connectionTypes.Input(
159 doc=("Per-visit consolidated exposure metadata built from calexps. "
160 "These catalogs use detector id for the id and must be sorted for "
161 "fast lookups of a detector."),
162 name="visitSummary",
163 storageClass="ExposureCatalog",
164 dimensions=("instrument", "visit"),
165 deferLoad=True,
166 multiple=True,
167 )
168 # TODO DM-28991: This does not load enough refcat data: the graph only to gets
169 # refcats that touch the tract, but we need to go larger because we're
170 # loading visits that may extend further. How to test that?
171 astrometryRefCat = pipeBase.connectionTypes.Input(
172 doc="The astrometry reference catalog to match to loaded input catalog sources.",
173 name="gaia_dr2_20200414",
174 storageClass="SimpleCatalog",
175 dimensions=("htm7",),
176 deferLoad=True,
177 multiple=True
178 )
179 photometryRefCat = pipeBase.connectionTypes.Input(
180 doc="The photometry reference catalog to match to loaded input catalog sources.",
181 name="ps1_pv3_3pi_20170110",
182 storageClass="SimpleCatalog",
183 dimensions=("htm7",),
184 deferLoad=True,
185 multiple=True
186 )
188 outputWcs = pipeBase.connectionTypes.Output(
189 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model."
190 " These catalogs only contain entries for detectors with an output, and use"
191 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
192 name="jointcalSkyWcsCatalog",
193 storageClass="ExposureCatalog",
194 dimensions=("instrument", "visit", "skymap", "tract"),
195 multiple=True
196 )
197 outputPhotoCalib = pipeBase.connectionTypes.Output(
198 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model."
199 " These catalogs only contain entries for detectors with an output, and use"
200 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
201 name="jointcalPhotoCalibCatalog",
202 storageClass="ExposureCatalog",
203 dimensions=("instrument", "visit", "skymap", "tract"),
204 multiple=True
205 )
208class JointcalConfig(pipeBase.PipelineTaskConfig,
209 pipelineConnections=JointcalTaskConnections):
210 """Configuration for JointcalTask"""
212 doAstrometry = pexConfig.Field(
213 doc="Fit astrometry and write the fitted result.",
214 dtype=bool,
215 default=True
216 )
217 doPhotometry = pexConfig.Field(
218 doc="Fit photometry and write the fitted result.",
219 dtype=bool,
220 default=True
221 )
222 coaddName = pexConfig.Field(
223 doc="Type of coadd, typically deep or goodSeeing",
224 dtype=str,
225 default="deep"
226 )
227 # TODO DM-29008: Change this to "ApFlux_12_0" before gen2 removal.
228 sourceFluxType = pexConfig.Field(
229 dtype=str,
230 doc="Source flux field to use in source selection and to get fluxes from the catalog.",
231 default='Calib'
232 )
233 positionErrorPedestal = pexConfig.Field(
234 doc="Systematic term to apply to the measured position error (pixels)",
235 dtype=float,
236 default=0.02,
237 )
238 photometryErrorPedestal = pexConfig.Field(
239 doc="Systematic term to apply to the measured error on flux or magnitude as a "
240 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
241 dtype=float,
242 default=0.0,
243 )
244 # TODO: DM-6885 matchCut should be an geom.Angle
245 matchCut = pexConfig.Field(
246 doc="Matching radius between fitted and reference stars (arcseconds)",
247 dtype=float,
248 default=3.0,
249 )
250 minMeasurements = pexConfig.Field(
251 doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
252 dtype=int,
253 default=2,
254 )
255 minMeasuredStarsPerCcd = pexConfig.Field(
256 doc="Minimum number of measuredStars per ccdImage before printing warnings",
257 dtype=int,
258 default=100,
259 )
260 minRefStarsPerCcd = pexConfig.Field(
261 doc="Minimum number of measuredStars per ccdImage before printing warnings",
262 dtype=int,
263 default=30,
264 )
265 allowLineSearch = pexConfig.Field(
266 doc="Allow a line search during minimization, if it is reasonable for the model"
267 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
268 dtype=bool,
269 default=False
270 )
271 astrometrySimpleOrder = pexConfig.Field(
272 doc="Polynomial order for fitting the simple astrometry model.",
273 dtype=int,
274 default=3,
275 )
276 astrometryChipOrder = pexConfig.Field(
277 doc="Order of the per-chip transform for the constrained astrometry model.",
278 dtype=int,
279 default=1,
280 )
281 astrometryVisitOrder = pexConfig.Field(
282 doc="Order of the per-visit transform for the constrained astrometry model.",
283 dtype=int,
284 default=5,
285 )
286 useInputWcs = pexConfig.Field(
287 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
288 dtype=bool,
289 default=True,
290 )
291 astrometryModel = pexConfig.ChoiceField(
292 doc="Type of model to fit to astrometry",
293 dtype=str,
294 default="constrained",
295 allowed={"simple": "One polynomial per ccd",
296 "constrained": "One polynomial per ccd, and one polynomial per visit"}
297 )
298 photometryModel = pexConfig.ChoiceField(
299 doc="Type of model to fit to photometry",
300 dtype=str,
301 default="constrainedMagnitude",
302 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
303 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
304 " fitting in flux space.",
305 "simpleMagnitude": "One constant zeropoint per ccd and visit,"
306 " fitting in magnitude space.",
307 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
308 " fitting in magnitude space.",
309 }
310 )
311 applyColorTerms = pexConfig.Field(
312 doc="Apply photometric color terms to reference stars?"
313 "Requires that colorterms be set to a ColortermLibrary",
314 dtype=bool,
315 default=False
316 )
317 colorterms = pexConfig.ConfigField(
318 doc="Library of photometric reference catalog name to color term dict.",
319 dtype=ColortermLibrary,
320 )
321 photometryVisitOrder = pexConfig.Field(
322 doc="Order of the per-visit polynomial transform for the constrained photometry model.",
323 dtype=int,
324 default=7,
325 )
326 photometryDoRankUpdate = pexConfig.Field(
327 doc=("Do the rank update step during minimization. "
328 "Skipping this can help deal with models that are too non-linear."),
329 dtype=bool,
330 default=True,
331 )
332 astrometryDoRankUpdate = pexConfig.Field(
333 doc=("Do the rank update step during minimization (should not change the astrometry fit). "
334 "Skipping this can help deal with models that are too non-linear."),
335 dtype=bool,
336 default=True,
337 )
338 outlierRejectSigma = pexConfig.Field(
339 doc="How many sigma to reject outliers at during minimization.",
340 dtype=float,
341 default=5.0,
342 )
343 maxPhotometrySteps = pexConfig.Field(
344 doc="Maximum number of minimize iterations to take when fitting photometry.",
345 dtype=int,
346 default=20,
347 )
348 maxAstrometrySteps = pexConfig.Field(
349 doc="Maximum number of minimize iterations to take when fitting photometry.",
350 dtype=int,
351 default=20,
352 )
353 astrometryRefObjLoader = pexConfig.ConfigurableField(
354 target=LoadIndexedReferenceObjectsTask,
355 doc="Reference object loader for astrometric fit",
356 )
357 photometryRefObjLoader = pexConfig.ConfigurableField(
358 target=LoadIndexedReferenceObjectsTask,
359 doc="Reference object loader for photometric fit",
360 )
361 sourceSelector = sourceSelectorRegistry.makeField(
362 doc="How to select sources for cross-matching",
363 default="astrometry"
364 )
365 astrometryReferenceSelector = pexConfig.ConfigurableField(
366 target=ReferenceSourceSelectorTask,
367 doc="How to down-select the loaded astrometry reference catalog.",
368 )
369 photometryReferenceSelector = pexConfig.ConfigurableField(
370 target=ReferenceSourceSelectorTask,
371 doc="How to down-select the loaded photometry reference catalog.",
372 )
373 astrometryReferenceErr = pexConfig.Field(
374 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
375 "If None, then raise an exception if the reference catalog is missing coordinate errors. "
376 "If specified, overrides any existing `coord_*Err` values."),
377 dtype=float,
378 default=None,
379 optional=True
380 )
382 # configs for outputting debug information
383 writeInitMatrix = pexConfig.Field(
384 dtype=bool,
385 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
386 "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory. "
387 "Note that these files are the dense versions of the matrix, and so may be very large."),
388 default=False
389 )
390 writeChi2FilesInitialFinal = pexConfig.Field(
391 dtype=bool,
392 doc="Write .csv files containing the contributions to chi2 for the initialization and final fit.",
393 default=False
394 )
395 writeChi2FilesOuterLoop = pexConfig.Field(
396 dtype=bool,
397 doc="Write .csv files containing the contributions to chi2 for the outer fit loop.",
398 default=False
399 )
400 writeInitialModel = pexConfig.Field(
401 dtype=bool,
402 doc=("Write the pre-initialization model to text files, for debugging."
403 " Output is written to `initial[Astro|Photo]metryModel.txt` in the current working directory."),
404 default=False
405 )
406 debugOutputPath = pexConfig.Field(
407 dtype=str,
408 default=".",
409 doc=("Path to write debug output files to. Used by "
410 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
411 )
412 detailedProfile = pexConfig.Field(
413 dtype=bool,
414 default=False,
415 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
416 )
418 def validate(self):
419 super().validate()
420 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0:
421 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
422 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
423 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms:
424 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;"
425 "applyColorTerms=True will be ignored.")
426 lsst.log.warn(msg)
428 def setDefaults(self):
429 # Use science source selector which can filter on extendedness, SNR, and whether blended
430 self.sourceSelector.name = 'science'
431 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing
432 self.sourceSelector['science'].doUnresolved = True
433 # with dependable signal to noise ratio.
434 self.sourceSelector['science'].doSignalToNoise = True
435 # Min SNR must be > 0 because jointcal cannot handle negative fluxes,
436 # and S/N > 10 to use sources that are not too faint, and thus better measured.
437 self.sourceSelector['science'].signalToNoise.minimum = 10.
438 # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive
439 fluxField = f"slot_{self.sourceFluxType}Flux_instFlux"
440 self.sourceSelector['science'].signalToNoise.fluxField = fluxField
441 self.sourceSelector['science'].signalToNoise.errField = fluxField + "Err"
442 # Do not trust blended sources' aperture fluxes which also depend on seeing
443 self.sourceSelector['science'].doIsolated = True
444 # Do not trust either flux or centroid measurements with flags,
445 # chosen from the usual QA flags for stars)
446 self.sourceSelector['science'].doFlags = True
447 badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated',
448 'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag',
449 'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter']
450 self.sourceSelector['science'].flags.bad = badFlags
452 # Default to Gaia-DR2 (with proper motions) for astrometry and
453 # PS1-DR1 for photometry, with a reasonable initial filterMap.
454 self.astrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
455 self.astrometryRefObjLoader.requireProperMotion = True
456 self.astrometryRefObjLoader.anyFilterMapsToThis = 'phot_g_mean'
457 self.photometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110"
460def writeModel(model, filename, log):
461 """Write model to outfile."""
462 with open(filename, "w") as file:
463 file.write(repr(model))
464 log.info("Wrote %s to file: %s", model, filename)
467@dataclasses.dataclass
468class JointcalInputData:
469 """The input data jointcal needs for each detector/visit."""
470 visit: int
471 """The visit identifier of this exposure."""
472 catalog: lsst.afw.table.SourceCatalog
473 """The catalog derived from this exposure."""
474 visitInfo: lsst.afw.image.VisitInfo
475 """The VisitInfo of this exposure."""
476 detector: lsst.afw.cameraGeom.Detector
477 """The detector of this exposure."""
478 photoCalib: lsst.afw.image.PhotoCalib
479 """The photometric calibration of this exposure."""
480 wcs: lsst.afw.geom.skyWcs
481 """The WCS of this exposure."""
482 bbox: lsst.geom.Box2I
483 """The bounding box of this exposure."""
484 filter: lsst.afw.image.FilterLabel
485 """The filter of this exposure."""
488class JointcalTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
489 """Astrometricly and photometricly calibrate across multiple visits of the
490 same field.
492 Parameters
493 ----------
494 butler : `lsst.daf.persistence.Butler`
495 The butler is passed to the refObjLoader constructor in case it is
496 needed. Ignored if the refObjLoader argument provides a loader directly.
497 Used to initialize the astrometry and photometry refObjLoaders.
498 initInputs : `dict`, optional
499 Dictionary used to initialize PipelineTasks (empty for jointcal).
500 """
502 ConfigClass = JointcalConfig
503 RunnerClass = JointcalRunner
504 _DefaultName = "jointcal"
506 def __init__(self, butler=None, initInputs=None, **kwargs):
507 super().__init__(**kwargs)
508 self.makeSubtask("sourceSelector")
509 if self.config.doAstrometry:
510 if initInputs is None:
511 # gen3 middleware does refcat things internally (and will not have a butler here)
512 self.makeSubtask('astrometryRefObjLoader', butler=butler)
513 self.makeSubtask("astrometryReferenceSelector")
514 else:
515 self.astrometryRefObjLoader = None
516 if self.config.doPhotometry:
517 if initInputs is None:
518 # gen3 middleware does refcat things internally (and will not have a butler here)
519 self.makeSubtask('photometryRefObjLoader', butler=butler)
520 self.makeSubtask("photometryReferenceSelector")
521 else:
522 self.photometryRefObjLoader = None
524 # To hold various computed metrics for use by tests
525 self.job = Job.load_metrics_package(subset='jointcal')
527 def runQuantum(self, butlerQC, inputRefs, outputRefs):
528 # We override runQuantum to set up the refObjLoaders and write the
529 # outputs to the correct refs.
530 inputs = butlerQC.get(inputRefs)
531 # We want the tract number for writing debug files
532 tract = butlerQC.quantum.dataId['tract']
533 if self.config.doAstrometry:
534 self.astrometryRefObjLoader = ReferenceObjectLoader(
535 dataIds=[ref.datasetRef.dataId
536 for ref in inputRefs.astrometryRefCat],
537 refCats=inputs.pop('astrometryRefCat'),
538 config=self.config.astrometryRefObjLoader,
539 log=self.log)
540 if self.config.doPhotometry:
541 self.photometryRefObjLoader = ReferenceObjectLoader(
542 dataIds=[ref.datasetRef.dataId
543 for ref in inputRefs.photometryRefCat],
544 refCats=inputs.pop('photometryRefCat'),
545 config=self.config.photometryRefObjLoader,
546 log=self.log)
547 outputs = self.run(**inputs, tract=tract)
548 if self.config.doAstrometry:
549 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
550 inputs['inputCamera'], "setWcs")
551 if self.config.doPhotometry:
552 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
553 inputs['inputCamera'], "setPhotoCalib")
555 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
556 """Persist the output datasets to their appropriate datarefs.
558 Parameters
559 ----------
560 butlerQC : `ButlerQuantumContext`
561 A butler which is specialized to operate in the context of a
562 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
563 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or
564 `dict` [`tuple, `lsst.afw.image.PhotoCalib`]
565 The fitted objects to persist.
566 outputRefs : `list` [`OutputQuantizedConnection`]
567 The DatasetRefs to persist the data to.
568 camera : `lsst.afw.cameraGeom.Camera`
569 The camera for this instrument, to get detector ids from.
570 setter : `str`
571 The method to call on the ExposureCatalog to set each output.
572 """
573 schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
574 schema.addField('visit', type='I', doc='Visit number')
576 def new_catalog(visit, size):
577 """Return an catalog ready to be filled with appropriate output."""
578 catalog = lsst.afw.table.ExposureCatalog(schema)
579 catalog.resize(size)
580 catalog['visit'] = visit
581 metadata = lsst.daf.base.PropertyList()
582 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
583 metadata.add("COMMENT", "Only detectors with data have entries.")
584 return catalog
586 # count how many detectors have output for each visit
587 detectors_per_visit = collections.defaultdict(int)
588 for key in outputs:
589 # key is (visit, detector_id), and we only need visit here
590 detectors_per_visit[key[0]] += 1
592 for ref in outputRefs:
593 visit = ref.dataId['visit']
594 catalog = new_catalog(visit, detectors_per_visit[visit])
595 # Iterate over every detector and skip the ones we don't have output for.
596 i = 0
597 for detector in camera:
598 detectorId = detector.getId()
599 key = (ref.dataId['visit'], detectorId)
600 if key not in outputs:
601 # skip detectors we don't have output for
602 self.log.debug("No %s output for detector %s in visit %s",
603 setter[3:], detectorId, visit)
604 continue
606 catalog[i].setId(detectorId)
607 getattr(catalog[i], setter)(outputs[key])
608 i += 1
610 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups
611 butlerQC.put(catalog, ref)
612 self.log.info("Wrote %s detectors to %s", i, ref)
614 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
615 # Docstring inherited.
617 # We take values out of the Parquet table, and put them in "flux_",
618 # and the config.sourceFluxType field is used during that extraction,
619 # so just use "flux" here.
620 sourceFluxField = "flux"
621 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
622 associations = lsst.jointcal.Associations()
623 self.focalPlaneBBox = inputCamera.getFpBBox()
624 oldWcsList, bands = self._load_data(inputSourceTableVisit,
625 inputVisitSummary,
626 associations,
627 jointcalControl,
628 inputCamera)
630 boundingCircle, center, radius, defaultFilter = self._prep_sky(associations, bands)
631 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList())
633 if self.config.doAstrometry:
634 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
635 name="astrometry",
636 refObjLoader=self.astrometryRefObjLoader,
637 referenceSelector=self.astrometryReferenceSelector,
638 fit_function=self._fit_astrometry,
639 tract=tract,
640 epoch=epoch)
641 astrometry_output = self._make_output(associations.getCcdImageList(),
642 astrometry.model,
643 "makeSkyWcs")
644 else:
645 astrometry_output = None
647 if self.config.doPhotometry:
648 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
649 name="photometry",
650 refObjLoader=self.photometryRefObjLoader,
651 referenceSelector=self.photometryReferenceSelector,
652 fit_function=self._fit_photometry,
653 tract=tract,
654 epoch=epoch,
655 reject_bad_fluxes=True)
656 photometry_output = self._make_output(associations.getCcdImageList(),
657 photometry.model,
658 "toPhotoCalib")
659 else:
660 photometry_output = None
662 return pipeBase.Struct(outputWcs=astrometry_output,
663 outputPhotoCalib=photometry_output,
664 job=self.job,
665 astrometryRefObjLoader=self.astrometryRefObjLoader,
666 photometryRefObjLoader=self.photometryRefObjLoader)
668 def _make_schema_table(self):
669 """Return an afw SourceTable to use as a base for creating the
670 SourceCatalog to insert values from the dataFrame into.
672 Returns
673 -------
674 table : `lsst.afw.table.SourceTable`
675 Table with schema and slots to use to make SourceCatalogs.
676 """
677 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
678 schema.addField("centroid_x", "D")
679 schema.addField("centroid_y", "D")
680 schema.addField("centroid_xErr", "F")
681 schema.addField("centroid_yErr", "F")
682 schema.addField("shape_xx", "D")
683 schema.addField("shape_yy", "D")
684 schema.addField("shape_xy", "D")
685 schema.addField("flux_instFlux", "D")
686 schema.addField("flux_instFluxErr", "D")
687 table = lsst.afw.table.SourceTable.make(schema)
688 table.defineCentroid("centroid")
689 table.defineShape("shape")
690 return table
692 def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId):
693 """Return an afw SourceCatalog extracted from a visit-level dataframe,
694 limited to just one detector.
696 Parameters
697 ----------
698 table : `lsst.afw.table.SourceTable`
699 Table factory to use to make the SourceCatalog that will be
700 populated with data from ``visitCatalog``.
701 visitCatalog : `pandas.DataFrame`
702 DataFrame to extract a detector catalog from.
703 detectorId : `int`
704 Numeric id of the detector to extract from ``visitCatalog``.
706 Returns
707 -------
708 catalog : `lsst.afw.table.SourceCatalog`
709 Detector-level catalog extracted from ``visitCatalog``.
710 """
711 # map from dataFrame column to afw table column
712 mapping = {'sourceId': 'id',
713 'x': 'centroid_x',
714 'y': 'centroid_y',
715 'xErr': 'centroid_xErr',
716 'yErr': 'centroid_yErr',
717 'Ixx': 'shape_xx',
718 'Iyy': 'shape_yy',
719 'Ixy': 'shape_xy',
720 f'{self.config.sourceFluxType}_instFlux': 'flux_instFlux',
721 f'{self.config.sourceFluxType}_instFluxErr': 'flux_instFluxErr',
722 }
723 catalog = lsst.afw.table.SourceCatalog(table)
724 matched = visitCatalog['ccd'] == detectorId
725 catalog.resize(sum(matched))
726 view = visitCatalog.loc[matched]
727 for dfCol, afwCol in mapping.items():
728 catalog[afwCol] = view[dfCol]
730 self.log.debug("%d sources selected in visit %d detector %d",
731 len(catalog),
732 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first
733 detectorId)
734 return catalog
736 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
737 jointcalControl, camera):
738 """Read the data that jointcal needs to run. (Gen3 version)
740 Modifies ``associations`` in-place with the loaded data.
742 Parameters
743 ----------
744 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
745 References to visit-level DataFrames to load the catalog data from.
746 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
747 Visit-level exposure summary catalog with metadata.
748 associations : `lsst.jointcal.Associations`
749 Object to add the loaded data to by constructing new CcdImages.
750 jointcalControl : `jointcal.JointcalControl`
751 Control object for C++ associations management.
752 camera : `lsst.afw.cameraGeom.Camera`
753 Camera object for detector geometry.
755 Returns
756 -------
757 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`]
758 The original WCS of the input data, to aid in writing tests.
759 bands : `list` [`str`]
760 The filter bands of each input dataset.
761 """
762 oldWcsList = []
763 filters = []
764 load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile else ''
765 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
766 table = self._make_schema_table() # every detector catalog has the same layout
767 # No guarantee that the input is in the same order of visits, so we have to map one of them.
768 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)}
769 detectorDict = {detector.getId(): detector for detector in camera}
771 for visitSummaryRef in inputVisitSummary:
772 visitSummary = visitSummaryRef.get()
773 visitCatalog = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]].get()
774 selected = self.sourceSelector.run(visitCatalog)
776 # Build a CcdImage for each detector in this visit.
777 detectors = {id: index for index, id in enumerate(visitSummary['id'])}
778 for id, index in detectors.items():
779 catalog = self._extract_detector_catalog_from_visit_catalog(table, selected.sourceCat, id)
780 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict)
781 result = self._build_ccdImage(data, associations, jointcalControl)
782 if result is not None:
783 oldWcsList.append(result.wcs)
784 # A visit has only one band, so we can just use the first.
785 filters.append(data.filter)
786 if len(filters) == 0:
787 raise RuntimeError("No data to process: did source selector remove all sources?")
788 filters = collections.Counter(filters)
790 return oldWcsList, filters
792 def _make_one_input_data(self, visitRecord, catalog, detectorDict):
793 """Return a data structure for this detector+visit."""
794 return JointcalInputData(visit=visitRecord['visit'],
795 catalog=catalog,
796 visitInfo=visitRecord.getVisitInfo(),
797 detector=detectorDict[visitRecord.getId()],
798 photoCalib=visitRecord.getPhotoCalib(),
799 wcs=visitRecord.getWcs(),
800 bbox=visitRecord.getBBox(),
801 # ExposureRecord doesn't have a FilterLabel yet,
802 # so we have to make one.
803 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'],
804 physical=visitRecord['physical_filter']))
806 # We don't currently need to persist the metadata.
807 # If we do in the future, we will have to add appropriate dataset templates
808 # to each obs package (the metadata template should look like `jointcal_wcs`).
809 def _getMetadataName(self):
810 return None
812 @classmethod
813 def _makeArgumentParser(cls):
814 """Create an argument parser"""
815 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
816 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
817 ContainerClass=PerTractCcdDataIdContainer)
818 return parser
820 def _build_ccdImage(self, data, associations, jointcalControl):
821 """
822 Extract the necessary things from this catalog+metadata to add a new
823 ccdImage.
825 Parameters
826 ----------
827 data : `JointcalInputData`
828 The loaded input data.
829 associations : `lsst.jointcal.Associations`
830 Object to add the info to, to construct a new CcdImage
831 jointcalControl : `jointcal.JointcalControl`
832 Control object for associations management
834 Returns
835 ------
836 namedtuple or `None`
837 ``wcs``
838 The TAN WCS of this image, read from the calexp
839 (`lsst.afw.geom.SkyWcs`).
840 ``key``
841 A key to identify this dataRef by its visit and ccd ids
842 (`namedtuple`).
843 `None`
844 if there are no sources in the loaded catalog.
845 """
846 if len(data.catalog) == 0:
847 self.log.warn("No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
848 return None
850 associations.createCcdImage(data.catalog,
851 data.wcs,
852 data.visitInfo,
853 data.bbox,
854 data.filter.physicalLabel,
855 data.photoCalib,
856 data.detector,
857 data.visit,
858 data.detector.getId(),
859 jointcalControl)
861 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key'))
862 Key = collections.namedtuple('Key', ('visit', 'ccd'))
863 return Result(data.wcs, Key(data.visit, data.detector.getId()))
865 def _readDataId(self, butler, dataId):
866 """Read all of the data for one dataId from the butler. (gen2 version)"""
867 # Not all instruments have `visit` in their dataIds.
868 if "visit" in dataId.keys():
869 visit = dataId["visit"]
870 else:
871 visit = butler.getButler().queryMetadata("calexp", ("visit"), butler.dataId)[0]
872 detector = butler.get('calexp_detector', dataId=dataId)
874 catalog = butler.get('src',
875 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
876 dataId=dataId)
877 goodSrc = self.sourceSelector.run(catalog)
878 self.log.debug("%d sources selected in visit %d detector %d",
879 len(goodSrc.sourceCat),
880 visit,
881 detector.getId())
882 return JointcalInputData(visit=visit,
883 catalog=goodSrc.sourceCat,
884 visitInfo=butler.get('calexp_visitInfo', dataId=dataId),
885 detector=detector,
886 photoCalib=butler.get('calexp_photoCalib', dataId=dataId),
887 wcs=butler.get('calexp_wcs', dataId=dataId),
888 bbox=butler.get('calexp_bbox', dataId=dataId),
889 filter=butler.get('calexp_filterLabel', dataId=dataId))
891 def loadData(self, dataRefs, associations, jointcalControl):
892 """Read the data that jointcal needs to run. (Gen2 version)"""
893 visit_ccd_to_dataRef = {}
894 oldWcsList = []
895 filters = []
896 load_cat_prof_file = 'jointcal_loadData.prof' if self.config.detailedProfile else ''
897 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
898 # Need the bounding-box of the focal plane (the same for all visits) for photometry visit models
899 camera = dataRefs[0].get('camera', immediate=True)
900 self.focalPlaneBBox = camera.getFpBBox()
901 for dataRef in dataRefs:
902 data = self._readDataId(dataRef.getButler(), dataRef.dataId)
903 result = self._build_ccdImage(data, associations, jointcalControl)
904 if result is None:
905 continue
906 oldWcsList.append(result.wcs)
907 visit_ccd_to_dataRef[result.key] = dataRef
908 filters.append(data.filter)
909 if len(filters) == 0:
910 raise RuntimeError("No data to process: did source selector remove all sources?")
911 filters = collections.Counter(filters)
913 return oldWcsList, filters, visit_ccd_to_dataRef
915 def _getDebugPath(self, filename):
916 """Constructs a path to filename using the configured debug path.
917 """
918 return os.path.join(self.config.debugOutputPath, filename)
920 def _prep_sky(self, associations, filters):
921 """Prepare on-sky and other data that must be computed after data has
922 been read.
923 """
924 associations.computeCommonTangentPoint()
926 boundingCircle = associations.computeBoundingCircle()
927 center = lsst.geom.SpherePoint(boundingCircle.getCenter())
928 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
930 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.")
932 # Determine a default filter band associated with the catalog. See DM-9093
933 defaultFilter = filters.most_common(1)[0][0]
934 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel)
936 return boundingCircle, center, radius, defaultFilter
938 @pipeBase.timeMethod
939 def runDataRef(self, dataRefs):
940 """
941 Jointly calibrate the astrometry and photometry across a set of images.
943 NOTE: this is for gen2 middleware only.
945 Parameters
946 ----------
947 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
948 List of data references to the exposures to be fit.
950 Returns
951 -------
952 result : `lsst.pipe.base.Struct`
953 Struct of metadata from the fit, containing:
955 ``dataRefs``
956 The provided data references that were fit (with updated WCSs)
957 ``oldWcsList``
958 The original WCS from each dataRef
959 ``metrics``
960 Dictionary of internally-computed metrics for testing/validation.
961 """
962 if len(dataRefs) == 0:
963 raise ValueError('Need a non-empty list of data references!')
965 exitStatus = 0 # exit status for shell
967 sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
968 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
969 associations = lsst.jointcal.Associations()
971 oldWcsList, filters, visit_ccd_to_dataRef = self.loadData(dataRefs,
972 associations,
973 jointcalControl)
975 boundingCircle, center, radius, defaultFilter = self._prep_sky(associations, filters)
976 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList())
978 tract = dataRefs[0].dataId['tract']
980 if self.config.doAstrometry:
981 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
982 name="astrometry",
983 refObjLoader=self.astrometryRefObjLoader,
984 referenceSelector=self.astrometryReferenceSelector,
985 fit_function=self._fit_astrometry,
986 tract=tract,
987 epoch=epoch)
988 self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
989 else:
990 astrometry = Astrometry(None, None, None)
992 if self.config.doPhotometry:
993 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
994 name="photometry",
995 refObjLoader=self.photometryRefObjLoader,
996 referenceSelector=self.photometryReferenceSelector,
997 fit_function=self._fit_photometry,
998 tract=tract,
999 epoch=epoch,
1000 reject_bad_fluxes=True)
1001 self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
1002 else:
1003 photometry = Photometry(None, None)
1005 return pipeBase.Struct(dataRefs=dataRefs,
1006 oldWcsList=oldWcsList,
1007 job=self.job,
1008 astrometryRefObjLoader=self.astrometryRefObjLoader,
1009 photometryRefObjLoader=self.photometryRefObjLoader,
1010 defaultFilter=defaultFilter,
1011 epoch=epoch,
1012 exitStatus=exitStatus)
1014 def _get_refcat_coordinate_error_override(self, refCat, name):
1015 """Check whether we should override the refcat coordinate errors, and
1016 return the overridden error if necessary.
1018 Parameters
1019 ----------
1020 refCat : `lsst.afw.table.SimpleCatalog`
1021 The reference catalog to check for a ``coord_raErr`` field.
1022 name : `str`
1023 Whether we are doing "astrometry" or "photometry".
1025 Returns
1026 -------
1027 refCoordErr : `float`
1028 The refcat coordinate error to use, or NaN if we are not overriding
1029 those fields.
1031 Raises
1032 ------
1033 lsst.pex.config.FieldValidationError
1034 Raised if the refcat does not contain coordinate errors and
1035 ``config.astrometryReferenceErr`` is not set.
1036 """
1037 # This value doesn't matter for photometry, so just set something to
1038 # keep old refcats from causing problems.
1039 if name.lower() == "photometry":
1040 if 'coord_raErr' not in refCat.schema:
1041 return 100
1042 else:
1043 return float('nan')
1045 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema:
1046 msg = ("Reference catalog does not contain coordinate errors, "
1047 "and config.astrometryReferenceErr not supplied.")
1048 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
1049 self.config,
1050 msg)
1052 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema:
1053 self.log.warn("Overriding reference catalog coordinate errors with %f/coordinate [mas]",
1054 self.config.astrometryReferenceErr)
1056 if self.config.astrometryReferenceErr is None:
1057 return float('nan')
1058 else:
1059 return self.config.astrometryReferenceErr
1061 def _compute_proper_motion_epoch(self, ccdImageList):
1062 """Return the proper motion correction epoch of the provided images.
1064 Parameters
1065 ----------
1066 ccdImageList : `list` [`lsst.jointcal.CcdImage`]
1067 The images to compute the appropriate epoch for.
1069 Returns
1070 -------
1071 epoch : `astropy.time.Time`
1072 The date to use for proper motion corrections.
1073 """
1074 mjds = [ccdImage.getMjd() for ccdImage in ccdImageList]
1075 return astropy.time.Time(np.mean(mjds), format='mjd', scale="tai")
1077 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
1078 tract="", match_cut=3.0,
1079 reject_bad_fluxes=False, *,
1080 name="", refObjLoader=None, referenceSelector=None,
1081 fit_function=None, epoch=None):
1082 """Load reference catalog, perform the fit, and return the result.
1084 Parameters
1085 ----------
1086 associations : `lsst.jointcal.Associations`
1087 The star/reference star associations to fit.
1088 defaultFilter : `lsst.afw.image.FilterLabel`
1089 filter to load from reference catalog.
1090 center : `lsst.geom.SpherePoint`
1091 ICRS center of field to load from reference catalog.
1092 radius : `lsst.geom.Angle`
1093 On-sky radius to load from reference catalog.
1094 name : `str`
1095 Name of thing being fit: "astrometry" or "photometry".
1096 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1097 Reference object loader to use to load a reference catalog.
1098 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1099 Selector to use to pick objects from the loaded reference catalog.
1100 fit_function : callable
1101 Function to call to perform fit (takes Associations object).
1102 tract : `str`, optional
1103 Name of tract currently being fit.
1104 match_cut : `float`, optional
1105 Radius in arcseconds to find cross-catalog matches to during
1106 associations.associateCatalogs.
1107 reject_bad_fluxes : `bool`, optional
1108 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
1109 epoch : `astropy.time.Time`, optional
1110 Epoch to which to correct refcat proper motion and parallax,
1111 or `None` to not apply such corrections.
1113 Returns
1114 -------
1115 result : `Photometry` or `Astrometry`
1116 Result of `fit_function()`
1117 """
1118 self.log.info("====== Now processing %s...", name)
1119 # TODO: this should not print "trying to invert a singular transformation:"
1120 # if it does that, something's not right about the WCS...
1121 associations.associateCatalogs(match_cut)
1122 add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
1123 associations.fittedStarListSize())
1125 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms
1126 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector,
1127 center, radius, defaultFilter,
1128 applyColorterms=applyColorterms,
1129 epoch=epoch)
1130 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name)
1132 associations.collectRefStars(refCat,
1133 self.config.matchCut*lsst.geom.arcseconds,
1134 fluxField,
1135 refCoordinateErr=refCoordErr,
1136 rejectBadFluxes=reject_bad_fluxes)
1137 add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
1138 associations.refStarListSize())
1140 associations.prepareFittedStars(self.config.minMeasurements)
1142 self._check_star_lists(associations, name)
1143 add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
1144 associations.nFittedStarsWithAssociatedRefStar())
1145 add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
1146 associations.fittedStarListSize())
1147 add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
1148 associations.nCcdImagesValidForFit())
1150 load_cat_prof_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else ''
1151 dataName = "{}_{}".format(tract, defaultFilter.bandLabel)
1152 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1153 result = fit_function(associations, dataName)
1154 # TODO DM-12446: turn this into a "butler save" somehow.
1155 # Save reference and measurement chi2 contributions for this data
1156 if self.config.writeChi2FilesInitialFinal:
1157 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}")
1158 result.fit.saveChi2Contributions(baseName+"{type}")
1159 self.log.info("Wrote chi2 contributions files: %s", baseName)
1161 return result
1163 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1164 applyColorterms=False, epoch=None):
1165 """Load the necessary reference catalog sources, convert fluxes to
1166 correct units, and apply color term corrections if requested.
1168 Parameters
1169 ----------
1170 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1171 The reference catalog loader to use to get the data.
1172 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1173 Source selector to apply to loaded reference catalog.
1174 center : `lsst.geom.SpherePoint`
1175 The center around which to load sources.
1176 radius : `lsst.geom.Angle`
1177 The radius around ``center`` to load sources in.
1178 filterLabel : `lsst.afw.image.FilterLabel`
1179 The camera filter to load fluxes for.
1180 applyColorterms : `bool`
1181 Apply colorterm corrections to the refcat for ``filterName``?
1182 epoch : `astropy.time.Time`, optional
1183 Epoch to which to correct refcat proper motion and parallax,
1184 or `None` to not apply such corrections.
1186 Returns
1187 -------
1188 refCat : `lsst.afw.table.SimpleCatalog`
1189 The loaded reference catalog.
1190 fluxField : `str`
1191 The name of the reference catalog flux field appropriate for ``filterName``.
1192 """
1193 skyCircle = refObjLoader.loadSkyCircle(center,
1194 radius,
1195 filterLabel.bandLabel,
1196 epoch=epoch)
1198 selected = referenceSelector.run(skyCircle.refCat)
1199 # Need memory contiguity to get reference filters as a vector.
1200 if not selected.sourceCat.isContiguous():
1201 refCat = selected.sourceCat.copy(deep=True)
1202 else:
1203 refCat = selected.sourceCat
1205 if applyColorterms:
1206 refCatName = refObjLoader.ref_dataset_name
1207 self.log.info("Applying color terms for physical filter=%r reference catalog=%s",
1208 filterLabel.physicalLabel, refCatName)
1209 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1210 refCatName,
1211 doRaise=True)
1213 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1214 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1215 # TODO: I didn't want to use this, but I'll deal with it in DM-16903
1216 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1218 return refCat, skyCircle.fluxField
1220 def _check_star_lists(self, associations, name):
1221 # TODO: these should be len(blah), but we need this properly wrapped first.
1222 if associations.nCcdImagesValidForFit() == 0:
1223 raise RuntimeError('No images in the ccdImageList!')
1224 if associations.fittedStarListSize() == 0:
1225 raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
1226 if associations.refStarListSize() == 0:
1227 raise RuntimeError('No stars in the {} reference star list!'.format(name))
1229 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1230 """Compute chi2, log it, validate the model, and return chi2.
1232 Parameters
1233 ----------
1234 associations : `lsst.jointcal.Associations`
1235 The star/reference star associations to fit.
1236 fit : `lsst.jointcal.FitterBase`
1237 The fitter to use for minimization.
1238 model : `lsst.jointcal.Model`
1239 The model being fit.
1240 chi2Label : `str`
1241 Label to describe the chi2 (e.g. "Initialized", "Final").
1242 writeChi2Name : `str`, optional
1243 Filename prefix to write the chi2 contributions to.
1244 Do not supply an extension: an appropriate one will be added.
1246 Returns
1247 -------
1248 chi2: `lsst.jointcal.Chi2Accumulator`
1249 The chi2 object for the current fitter and model.
1251 Raises
1252 ------
1253 FloatingPointError
1254 Raised if chi2 is infinite or NaN.
1255 ValueError
1256 Raised if the model is not valid.
1257 """
1258 if writeChi2Name is not None:
1259 fullpath = self._getDebugPath(writeChi2Name)
1260 fit.saveChi2Contributions(fullpath+"{type}")
1261 self.log.info("Wrote chi2 contributions files: %s", fullpath)
1263 chi2 = fit.computeChi2()
1264 self.log.info("%s %s", chi2Label, chi2)
1265 self._check_stars(associations)
1266 if not np.isfinite(chi2.chi2):
1267 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}')
1268 if not model.validate(associations.getCcdImageList(), chi2.ndof):
1269 raise ValueError("Model is not valid: check log messages for warnings.")
1270 return chi2
1272 def _fit_photometry(self, associations, dataName=None):
1273 """
1274 Fit the photometric data.
1276 Parameters
1277 ----------
1278 associations : `lsst.jointcal.Associations`
1279 The star/reference star associations to fit.
1280 dataName : `str`
1281 Name of the data being processed (e.g. "1234_HSC-Y"), for
1282 identifying debugging files.
1284 Returns
1285 -------
1286 fit_result : `namedtuple`
1287 fit : `lsst.jointcal.PhotometryFit`
1288 The photometric fitter used to perform the fit.
1289 model : `lsst.jointcal.PhotometryModel`
1290 The photometric model that was fit.
1291 """
1292 self.log.info("=== Starting photometric fitting...")
1294 # TODO: should use pex.config.RegistryField here (see DM-9195)
1295 if self.config.photometryModel == "constrainedFlux":
1296 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
1297 self.focalPlaneBBox,
1298 visitOrder=self.config.photometryVisitOrder,
1299 errorPedestal=self.config.photometryErrorPedestal)
1300 # potentially nonlinear problem, so we may need a line search to converge.
1301 doLineSearch = self.config.allowLineSearch
1302 elif self.config.photometryModel == "constrainedMagnitude":
1303 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
1304 self.focalPlaneBBox,
1305 visitOrder=self.config.photometryVisitOrder,
1306 errorPedestal=self.config.photometryErrorPedestal)
1307 # potentially nonlinear problem, so we may need a line search to converge.
1308 doLineSearch = self.config.allowLineSearch
1309 elif self.config.photometryModel == "simpleFlux":
1310 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
1311 errorPedestal=self.config.photometryErrorPedestal)
1312 doLineSearch = False # purely linear in model parameters, so no line search needed
1313 elif self.config.photometryModel == "simpleMagnitude":
1314 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
1315 errorPedestal=self.config.photometryErrorPedestal)
1316 doLineSearch = False # purely linear in model parameters, so no line search needed
1318 fit = lsst.jointcal.PhotometryFit(associations, model)
1319 # TODO DM-12446: turn this into a "butler save" somehow.
1320 # Save reference and measurement chi2 contributions for this data
1321 if self.config.writeChi2FilesInitialFinal:
1322 baseName = f"photometry_initial_chi2-{dataName}"
1323 else:
1324 baseName = None
1325 if self.config.writeInitialModel:
1326 fullpath = self._getDebugPath("initialPhotometryModel.txt")
1327 writeModel(model, fullpath, self.log)
1328 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
1330 def getChi2Name(whatToFit):
1331 if self.config.writeChi2FilesOuterLoop:
1332 return f"photometry_init-%s_chi2-{dataName}" % whatToFit
1333 else:
1334 return None
1336 # The constrained model needs the visit transform fit first; the chip
1337 # transform is initialized from the singleFrame PhotoCalib, so it's close.
1338 dumpMatrixFile = self._getDebugPath("photometry_preinit") if self.config.writeInitMatrix else ""
1339 if self.config.photometryModel.startswith("constrained"):
1340 # no line search: should be purely (or nearly) linear,
1341 # and we want a large step size to initialize with.
1342 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
1343 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit",
1344 writeChi2Name=getChi2Name("ModelVisit"))
1345 dumpMatrixFile = "" # so we don't redo the output on the next step
1347 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1348 self._logChi2AndValidate(associations, fit, model, "Initialize Model",
1349 writeChi2Name=getChi2Name("Model"))
1351 fit.minimize("Fluxes") # no line search: always purely linear.
1352 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes",
1353 writeChi2Name=getChi2Name("Fluxes"))
1355 fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
1356 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes",
1357 writeChi2Name=getChi2Name("ModelFluxes"))
1359 model.freezeErrorTransform()
1360 self.log.debug("Photometry error scales are frozen.")
1362 chi2 = self._iterate_fit(associations,
1363 fit,
1364 self.config.maxPhotometrySteps,
1365 "photometry",
1366 "Model Fluxes",
1367 doRankUpdate=self.config.photometryDoRankUpdate,
1368 doLineSearch=doLineSearch,
1369 dataName=dataName)
1371 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
1372 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
1373 return Photometry(fit, model)
1375 def _fit_astrometry(self, associations, dataName=None):
1376 """
1377 Fit the astrometric data.
1379 Parameters
1380 ----------
1381 associations : `lsst.jointcal.Associations`
1382 The star/reference star associations to fit.
1383 dataName : `str`
1384 Name of the data being processed (e.g. "1234_HSC-Y"), for
1385 identifying debugging files.
1387 Returns
1388 -------
1389 fit_result : `namedtuple`
1390 fit : `lsst.jointcal.AstrometryFit`
1391 The astrometric fitter used to perform the fit.
1392 model : `lsst.jointcal.AstrometryModel`
1393 The astrometric model that was fit.
1394 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
1395 The model for the sky to tangent plane projection that was used in the fit.
1396 """
1398 self.log.info("=== Starting astrometric fitting...")
1400 associations.deprojectFittedStars()
1402 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
1403 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
1404 # them so carefully?
1405 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
1407 if self.config.astrometryModel == "constrained":
1408 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
1409 sky_to_tan_projection,
1410 chipOrder=self.config.astrometryChipOrder,
1411 visitOrder=self.config.astrometryVisitOrder)
1412 elif self.config.astrometryModel == "simple":
1413 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
1414 sky_to_tan_projection,
1415 self.config.useInputWcs,
1416 nNotFit=0,
1417 order=self.config.astrometrySimpleOrder)
1419 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
1420 # TODO DM-12446: turn this into a "butler save" somehow.
1421 # Save reference and measurement chi2 contributions for this data
1422 if self.config.writeChi2FilesInitialFinal:
1423 baseName = f"astrometry_initial_chi2-{dataName}"
1424 else:
1425 baseName = None
1426 if self.config.writeInitialModel:
1427 fullpath = self._getDebugPath("initialAstrometryModel.txt")
1428 writeModel(model, fullpath, self.log)
1429 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
1431 def getChi2Name(whatToFit):
1432 if self.config.writeChi2FilesOuterLoop:
1433 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
1434 else:
1435 return None
1437 dumpMatrixFile = self._getDebugPath("astrometry_preinit") if self.config.writeInitMatrix else ""
1438 # The constrained model needs the visit transform fit first; the chip
1439 # transform is initialized from the detector's cameraGeom, so it's close.
1440 if self.config.astrometryModel == "constrained":
1441 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1442 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit",
1443 writeChi2Name=getChi2Name("DistortionsVisit"))
1444 dumpMatrixFile = "" # so we don't redo the output on the next step
1446 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
1447 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions",
1448 writeChi2Name=getChi2Name("Distortions"))
1450 fit.minimize("Positions")
1451 self._logChi2AndValidate(associations, fit, model, "Initialize Positions",
1452 writeChi2Name=getChi2Name("Positions"))
1454 fit.minimize("Distortions Positions")
1455 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions",
1456 writeChi2Name=getChi2Name("DistortionsPositions"))
1458 chi2 = self._iterate_fit(associations,
1459 fit,
1460 self.config.maxAstrometrySteps,
1461 "astrometry",
1462 "Distortions Positions",
1463 doRankUpdate=self.config.astrometryDoRankUpdate,
1464 dataName=dataName)
1466 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
1467 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
1469 return Astrometry(fit, model, sky_to_tan_projection)
1471 def _check_stars(self, associations):
1472 """Count measured and reference stars per ccd and warn/log them."""
1473 for ccdImage in associations.getCcdImageList():
1474 nMeasuredStars, nRefStars = ccdImage.countStars()
1475 self.log.debug("ccdImage %s has %s measured and %s reference stars",
1476 ccdImage.getName(), nMeasuredStars, nRefStars)
1477 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1478 self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
1479 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1480 if nRefStars < self.config.minRefStarsPerCcd:
1481 self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
1482 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1484 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1485 dataName="",
1486 doRankUpdate=True,
1487 doLineSearch=False):
1488 """Run fitter.minimize up to max_steps times, returning the final chi2.
1490 Parameters
1491 ----------
1492 associations : `lsst.jointcal.Associations`
1493 The star/reference star associations to fit.
1494 fitter : `lsst.jointcal.FitterBase`
1495 The fitter to use for minimization.
1496 max_steps : `int`
1497 Maximum number of steps to run outlier rejection before declaring
1498 convergence failure.
1499 name : {'photometry' or 'astrometry'}
1500 What type of data are we fitting (for logs and debugging files).
1501 whatToFit : `str`
1502 Passed to ``fitter.minimize()`` to define the parameters to fit.
1503 dataName : `str`, optional
1504 Descriptive name for this dataset (e.g. tract and filter),
1505 for debugging.
1506 doRankUpdate : `bool`, optional
1507 Do an Eigen rank update during minimization, or recompute the full
1508 matrix and gradient?
1509 doLineSearch : `bool`, optional
1510 Do a line search for the optimum step during minimization?
1512 Returns
1513 -------
1514 chi2: `lsst.jointcal.Chi2Statistic`
1515 The final chi2 after the fit converges, or is forced to end.
1517 Raises
1518 ------
1519 FloatingPointError
1520 Raised if the fitter fails with a non-finite value.
1521 RuntimeError
1522 Raised if the fitter fails for some other reason;
1523 log messages will provide further details.
1524 """
1525 dumpMatrixFile = self._getDebugPath(f"{name}_postinit") if self.config.writeInitMatrix else ""
1526 oldChi2 = lsst.jointcal.Chi2Statistic()
1527 oldChi2.chi2 = float("inf")
1528 for i in range(max_steps):
1529 if self.config.writeChi2FilesOuterLoop:
1530 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
1531 else:
1532 writeChi2Name = None
1533 result = fitter.minimize(whatToFit,
1534 self.config.outlierRejectSigma,
1535 doRankUpdate=doRankUpdate,
1536 doLineSearch=doLineSearch,
1537 dumpMatrixFile=dumpMatrixFile)
1538 dumpMatrixFile = "" # clear it so we don't write the matrix again.
1539 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(),
1540 f"Fit iteration {i}", writeChi2Name=writeChi2Name)
1542 if result == MinimizeResult.Converged:
1543 if doRankUpdate:
1544 self.log.debug("fit has converged - no more outliers - redo minimization "
1545 "one more time in case we have lost accuracy in rank update.")
1546 # Redo minimization one more time in case we have lost accuracy in rank update
1547 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
1548 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1550 # log a message for a large final chi2, TODO: DM-15247 for something better
1551 if chi2.chi2/chi2.ndof >= 4.0:
1552 self.log.error("Potentially bad fit: High chi-squared/ndof.")
1554 break
1555 elif result == MinimizeResult.Chi2Increased:
1556 self.log.warn("Still some outliers remaining but chi2 increased - retry")
1557 # Check whether the increase was large enough to cause trouble.
1558 chi2Ratio = chi2.chi2 / oldChi2.chi2
1559 if chi2Ratio > 1.5:
1560 self.log.warn('Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1561 chi2.chi2, oldChi2.chi2, chi2Ratio)
1562 # Based on a variety of HSC jointcal logs (see DM-25779), it
1563 # appears that chi2 increases more than a factor of ~2 always
1564 # result in the fit diverging rapidly and ending at chi2 > 1e10.
1565 # Using 10 as the "failure" threshold gives some room between
1566 # leaving a warning and bailing early.
1567 if chi2Ratio > 10:
1568 msg = ("Large chi2 increase between steps: fit likely cannot converge."
1569 " Try setting one or more of the `writeChi2*` config fields and looking"
1570 " at how individual star chi2-values evolve during the fit.")
1571 raise RuntimeError(msg)
1572 oldChi2 = chi2
1573 elif result == MinimizeResult.NonFinite:
1574 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1575 # TODO DM-12446: turn this into a "butler save" somehow.
1576 fitter.saveChi2Contributions(filename+"{type}")
1577 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1578 raise FloatingPointError(msg.format(filename))
1579 elif result == MinimizeResult.Failed:
1580 raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1581 else:
1582 raise RuntimeError("Unxepected return code from minimize().")
1583 else:
1584 self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1586 return chi2
1588 def _make_output(self, ccdImageList, model, func):
1589 """Return the internal jointcal models converted to the afw
1590 structures that will be saved to disk.
1592 Parameters
1593 ----------
1594 ccdImageList : `lsst.jointcal.CcdImageList`
1595 The list of CcdImages to get the output for.
1596 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel`
1597 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`.
1598 func : `str`
1599 The name of the function to call on ``model`` to get the converted
1600 structure. Must accept an `lsst.jointcal.CcdImage`.
1602 Returns
1603 -------
1604 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or
1605 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`]
1606 The data to be saved, keyed on (visit, detector).
1607 """
1608 output = {}
1609 for ccdImage in ccdImageList:
1610 ccd = ccdImage.ccdId
1611 visit = ccdImage.visit
1612 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd)
1613 output[(visit, ccd)] = getattr(model, func)(ccdImage)
1614 return output
1616 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1617 """
1618 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1620 Parameters
1621 ----------
1622 associations : `lsst.jointcal.Associations`
1623 The star/reference star associations to fit.
1624 model : `lsst.jointcal.AstrometryModel`
1625 The astrometric model that was fit.
1626 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1627 Dict of ccdImage identifiers to dataRefs that were fit.
1628 """
1629 ccdImageList = associations.getCcdImageList()
1630 output = self._make_output(ccdImageList, model, "makeSkyWcs")
1631 for key, skyWcs in output.items():
1632 dataRef = visit_ccd_to_dataRef[key]
1633 try:
1634 dataRef.put(skyWcs, 'jointcal_wcs')
1635 except pexExceptions.Exception as e:
1636 self.log.fatal('Failed to write updated Wcs: %s', str(e))
1637 raise e
1639 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1640 """
1641 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1643 Parameters
1644 ----------
1645 associations : `lsst.jointcal.Associations`
1646 The star/reference star associations to fit.
1647 model : `lsst.jointcal.PhotometryModel`
1648 The photoometric model that was fit.
1649 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1650 Dict of ccdImage identifiers to dataRefs that were fit.
1651 """
1653 ccdImageList = associations.getCcdImageList()
1654 output = self._make_output(ccdImageList, model, "toPhotoCalib")
1655 for key, photoCalib in output.items():
1656 dataRef = visit_ccd_to_dataRef[key]
1657 try:
1658 dataRef.put(photoCalib, 'jointcal_photoCalib')
1659 except pexExceptions.Exception as e:
1660 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1661 raise e