lsst.jointcal g84d6eb7eb7+1652af9b63
jointcal.py
Go to the documentation of this file.
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import dataclasses
23import collections
24import os
25
26import astropy.time
27import numpy as np
28import astropy.units as u
29
30import lsst.geom
31import lsst.utils
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34from lsst.afw.image import fluxErrFromABMagErr
35import lsst.pex.exceptions as pexExceptions
37import lsst.afw.table
38import lsst.log
39from lsst.obs.base import Instrument
40from lsst.pipe.tasks.colorterms import ColortermLibrary
41from lsst.verify import Job, Measurement
42
43from lsst.meas.algorithms import (LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask,
44 ReferenceObjectLoader)
45from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
46from lsst.utils.timer import timeMethod
47
48from .dataIds import PerTractCcdDataIdContainer
49
50import lsst.jointcal
51from lsst.jointcal import MinimizeResult
52
53__all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
54
55Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
56Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
57
58
59# TODO: move this to MeasurementSet in lsst.verify per DM-12655.
60def add_measurement(job, name, value):
61 meas = Measurement(job.metrics[name], value)
62 job.measurements.insert(meas)
63
64
65class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
66 """Subclass of TaskRunner for jointcalTask (gen2)
67
68 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
69 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
70 single dataRef, are are called repeatedly). This class transforms the processed
71 arguments generated by the ArgumentParser into the arguments expected by
72 Jointcal.runDataRef().
73
74 See pipeBase.TaskRunner for more information.
75 """
76
77 @staticmethod
78 def getTargetList(parsedCmd, **kwargs):
79 """
80 Return a list of tuples per tract, each containing (dataRefs, kwargs).
81
82 Jointcal operates on lists of dataRefs simultaneously.
83 """
84 kwargs['butler'] = parsedCmd.butler
85
86 # organize data IDs by tract
87 refListDict = {}
88 for ref in parsedCmd.id.refList:
89 refListDict.setdefault(ref.dataId["tract"], []).append(ref)
90 # we call runDataRef() once with each tract
91 result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
92 return result
93
94 def __call__(self, args):
95 """
96 Parameters
97 ----------
98 args
99 Arguments for Task.runDataRef()
100
101 Returns
102 -------
103 pipe.base.Struct
104 if self.doReturnResults is False:
105
106 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
107
108 if self.doReturnResults is True:
109
110 - ``result``: the result of calling jointcal.runDataRef()
111 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
112 """
113 exitStatus = 0 # exit status for shell
114
115 # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
116 dataRefList, kwargs = args
117 butler = kwargs.pop('butler')
118 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
119 result = None
120 try:
121 result = task.runDataRef(dataRefList, **kwargs)
122 exitStatus = result.exitStatus
123 job_path = butler.get('verify_job_filename')
124 result.job.write(job_path[0])
125 except Exception as e: # catch everything, sort it out later.
126 if self.doRaise:
127 raise e
128 else:
129 exitStatus = 1
130 eName = type(e).__name__
131 tract = dataRefList[0].dataId['tract']
132 task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
133
134 # Put the butler back into kwargs for the other Tasks.
135 kwargs['butler'] = butler
136 if self.doReturnResults:
137 return pipeBase.Struct(result=result, exitStatus=exitStatus)
138 else:
139 return pipeBase.Struct(exitStatus=exitStatus)
140
141
142def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
143 """Lookup function that asserts/hopes that a static calibration dataset
144 exists in a particular collection, since this task can't provide a single
145 date/time to use to search for one properly.
146
147 This is mostly useful for the ``camera`` dataset, in cases where the task's
148 quantum dimensions do *not* include something temporal, like ``exposure``
149 or ``visit``.
150
151 Parameters
152 ----------
153 datasetType : `lsst.daf.butler.DatasetType`
154 Type of dataset being searched for.
155 registry : `lsst.daf.butler.Registry`
156 Data repository registry to search.
157 quantumDataId : `lsst.daf.butler.DataCoordinate`
158 Data ID of the quantum this camera should match.
159 collections : `Iterable` [ `str` ]
160 Collections that should be searched - but this lookup function works
161 by ignoring this in favor of a more-or-less hard-coded value.
162
163 Returns
164 -------
165 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
166 Iterator over dataset references; should have only one element.
167
168 Notes
169 -----
170 This implementation duplicates one in fgcmcal, and is at least quite
171 similar to another in cp_pipe. This duplicate has the most documentation.
172 Fixing this is DM-29661.
173 """
174 instrument = Instrument.fromName(quantumDataId["instrument"], registry)
175 unboundedCollection = instrument.makeUnboundedCalibrationRunName()
176 return registry.queryDatasets(datasetType,
177 dataId=quantumDataId,
178 collections=[unboundedCollection],
179 findFirst=True)
180
181
182def lookupVisitRefCats(datasetType, registry, quantumDataId, collections):
183 """Lookup function that finds all refcats for all visits that overlap a
184 tract, rather than just the refcats that directly overlap the tract.
185
186 Parameters
187 ----------
188 datasetType : `lsst.daf.butler.DatasetType`
189 Type of dataset being searched for.
190 registry : `lsst.daf.butler.Registry`
191 Data repository registry to search.
192 quantumDataId : `lsst.daf.butler.DataCoordinate`
193 Data ID of the quantum; expected to be something we can use as a
194 constraint to query for overlapping visits.
195 collections : `Iterable` [ `str` ]
196 Collections to search.
197
198 Returns
199 -------
200 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
201 Iterator over refcat references.
202 """
203 refs = set()
204 # Use .expanded() on the query methods below because we need data IDs with
205 # regions, both in the outer loop over visits (queryDatasets will expand
206 # any data ID we give it, but doing it up-front in bulk is much more
207 # efficient) and in the data IDs of the DatasetRefs this function yields
208 # (because the RefCatLoader relies on them to do some of its own
209 # filtering).
210 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()):
211 refs.update(
212 registry.queryDatasets(
213 datasetType,
214 collections=collections,
215 dataId=visit_data_id,
216 findFirst=True,
217 ).expanded()
218 )
219 yield from refs
220
221
222class JointcalTaskConnections(pipeBase.PipelineTaskConnections,
223 dimensions=("skymap", "tract", "instrument", "physical_filter")):
224 """Middleware input/output connections for jointcal data."""
225 inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
226 doc="The camera instrument that took these observations.",
227 name="camera",
228 storageClass="Camera",
229 dimensions=("instrument",),
230 isCalibration=True,
231 lookupFunction=lookupStaticCalibrations,
232 )
233 inputSourceTableVisit = pipeBase.connectionTypes.Input(
234 doc="Source table in parquet format, per visit",
235 name="sourceTable_visit",
236 storageClass="DataFrame",
237 dimensions=("instrument", "visit"),
238 deferLoad=True,
239 multiple=True,
240 )
241 inputVisitSummary = pipeBase.connectionTypes.Input(
242 doc=("Per-visit consolidated exposure metadata built from calexps. "
243 "These catalogs use detector id for the id and must be sorted for "
244 "fast lookups of a detector."),
245 name="visitSummary",
246 storageClass="ExposureCatalog",
247 dimensions=("instrument", "visit"),
248 deferLoad=True,
249 multiple=True,
250 )
251 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
252 doc="The astrometry reference catalog to match to loaded input catalog sources.",
253 name="gaia_dr2_20200414",
254 storageClass="SimpleCatalog",
255 dimensions=("skypix",),
256 deferLoad=True,
257 multiple=True,
258 lookupFunction=lookupVisitRefCats,
259 )
260 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
261 doc="The photometry reference catalog to match to loaded input catalog sources.",
262 name="ps1_pv3_3pi_20170110",
263 storageClass="SimpleCatalog",
264 dimensions=("skypix",),
265 deferLoad=True,
266 multiple=True,
267 lookupFunction=lookupVisitRefCats,
268 )
269
270 outputWcs = pipeBase.connectionTypes.Output(
271 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model."
272 " These catalogs only contain entries for detectors with an output, and use"
273 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
274 name="jointcalSkyWcsCatalog",
275 storageClass="ExposureCatalog",
276 dimensions=("instrument", "visit", "skymap", "tract"),
277 multiple=True
278 )
279 outputPhotoCalib = pipeBase.connectionTypes.Output(
280 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model."
281 " These catalogs only contain entries for detectors with an output, and use"
282 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
283 name="jointcalPhotoCalibCatalog",
284 storageClass="ExposureCatalog",
285 dimensions=("instrument", "visit", "skymap", "tract"),
286 multiple=True
287 )
288
289 # measurements of metrics
290 # The vars() trick used here allows us to set class attributes
291 # programatically. Taken from:
292 # https://stackoverflow.com/questions/2519807/setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class
293 for name in ("astrometry", "photometry"):
294 vars()[f"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output(
295 doc=f"The number of cross-matched fittedStars for {name}",
296 name=f"metricvalue_jointcal_{name}_matched_fittedStars",
297 storageClass="MetricValue",
298 dimensions=("skymap", "tract", "instrument", "physical_filter"),
299 )
300 vars()[f"{name}_collected_refStars"] = pipeBase.connectionTypes.Output(
301 doc=f"The number of {name} reference stars drawn from the reference catalog, before matching.",
302 name=f"metricvalue_jointcal_{name}_collected_refStars",
303 storageClass="MetricValue",
304 dimensions=("skymap", "tract", "instrument", "physical_filter"),
305 )
306 vars()[f"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output(
307 doc=f"The number of {name} reference stars matched to fittedStars.",
308 name=f"metricvalue_jointcal_{name}_prepared_refStars",
309 storageClass="MetricValue",
310 dimensions=("skymap", "tract", "instrument", "physical_filter"),
311 )
312 vars()[f"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output(
313 doc=f"The number of cross-matched fittedStars after cleanup, for {name}.",
314 name=f"metricvalue_jointcal_{name}_prepared_fittedStars",
315 storageClass="MetricValue",
316 dimensions=("skymap", "tract", "instrument", "physical_filter"),
317 )
318 vars()[f"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output(
319 doc=f"The number of ccdImages that will be fit for {name}, after cleanup.",
320 name=f"metricvalue_jointcal_{name}_prepared_ccdImages",
321 storageClass="MetricValue",
322 dimensions=("skymap", "tract", "instrument", "physical_filter"),
323 )
324 vars()[f"{name}_final_chi2"] = pipeBase.connectionTypes.Output(
325 doc=f"The final chi2 of the {name} fit.",
326 name=f"metricvalue_jointcal_{name}_final_chi2",
327 storageClass="MetricValue",
328 dimensions=("skymap", "tract", "instrument", "physical_filter"),
329 )
330 vars()[f"{name}_final_ndof"] = pipeBase.connectionTypes.Output(
331 doc=f"The number of degrees of freedom of the fitted {name}.",
332 name=f"metricvalue_jointcal_{name}_final_ndof",
333 storageClass="MetricValue",
334 dimensions=("skymap", "tract", "instrument", "physical_filter"),
335 )
336
337 def __init__(self, *, config=None):
338 super().__init__(config=config)
339 # When we are only doing one of astrometry or photometry, we don't
340 # need the reference catalog or produce the outputs for the other.
341 # This informs the middleware of that when the QuantumGraph is
342 # generated, so we don't block on getting something we won't need or
343 # create an expectation that downstream tasks will be able to consume
344 # something we won't produce.
345 if not config.doAstrometry:
346 self.prerequisiteInputs.remove("astrometryRefCat")
347 self.outputs.remove("outputWcs")
348 for key in list(self.outputs):
349 if "metricvalue_jointcal_astrometry" in key:
350 self.outputs.remove(key)
351 if not config.doPhotometry:
352 self.prerequisiteInputs.remove("photometryRefCat")
353 self.outputs.remove("outputPhotoCalib")
354 for key in list(self.outputs):
355 if "metricvalue_jointcal_photometry" in key:
356 self.outputs.remove(key)
357
358
359class JointcalConfig(pipeBase.PipelineTaskConfig,
360 pipelineConnections=JointcalTaskConnections):
361 """Configuration for JointcalTask"""
362
363 doAstrometry = pexConfig.Field(
364 doc="Fit astrometry and write the fitted result.",
365 dtype=bool,
366 default=True
367 )
368 doPhotometry = pexConfig.Field(
369 doc="Fit photometry and write the fitted result.",
370 dtype=bool,
371 default=True
372 )
373 coaddName = pexConfig.Field(
374 doc="Type of coadd, typically deep or goodSeeing",
375 deprecated="Only applies to gen2; will be removed when gen2 support is removed (DM-20572).",
376 dtype=str,
377 default="deep"
378 )
379 # TODO DM-29008: Change this to "ApFlux_12_0" before gen2 removal.
380 sourceFluxType = pexConfig.Field(
381 dtype=str,
382 doc="Source flux field to use in source selection and to get fluxes from the catalog.",
383 default='Calib'
384 )
385 positionErrorPedestal = pexConfig.Field(
386 doc="Systematic term to apply to the measured position error (pixels)",
387 dtype=float,
388 default=0.02,
389 )
390 photometryErrorPedestal = pexConfig.Field(
391 doc="Systematic term to apply to the measured error on flux or magnitude as a "
392 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
393 dtype=float,
394 default=0.0,
395 )
396 # TODO: DM-6885 matchCut should be an geom.Angle
397 matchCut = pexConfig.Field(
398 doc="Matching radius between fitted and reference stars (arcseconds)",
399 dtype=float,
400 default=3.0,
401 )
402 minMeasurements = pexConfig.Field(
403 doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
404 dtype=int,
405 default=2,
406 )
407 minMeasuredStarsPerCcd = pexConfig.Field(
408 doc="Minimum number of measuredStars per ccdImage before printing warnings",
409 dtype=int,
410 default=100,
411 )
412 minRefStarsPerCcd = pexConfig.Field(
413 doc="Minimum number of measuredStars per ccdImage before printing warnings",
414 dtype=int,
415 default=30,
416 )
417 allowLineSearch = pexConfig.Field(
418 doc="Allow a line search during minimization, if it is reasonable for the model"
419 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
420 dtype=bool,
421 default=False
422 )
423 astrometrySimpleOrder = pexConfig.Field(
424 doc="Polynomial order for fitting the simple astrometry model.",
425 dtype=int,
426 default=3,
427 )
428 astrometryChipOrder = pexConfig.Field(
429 doc="Order of the per-chip transform for the constrained astrometry model.",
430 dtype=int,
431 default=1,
432 )
433 astrometryVisitOrder = pexConfig.Field(
434 doc="Order of the per-visit transform for the constrained astrometry model.",
435 dtype=int,
436 default=5,
437 )
438 useInputWcs = pexConfig.Field(
439 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
440 dtype=bool,
441 default=True,
442 )
443 astrometryModel = pexConfig.ChoiceField(
444 doc="Type of model to fit to astrometry",
445 dtype=str,
446 default="constrained",
447 allowed={"simple": "One polynomial per ccd",
448 "constrained": "One polynomial per ccd, and one polynomial per visit"}
449 )
450 photometryModel = pexConfig.ChoiceField(
451 doc="Type of model to fit to photometry",
452 dtype=str,
453 default="constrainedMagnitude",
454 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
455 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
456 " fitting in flux space.",
457 "simpleMagnitude": "One constant zeropoint per ccd and visit,"
458 " fitting in magnitude space.",
459 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
460 " fitting in magnitude space.",
461 }
462 )
463 applyColorTerms = pexConfig.Field(
464 doc="Apply photometric color terms to reference stars?"
465 "Requires that colorterms be set to a ColortermLibrary",
466 dtype=bool,
467 default=False
468 )
469 colorterms = pexConfig.ConfigField(
470 doc="Library of photometric reference catalog name to color term dict.",
471 dtype=ColortermLibrary,
472 )
473 photometryVisitOrder = pexConfig.Field(
474 doc="Order of the per-visit polynomial transform for the constrained photometry model.",
475 dtype=int,
476 default=7,
477 )
478 photometryDoRankUpdate = pexConfig.Field(
479 doc=("Do the rank update step during minimization. "
480 "Skipping this can help deal with models that are too non-linear."),
481 dtype=bool,
482 default=True,
483 )
484 astrometryDoRankUpdate = pexConfig.Field(
485 doc=("Do the rank update step during minimization (should not change the astrometry fit). "
486 "Skipping this can help deal with models that are too non-linear."),
487 dtype=bool,
488 default=True,
489 )
490 outlierRejectSigma = pexConfig.Field(
491 doc="How many sigma to reject outliers at during minimization.",
492 dtype=float,
493 default=5.0,
494 )
495 astrometryOutlierRelativeTolerance = pexConfig.Field(
496 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will "
497 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set "
498 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002"
499 "as a balance between a shorter minimization runtime and achieving a final fitted model that is"
500 "close to the solution found when removing all outliers."),
501 dtype=float,
502 default=0,
503 )
504 maxPhotometrySteps = pexConfig.Field(
505 doc="Maximum number of minimize iterations to take when fitting photometry.",
506 dtype=int,
507 default=20,
508 )
509 maxAstrometrySteps = pexConfig.Field(
510 doc="Maximum number of minimize iterations to take when fitting astrometry.",
511 dtype=int,
512 default=20,
513 )
514 astrometryRefObjLoader = pexConfig.ConfigurableField(
515 target=LoadIndexedReferenceObjectsTask,
516 doc="Reference object loader for astrometric fit",
517 )
518 photometryRefObjLoader = pexConfig.ConfigurableField(
519 target=LoadIndexedReferenceObjectsTask,
520 doc="Reference object loader for photometric fit",
521 )
522 sourceSelector = sourceSelectorRegistry.makeField(
523 doc="How to select sources for cross-matching",
524 default="astrometry"
525 )
526 astrometryReferenceSelector = pexConfig.ConfigurableField(
527 target=ReferenceSourceSelectorTask,
528 doc="How to down-select the loaded astrometry reference catalog.",
529 )
530 photometryReferenceSelector = pexConfig.ConfigurableField(
531 target=ReferenceSourceSelectorTask,
532 doc="How to down-select the loaded photometry reference catalog.",
533 )
534 astrometryReferenceErr = pexConfig.Field(
535 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
536 "If None, then raise an exception if the reference catalog is missing coordinate errors. "
537 "If specified, overrides any existing `coord_*Err` values."),
538 dtype=float,
539 default=None,
540 optional=True
541 )
542
543 # configs for outputting debug information
544 writeInitMatrix = pexConfig.Field(
545 dtype=bool,
546 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
547 "Output files will be written to `config.debugOutputPath` and will "
548 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. "
549 "Note that these files are the dense versions of the matrix, and so may be very large."),
550 default=False
551 )
552 writeChi2FilesInitialFinal = pexConfig.Field(
553 dtype=bool,
554 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. "
555 "Output files will be written to `config.debugOutputPath` and will "
556 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."),
557 default=False
558 )
559 writeChi2FilesOuterLoop = pexConfig.Field(
560 dtype=bool,
561 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. "
562 "Output files will be written to `config.debugOutputPath` and will "
563 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."),
564 default=False
565 )
566 writeInitialModel = pexConfig.Field(
567 dtype=bool,
568 doc=("Write the pre-initialization model to text files, for debugging. "
569 "Output files will be written to `config.debugOutputPath` and will be "
570 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."),
571 default=False
572 )
573 debugOutputPath = pexConfig.Field(
574 dtype=str,
575 default=".",
576 doc=("Path to write debug output files to. Used by "
577 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
578 )
579 detailedProfile = pexConfig.Field(
580 dtype=bool,
581 default=False,
582 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
583 )
584
585 def validate(self):
586 super().validate()
587 if self.doPhotometrydoPhotometry and self.applyColorTermsapplyColorTerms and len(self.colortermscolorterms.data) == 0:
588 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
589 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
590 if self.doAstrometrydoAstrometry and not self.doPhotometrydoPhotometry and self.applyColorTermsapplyColorTerms:
591 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;"
592 "applyColorTerms=True will be ignored.")
593 lsst.log.warning(msg)
594
595 def setDefaults(self):
596 # Use science source selector which can filter on extendedness, SNR, and whether blended
597 self.sourceSelectorsourceSelector.name = 'science'
598 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing
599 self.sourceSelectorsourceSelector['science'].doUnresolved = True
600 # with dependable signal to noise ratio.
601 self.sourceSelectorsourceSelector['science'].doSignalToNoise = True
602 # Min SNR must be > 0 because jointcal cannot handle negative fluxes,
603 # and S/N > 10 to use sources that are not too faint, and thus better measured.
604 self.sourceSelectorsourceSelector['science'].signalToNoise.minimum = 10.
605 # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive
606 fluxField = f"slot_{self.sourceFluxType}Flux_instFlux"
607 self.sourceSelectorsourceSelector['science'].signalToNoise.fluxField = fluxField
608 self.sourceSelectorsourceSelector['science'].signalToNoise.errField = fluxField + "Err"
609 # Do not trust blended sources' aperture fluxes which also depend on seeing
610 self.sourceSelectorsourceSelector['science'].doIsolated = True
611 # Do not trust either flux or centroid measurements with flags,
612 # chosen from the usual QA flags for stars)
613 self.sourceSelectorsourceSelector['science'].doFlags = True
614 badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated',
615 'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag',
616 'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter']
617 self.sourceSelectorsourceSelector['science'].flags.bad = badFlags
618
619 # Default to Gaia-DR2 (with proper motions) for astrometry and
620 # PS1-DR1 for photometry, with a reasonable initial filterMap.
621 self.astrometryRefObjLoaderastrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
622 self.astrometryRefObjLoaderastrometryRefObjLoader.requireProperMotion = True
623 self.astrometryRefObjLoaderastrometryRefObjLoader.anyFilterMapsToThis = 'phot_g_mean'
624 self.photometryRefObjLoaderphotometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110"
625
626
627def writeModel(model, filename, log):
628 """Write model to outfile."""
629 with open(filename, "w") as file:
630 file.write(repr(model))
631 log.info("Wrote %s to file: %s", model, filename)
632
633
634@dataclasses.dataclass
636 """The input data jointcal needs for each detector/visit."""
637 visit: int
638 """The visit identifier of this exposure."""
640 """The catalog derived from this exposure."""
641 visitInfo: lsst.afw.image.VisitInfo
642 """The VisitInfo of this exposure."""
644 """The detector of this exposure."""
645 photoCalib: lsst.afw.image.PhotoCalib
646 """The photometric calibration of this exposure."""
648 """The WCS of this exposure."""
649 bbox: lsst.geom.Box2I
650 """The bounding box of this exposure."""
652 """The filter of this exposure."""
653
654
655class JointcalTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
656 """Astrometricly and photometricly calibrate across multiple visits of the
657 same field.
658
659 Parameters
660 ----------
661 butler : `lsst.daf.persistence.Butler`
662 The butler is passed to the refObjLoader constructor in case it is
663 needed. Ignored if the refObjLoader argument provides a loader directly.
664 Used to initialize the astrometry and photometry refObjLoaders.
665 initInputs : `dict`, optional
666 Dictionary used to initialize PipelineTasks (empty for jointcal).
667 """
668
669 ConfigClass = JointcalConfig
670 RunnerClass = JointcalRunner
671 _DefaultName = "jointcal"
672
673 def __init__(self, butler=None, initInputs=None, **kwargs):
674 super().__init__(**kwargs)
675 self.makeSubtask("sourceSelector")
676 if self.config.doAstrometry:
677 if initInputs is None:
678 # gen3 middleware does refcat things internally (and will not have a butler here)
679 self.makeSubtask('astrometryRefObjLoader', butler=butler)
680 self.makeSubtask("astrometryReferenceSelector")
681 else:
682 self.astrometryRefObjLoaderastrometryRefObjLoader = None
683 if self.config.doPhotometry:
684 if initInputs is None:
685 # gen3 middleware does refcat things internally (and will not have a butler here)
686 self.makeSubtask('photometryRefObjLoader', butler=butler)
687 self.makeSubtask("photometryReferenceSelector")
688 else:
689 self.photometryRefObjLoaderphotometryRefObjLoader = None
690
691 # To hold various computed metrics for use by tests
692 self.jobjob = Job.load_metrics_package(subset='jointcal')
693
694 def runQuantum(self, butlerQC, inputRefs, outputRefs):
695 # We override runQuantum to set up the refObjLoaders and write the
696 # outputs to the correct refs.
697 inputs = butlerQC.get(inputRefs)
698 # We want the tract number for writing debug files
699 tract = butlerQC.quantum.dataId['tract']
700 if self.config.doAstrometry:
701 self.astrometryRefObjLoaderastrometryRefObjLoader = ReferenceObjectLoader(
702 dataIds=[ref.datasetRef.dataId
703 for ref in inputRefs.astrometryRefCat],
704 refCats=inputs.pop('astrometryRefCat'),
705 config=self.config.astrometryRefObjLoader,
706 log=self.log)
707 if self.config.doPhotometry:
708 self.photometryRefObjLoaderphotometryRefObjLoader = ReferenceObjectLoader(
709 dataIds=[ref.datasetRef.dataId
710 for ref in inputRefs.photometryRefCat],
711 refCats=inputs.pop('photometryRefCat'),
712 config=self.config.photometryRefObjLoader,
713 log=self.log)
714 outputs = self.runrun(**inputs, tract=tract)
715 self._put_metrics_put_metrics(butlerQC, outputs.job, outputRefs)
716 if self.config.doAstrometry:
717 self._put_output_put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
718 inputs['inputCamera'], "setWcs")
719 if self.config.doPhotometry:
720 self._put_output_put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
721 inputs['inputCamera'], "setPhotoCalib")
722
723 def _put_metrics(self, butlerQC, job, outputRefs):
724 """Persist all measured metrics stored in a job.
725
726 Parameters
727 ----------
728 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
729 A butler which is specialized to operate in the context of a
730 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
731 job : `lsst.verify.job.Job`
732 Measurements of metrics to persist.
733 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
734 The DatasetRefs to persist the data to.
735 """
736 for key in job.measurements.keys():
737 butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace('jointcal.', '')))
738
739 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
740 """Persist the output datasets to their appropriate datarefs.
741
742 Parameters
743 ----------
744 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
745 A butler which is specialized to operate in the context of a
746 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
747 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or
748 `dict` [`tuple, `lsst.afw.image.PhotoCalib`]
749 The fitted objects to persist.
750 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
751 The DatasetRefs to persist the data to.
753 The camera for this instrument, to get detector ids from.
754 setter : `str`
755 The method to call on the ExposureCatalog to set each output.
756 """
758 schema.addField('visit', type='I', doc='Visit number')
759
760 def new_catalog(visit, size):
761 """Return an catalog ready to be filled with appropriate output."""
762 catalog = lsst.afw.table.ExposureCatalog(schema)
763 catalog.resize(size)
764 catalog['visit'] = visit
765 metadata = lsst.daf.base.PropertyList()
766 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
767 metadata.add("COMMENT", "Only detectors with data have entries.")
768 return catalog
769
770 # count how many detectors have output for each visit
771 detectors_per_visit = collections.defaultdict(int)
772 for key in outputs:
773 # key is (visit, detector_id), and we only need visit here
774 detectors_per_visit[key[0]] += 1
775
776 for ref in outputRefs:
777 visit = ref.dataId['visit']
778 catalog = new_catalog(visit, detectors_per_visit[visit])
779 # Iterate over every detector and skip the ones we don't have output for.
780 i = 0
781 for detector in camera:
782 detectorId = detector.getId()
783 key = (ref.dataId['visit'], detectorId)
784 if key not in outputs:
785 # skip detectors we don't have output for
786 self.log.debug("No %s output for detector %s in visit %s",
787 setter[3:], detectorId, visit)
788 continue
789
790 catalog[i].setId(detectorId)
791 getattr(catalog[i], setter)(outputs[key])
792 i += 1
793
794 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups
795 butlerQC.put(catalog, ref)
796 self.log.info("Wrote %s detectors to %s", i, ref)
797
798 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
799 # Docstring inherited.
800
801 # We take values out of the Parquet table, and put them in "flux_",
802 # and the config.sourceFluxType field is used during that extraction,
803 # so just use "flux" here.
804 sourceFluxField = "flux"
805 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
806 associations = lsst.jointcal.Associations()
807 self.focalPlaneBBoxfocalPlaneBBox = inputCamera.getFpBBox()
808 oldWcsList, bands = self._load_data_load_data(inputSourceTableVisit,
809 inputVisitSummary,
810 associations,
811 jointcalControl,
812 inputCamera)
813
814 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky_prep_sky(associations, bands)
815
816 if self.config.doAstrometry:
817 astrometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
818 name="astrometry",
819 refObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
820 referenceSelector=self.astrometryReferenceSelector,
821 fit_function=self._fit_astrometry_fit_astrometry,
822 tract=tract,
823 epoch=epoch)
824 astrometry_output = self._make_output_make_output(associations.getCcdImageList(),
825 astrometry.model,
826 "makeSkyWcs")
827 else:
828 astrometry_output = None
829
830 if self.config.doPhotometry:
831 photometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
832 name="photometry",
833 refObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
834 referenceSelector=self.photometryReferenceSelector,
835 fit_function=self._fit_photometry_fit_photometry,
836 tract=tract,
837 epoch=epoch,
838 reject_bad_fluxes=True)
839 photometry_output = self._make_output_make_output(associations.getCcdImageList(),
840 photometry.model,
841 "toPhotoCalib")
842 else:
843 photometry_output = None
844
845 return pipeBase.Struct(outputWcs=astrometry_output,
846 outputPhotoCalib=photometry_output,
847 job=self.jobjob,
848 astrometryRefObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
849 photometryRefObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader)
850
851 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
852 jointcalControl, camera):
853 """Read the data that jointcal needs to run. (Gen3 version)
854
855 Modifies ``associations`` in-place with the loaded data.
856
857 Parameters
858 ----------
859 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
860 References to visit-level DataFrames to load the catalog data from.
861 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
862 Visit-level exposure summary catalog with metadata.
863 associations : `lsst.jointcal.Associations`
864 Object to add the loaded data to by constructing new CcdImages.
865 jointcalControl : `jointcal.JointcalControl`
866 Control object for C++ associations management.
868 Camera object for detector geometry.
869
870 Returns
871 -------
872 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`]
873 The original WCS of the input data, to aid in writing tests.
874 bands : `list` [`str`]
875 The filter bands of each input dataset.
876 """
877 oldWcsList = []
878 filters = []
879 load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile else ''
880 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
881 table = make_schema_table() # every detector catalog has the same layout
882 # No guarantee that the input is in the same order of visits, so we have to map one of them.
883 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)}
884 detectorDict = {detector.getId(): detector for detector in camera}
885
886 columns = None
887
888 for visitSummaryRef in inputVisitSummary:
889 visitSummary = visitSummaryRef.get()
890
891 dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]]
892 if columns is None:
893 inColumns = dataRef.get(component='columns')
894 columns, detColumn, ixxColumns = get_sourceTable_visit_columns(inColumns,
895 self.config,
896 self.sourceSelector)
897 visitCatalog = dataRef.get(parameters={'columns': columns})
898
899 selected = self.sourceSelector.run(visitCatalog)
900
901 # Build a CcdImage for each detector in this visit.
902 detectors = {id: index for index, id in enumerate(visitSummary['id'])}
903 for id, index in detectors.items():
905 selected.sourceCat,
906 id,
907 detColumn,
908 ixxColumns,
909 self.config.sourceFluxType,
910 self.log)
911 if catalog is None:
912 continue
913 data = self._make_one_input_data_make_one_input_data(visitSummary[index], catalog, detectorDict)
914 result = self._build_ccdImage_build_ccdImage(data, associations, jointcalControl)
915 if result is not None:
916 oldWcsList.append(result.wcs)
917 # A visit has only one band, so we can just use the first.
918 filters.append(data.filter)
919 if len(filters) == 0:
920 raise RuntimeError("No data to process: did source selector remove all sources?")
921 filters = collections.Counter(filters)
922
923 return oldWcsList, filters
924
925 def _make_one_input_data(self, visitRecord, catalog, detectorDict):
926 """Return a data structure for this detector+visit."""
927 return JointcalInputData(visit=visitRecord['visit'],
928 catalog=catalog,
929 visitInfo=visitRecord.getVisitInfo(),
930 detector=detectorDict[visitRecord.getId()],
931 photoCalib=visitRecord.getPhotoCalib(),
932 wcs=visitRecord.getWcs(),
933 bbox=visitRecord.getBBox(),
934 # ExposureRecord doesn't have a FilterLabel yet,
935 # so we have to make one.
936 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'],
937 physical=visitRecord['physical_filter']))
938
939 # We don't currently need to persist the metadata.
940 # If we do in the future, we will have to add appropriate dataset templates
941 # to each obs package (the metadata template should look like `jointcal_wcs`).
942 def _getMetadataName(self):
943 return None
944
945 @classmethod
946 def _makeArgumentParser(cls):
947 """Create an argument parser"""
948 parser = pipeBase.ArgumentParser(name=cls._DefaultName_DefaultName)
949 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
950 ContainerClass=PerTractCcdDataIdContainer)
951 return parser
952
953 def _build_ccdImage(self, data, associations, jointcalControl):
954 """
955 Extract the necessary things from this catalog+metadata to add a new
956 ccdImage.
957
958 Parameters
959 ----------
960 data : `JointcalInputData`
961 The loaded input data.
962 associations : `lsst.jointcal.Associations`
963 Object to add the info to, to construct a new CcdImage
964 jointcalControl : `jointcal.JointcalControl`
965 Control object for associations management
966
967 Returns
968 ------
969 namedtuple or `None`
970 ``wcs``
971 The TAN WCS of this image, read from the calexp
973 ``key``
974 A key to identify this dataRef by its visit and ccd ids
975 (`namedtuple`).
976 `None`
977 if there are no sources in the loaded catalog.
978 """
979 if len(data.catalog) == 0:
980 self.log.warning("No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
981 return None
982
983 associations.createCcdImage(data.catalog,
984 data.wcs,
985 data.visitInfo,
986 data.bbox,
987 data.filter.physicalLabel,
988 data.photoCalib,
989 data.detector,
990 data.visit,
991 data.detector.getId(),
992 jointcalControl)
993
994 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key'))
995 Key = collections.namedtuple('Key', ('visit', 'ccd'))
996 return Result(data.wcs, Key(data.visit, data.detector.getId()))
997
998 def _readDataId(self, butler, dataId):
999 """Read all of the data for one dataId from the butler. (gen2 version)"""
1000 # Not all instruments have `visit` in their dataIds.
1001 if "visit" in dataId.keys():
1002 visit = dataId["visit"]
1003 else:
1004 visit = butler.getButler().queryMetadata("calexp", ("visit"), butler.dataId)[0]
1005 detector = butler.get('calexp_detector', dataId=dataId)
1006
1007 catalog = butler.get('src',
1008 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
1009 dataId=dataId)
1010 goodSrc = self.sourceSelector.run(catalog)
1011 self.log.debug("%d sources selected in visit %d detector %d",
1012 len(goodSrc.sourceCat),
1013 visit,
1014 detector.getId())
1015 return JointcalInputData(visit=visit,
1016 catalog=goodSrc.sourceCat,
1017 visitInfo=butler.get('calexp_visitInfo', dataId=dataId),
1018 detector=detector,
1019 photoCalib=butler.get('calexp_photoCalib', dataId=dataId),
1020 wcs=butler.get('calexp_wcs', dataId=dataId),
1021 bbox=butler.get('calexp_bbox', dataId=dataId),
1022 filter=butler.get('calexp_filterLabel', dataId=dataId))
1023
1024 def loadData(self, dataRefs, associations, jointcalControl):
1025 """Read the data that jointcal needs to run. (Gen2 version)"""
1026 visit_ccd_to_dataRef = {}
1027 oldWcsList = []
1028 filters = []
1029 load_cat_prof_file = 'jointcal_loadData.prof' if self.config.detailedProfile else ''
1030 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1031 # Need the bounding-box of the focal plane (the same for all visits) for photometry visit models
1032 camera = dataRefs[0].get('camera', immediate=True)
1033 self.focalPlaneBBoxfocalPlaneBBox = camera.getFpBBox()
1034 for dataRef in dataRefs:
1035 data = self._readDataId_readDataId(dataRef.getButler(), dataRef.dataId)
1036 result = self._build_ccdImage_build_ccdImage(data, associations, jointcalControl)
1037 if result is None:
1038 continue
1039 oldWcsList.append(result.wcs)
1040 visit_ccd_to_dataRef[result.key] = dataRef
1041 filters.append(data.filter)
1042 if len(filters) == 0:
1043 raise RuntimeError("No data to process: did source selector remove all sources?")
1044 filters = collections.Counter(filters)
1045
1046 return oldWcsList, filters, visit_ccd_to_dataRef
1047
1048 def _getDebugPath(self, filename):
1049 """Constructs a path to filename using the configured debug path.
1050 """
1051 return os.path.join(self.config.debugOutputPath, filename)
1052
1053 def _prep_sky(self, associations, filters):
1054 """Prepare on-sky and other data that must be computed after data has
1055 been read.
1056 """
1057 associations.computeCommonTangentPoint()
1058
1059 boundingCircle = associations.computeBoundingCircle()
1060 center = lsst.geom.SpherePoint(boundingCircle.getCenter())
1061 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
1062
1063 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.")
1064
1065 # Determine a default filter band associated with the catalog. See DM-9093
1066 defaultFilter = filters.most_common(1)[0][0]
1067 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel)
1068
1069 # compute and set the reference epoch of the observations, for proper motion corrections
1070 epoch = self._compute_proper_motion_epoch_compute_proper_motion_epoch(associations.getCcdImageList())
1071 associations.setEpoch(epoch.jyear)
1072
1073 return boundingCircle, center, radius, defaultFilter, epoch
1074
1075 @timeMethod
1076 def runDataRef(self, dataRefs):
1077 """
1078 Jointly calibrate the astrometry and photometry across a set of images.
1079
1080 NOTE: this is for gen2 middleware only.
1081
1082 Parameters
1083 ----------
1084 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
1085 List of data references to the exposures to be fit.
1086
1087 Returns
1088 -------
1089 result : `lsst.pipe.base.Struct`
1090 Struct of metadata from the fit, containing:
1091
1092 ``dataRefs``
1093 The provided data references that were fit (with updated WCSs)
1094 ``oldWcsList``
1095 The original WCS from each dataRef
1096 ``metrics``
1097 Dictionary of internally-computed metrics for testing/validation.
1098 """
1099 if len(dataRefs) == 0:
1100 raise ValueError('Need a non-empty list of data references!')
1101
1102 exitStatus = 0 # exit status for shell
1103
1104 sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
1105 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
1106 associations = lsst.jointcal.Associations()
1107
1108 oldWcsList, filters, visit_ccd_to_dataRef = self.loadDataloadData(dataRefs,
1109 associations,
1110 jointcalControl)
1111
1112 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky_prep_sky(associations, filters)
1113
1114 tract = dataRefs[0].dataId['tract']
1115
1116 if self.config.doAstrometry:
1117 astrometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1118 name="astrometry",
1119 refObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
1120 referenceSelector=self.astrometryReferenceSelector,
1121 fit_function=self._fit_astrometry_fit_astrometry,
1122 tract=tract,
1123 epoch=epoch)
1124 self._write_astrometry_results_write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
1125 else:
1126 astrometry = Astrometry(None, None, None)
1127
1128 if self.config.doPhotometry:
1129 photometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1130 name="photometry",
1131 refObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
1132 referenceSelector=self.photometryReferenceSelector,
1133 fit_function=self._fit_photometry_fit_photometry,
1134 tract=tract,
1135 epoch=epoch,
1136 reject_bad_fluxes=True)
1137 self._write_photometry_results_write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
1138 else:
1139 photometry = Photometry(None, None)
1140
1141 return pipeBase.Struct(dataRefs=dataRefs,
1142 oldWcsList=oldWcsList,
1143 job=self.jobjob,
1144 astrometryRefObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
1145 photometryRefObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
1146 defaultFilter=defaultFilter,
1147 epoch=epoch,
1148 exitStatus=exitStatus)
1149
1150 def _get_refcat_coordinate_error_override(self, refCat, name):
1151 """Check whether we should override the refcat coordinate errors, and
1152 return the overridden error if necessary.
1153
1154 Parameters
1155 ----------
1157 The reference catalog to check for a ``coord_raErr`` field.
1158 name : `str`
1159 Whether we are doing "astrometry" or "photometry".
1160
1161 Returns
1162 -------
1163 refCoordErr : `float`
1164 The refcat coordinate error to use, or NaN if we are not overriding
1165 those fields.
1166
1167 Raises
1168 ------
1169 lsst.pex.config.FieldValidationError
1170 Raised if the refcat does not contain coordinate errors and
1171 ``config.astrometryReferenceErr`` is not set.
1172 """
1173 # This value doesn't matter for photometry, so just set something to
1174 # keep old refcats from causing problems.
1175 if name.lower() == "photometry":
1176 if 'coord_raErr' not in refCat.schema:
1177 return 100
1178 else:
1179 return float('nan')
1180
1181 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema:
1182 msg = ("Reference catalog does not contain coordinate errors, "
1183 "and config.astrometryReferenceErr not supplied.")
1184 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
1185 self.config,
1186 msg)
1187
1188 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema:
1189 self.log.warning("Overriding reference catalog coordinate errors with %f/coordinate [mas]",
1190 self.config.astrometryReferenceErr)
1191
1192 if self.config.astrometryReferenceErr is None:
1193 return float('nan')
1194 else:
1195 return self.config.astrometryReferenceErr
1196
1197 def _compute_proper_motion_epoch(self, ccdImageList):
1198 """Return the proper motion correction epoch of the provided images.
1199
1200 Parameters
1201 ----------
1202 ccdImageList : `list` [`lsst.jointcal.CcdImage`]
1203 The images to compute the appropriate epoch for.
1204
1205 Returns
1206 -------
1207 epoch : `astropy.time.Time`
1208 The date to use for proper motion corrections.
1209 """
1210 return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]),
1211 format="jyear",
1212 scale="tai")
1213
1214 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
1215 tract="", match_cut=3.0,
1216 reject_bad_fluxes=False, *,
1217 name="", refObjLoader=None, referenceSelector=None,
1218 fit_function=None, epoch=None):
1219 """Load reference catalog, perform the fit, and return the result.
1220
1221 Parameters
1222 ----------
1223 associations : `lsst.jointcal.Associations`
1224 The star/reference star associations to fit.
1225 defaultFilter : `lsst.afw.image.FilterLabel`
1226 filter to load from reference catalog.
1227 center : `lsst.geom.SpherePoint`
1228 ICRS center of field to load from reference catalog.
1229 radius : `lsst.geom.Angle`
1230 On-sky radius to load from reference catalog.
1231 name : `str`
1232 Name of thing being fit: "astrometry" or "photometry".
1233 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1234 Reference object loader to use to load a reference catalog.
1235 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1236 Selector to use to pick objects from the loaded reference catalog.
1237 fit_function : callable
1238 Function to call to perform fit (takes Associations object).
1239 tract : `str`, optional
1240 Name of tract currently being fit.
1241 match_cut : `float`, optional
1242 Radius in arcseconds to find cross-catalog matches to during
1243 associations.associateCatalogs.
1244 reject_bad_fluxes : `bool`, optional
1245 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
1246 epoch : `astropy.time.Time`, optional
1247 Epoch to which to correct refcat proper motion and parallax,
1248 or `None` to not apply such corrections.
1249
1250 Returns
1251 -------
1252 result : `Photometry` or `Astrometry`
1253 Result of `fit_function()`
1254 """
1255 self.log.info("====== Now processing %s...", name)
1256 # TODO: this should not print "trying to invert a singular transformation:"
1257 # if it does that, something's not right about the WCS...
1258 associations.associateCatalogs(match_cut)
1259 add_measurement(self.jobjob, 'jointcal.%s_matched_fittedStars' % name,
1260 associations.fittedStarListSize())
1261
1262 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms
1263 refCat, fluxField = self._load_reference_catalog_load_reference_catalog(refObjLoader, referenceSelector,
1264 center, radius, defaultFilter,
1265 applyColorterms=applyColorterms,
1266 epoch=epoch)
1267 refCoordErr = self._get_refcat_coordinate_error_override_get_refcat_coordinate_error_override(refCat, name)
1268
1269 associations.collectRefStars(refCat,
1270 self.config.matchCut*lsst.geom.arcseconds,
1271 fluxField,
1272 refCoordinateErr=refCoordErr,
1273 rejectBadFluxes=reject_bad_fluxes)
1274 add_measurement(self.jobjob, 'jointcal.%s_collected_refStars' % name,
1275 associations.refStarListSize())
1276
1277 associations.prepareFittedStars(self.config.minMeasurements)
1278
1279 self._check_star_lists_check_star_lists(associations, name)
1280 add_measurement(self.jobjob, 'jointcal.%s_prepared_refStars' % name,
1281 associations.nFittedStarsWithAssociatedRefStar())
1282 add_measurement(self.jobjob, 'jointcal.%s_prepared_fittedStars' % name,
1283 associations.fittedStarListSize())
1284 add_measurement(self.jobjob, 'jointcal.%s_prepared_ccdImages' % name,
1285 associations.nCcdImagesValidForFit())
1286
1287 load_cat_prof_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else ''
1288 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel)
1289 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1290 result = fit_function(associations, dataName)
1291 # TODO DM-12446: turn this into a "butler save" somehow.
1292 # Save reference and measurement chi2 contributions for this data
1293 if self.config.writeChi2FilesInitialFinal:
1294 baseName = self._getDebugPath_getDebugPath(f"{name}_final_chi2-{dataName}")
1295 result.fit.saveChi2Contributions(baseName+"{type}")
1296 self.log.info("Wrote chi2 contributions files: %s", baseName)
1297
1298 return result
1299
1300 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1301 applyColorterms=False, epoch=None):
1302 """Load the necessary reference catalog sources, convert fluxes to
1303 correct units, and apply color term corrections if requested.
1304
1305 Parameters
1306 ----------
1307 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1308 The reference catalog loader to use to get the data.
1309 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1310 Source selector to apply to loaded reference catalog.
1311 center : `lsst.geom.SpherePoint`
1312 The center around which to load sources.
1313 radius : `lsst.geom.Angle`
1314 The radius around ``center`` to load sources in.
1315 filterLabel : `lsst.afw.image.FilterLabel`
1316 The camera filter to load fluxes for.
1317 applyColorterms : `bool`
1318 Apply colorterm corrections to the refcat for ``filterName``?
1319 epoch : `astropy.time.Time`, optional
1320 Epoch to which to correct refcat proper motion and parallax,
1321 or `None` to not apply such corrections.
1322
1323 Returns
1324 -------
1326 The loaded reference catalog.
1327 fluxField : `str`
1328 The name of the reference catalog flux field appropriate for ``filterName``.
1329 """
1330 skyCircle = refObjLoader.loadSkyCircle(center,
1331 radius,
1332 filterLabel.bandLabel,
1333 epoch=epoch)
1334
1335 selected = referenceSelector.run(skyCircle.refCat)
1336 # Need memory contiguity to get reference filters as a vector.
1337 if not selected.sourceCat.isContiguous():
1338 refCat = selected.sourceCat.copy(deep=True)
1339 else:
1340 refCat = selected.sourceCat
1341
1342 if applyColorterms:
1343 try:
1344 refCatName = refObjLoader.ref_dataset_name # Gen2
1345 except AttributeError:
1346 refCatName = self.config.connections.photometryRefCat # Gen3
1347
1348 self.log.info("Applying color terms for physical filter=%r reference catalog=%s",
1349 filterLabel.physicalLabel, refCatName)
1350 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1351 refCatName,
1352 doRaise=True)
1353
1354 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1355 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1356 # TODO: I didn't want to use this, but I'll deal with it in DM-16903
1357 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1358
1359 return refCat, skyCircle.fluxField
1360
1361 def _check_star_lists(self, associations, name):
1362 # TODO: these should be len(blah), but we need this properly wrapped first.
1363 if associations.nCcdImagesValidForFit() == 0:
1364 raise RuntimeError('No images in the ccdImageList!')
1365 if associations.fittedStarListSize() == 0:
1366 raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
1367 if associations.refStarListSize() == 0:
1368 raise RuntimeError('No stars in the {} reference star list!'.format(name))
1369
1370 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1371 """Compute chi2, log it, validate the model, and return chi2.
1372
1373 Parameters
1374 ----------
1375 associations : `lsst.jointcal.Associations`
1376 The star/reference star associations to fit.
1378 The fitter to use for minimization.
1379 model : `lsst.jointcal.Model`
1380 The model being fit.
1381 chi2Label : `str`
1382 Label to describe the chi2 (e.g. "Initialized", "Final").
1383 writeChi2Name : `str`, optional
1384 Filename prefix to write the chi2 contributions to.
1385 Do not supply an extension: an appropriate one will be added.
1386
1387 Returns
1388 -------
1390 The chi2 object for the current fitter and model.
1391
1392 Raises
1393 ------
1394 FloatingPointError
1395 Raised if chi2 is infinite or NaN.
1396 ValueError
1397 Raised if the model is not valid.
1398 """
1399 if writeChi2Name is not None:
1400 fullpath = self._getDebugPath_getDebugPath(writeChi2Name)
1401 fit.saveChi2Contributions(fullpath+"{type}")
1402 self.log.info("Wrote chi2 contributions files: %s", fullpath)
1403
1404 chi2 = fit.computeChi2()
1405 self.log.info("%s %s", chi2Label, chi2)
1406 self._check_stars_check_stars(associations)
1407 if not np.isfinite(chi2.chi2):
1408 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}')
1409 if not model.validate(associations.getCcdImageList(), chi2.ndof):
1410 raise ValueError("Model is not valid: check log messages for warnings.")
1411 return chi2
1412
1413 def _fit_photometry(self, associations, dataName=None):
1414 """
1415 Fit the photometric data.
1416
1417 Parameters
1418 ----------
1419 associations : `lsst.jointcal.Associations`
1420 The star/reference star associations to fit.
1421 dataName : `str`
1422 Name of the data being processed (e.g. "1234_HSC-Y"), for
1423 identifying debugging files.
1424
1425 Returns
1426 -------
1427 fit_result : `namedtuple`
1429 The photometric fitter used to perform the fit.
1431 The photometric model that was fit.
1432 """
1433 self.log.info("=== Starting photometric fitting...")
1434
1435 # TODO: should use pex.config.RegistryField here (see DM-9195)
1436 if self.config.photometryModel == "constrainedFlux":
1437 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
1438 self.focalPlaneBBoxfocalPlaneBBox,
1439 visitOrder=self.config.photometryVisitOrder,
1440 errorPedestal=self.config.photometryErrorPedestal)
1441 # potentially nonlinear problem, so we may need a line search to converge.
1442 doLineSearch = self.config.allowLineSearch
1443 elif self.config.photometryModel == "constrainedMagnitude":
1444 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
1445 self.focalPlaneBBoxfocalPlaneBBox,
1446 visitOrder=self.config.photometryVisitOrder,
1447 errorPedestal=self.config.photometryErrorPedestal)
1448 # potentially nonlinear problem, so we may need a line search to converge.
1449 doLineSearch = self.config.allowLineSearch
1450 elif self.config.photometryModel == "simpleFlux":
1451 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
1452 errorPedestal=self.config.photometryErrorPedestal)
1453 doLineSearch = False # purely linear in model parameters, so no line search needed
1454 elif self.config.photometryModel == "simpleMagnitude":
1455 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
1456 errorPedestal=self.config.photometryErrorPedestal)
1457 doLineSearch = False # purely linear in model parameters, so no line search needed
1458
1459 fit = lsst.jointcal.PhotometryFit(associations, model)
1460 # TODO DM-12446: turn this into a "butler save" somehow.
1461 # Save reference and measurement chi2 contributions for this data
1462 if self.config.writeChi2FilesInitialFinal:
1463 baseName = f"photometry_initial_chi2-{dataName}"
1464 else:
1465 baseName = None
1466 if self.config.writeInitialModel:
1467 fullpath = self._getDebugPath_getDebugPath(f"initial_photometry_model-{dataName}.txt")
1468 writeModel(model, fullpath, self.log)
1469 self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
1470
1471 def getChi2Name(whatToFit):
1472 if self.config.writeChi2FilesOuterLoop:
1473 return f"photometry_init-%s_chi2-{dataName}" % whatToFit
1474 else:
1475 return None
1476
1477 # The constrained model needs the visit transform fit first; the chip
1478 # transform is initialized from the singleFrame PhotoCalib, so it's close.
1479 if self.config.writeInitMatrix:
1480 dumpMatrixFile = self._getDebugPath_getDebugPath(f"photometry_preinit-{dataName}")
1481 else:
1482 dumpMatrixFile = ""
1483 if self.config.photometryModel.startswith("constrained"):
1484 # no line search: should be purely (or nearly) linear,
1485 # and we want a large step size to initialize with.
1486 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
1487 self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize ModelVisit",
1488 writeChi2Name=getChi2Name("ModelVisit"))
1489 dumpMatrixFile = "" # so we don't redo the output on the next step
1490
1491 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1492 self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Model",
1493 writeChi2Name=getChi2Name("Model"))
1494
1495 fit.minimize("Fluxes") # no line search: always purely linear.
1496 self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Fluxes",
1497 writeChi2Name=getChi2Name("Fluxes"))
1498
1499 fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
1500 self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes",
1501 writeChi2Name=getChi2Name("ModelFluxes"))
1502
1503 model.freezeErrorTransform()
1504 self.log.debug("Photometry error scales are frozen.")
1505
1506 chi2 = self._iterate_fit_iterate_fit(associations,
1507 fit,
1508 self.config.maxPhotometrySteps,
1509 "photometry",
1510 "Model Fluxes",
1511 doRankUpdate=self.config.photometryDoRankUpdate,
1512 doLineSearch=doLineSearch,
1513 dataName=dataName)
1514
1515 add_measurement(self.jobjob, 'jointcal.photometry_final_chi2', chi2.chi2)
1516 add_measurement(self.jobjob, 'jointcal.photometry_final_ndof', chi2.ndof)
1517 return Photometry(fit, model)
1518
1519 def _fit_astrometry(self, associations, dataName=None):
1520 """
1521 Fit the astrometric data.
1522
1523 Parameters
1524 ----------
1525 associations : `lsst.jointcal.Associations`
1526 The star/reference star associations to fit.
1527 dataName : `str`
1528 Name of the data being processed (e.g. "1234_HSC-Y"), for
1529 identifying debugging files.
1530
1531 Returns
1532 -------
1533 fit_result : `namedtuple`
1535 The astrometric fitter used to perform the fit.
1537 The astrometric model that was fit.
1538 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
1539 The model for the sky to tangent plane projection that was used in the fit.
1540 """
1541
1542 self.log.info("=== Starting astrometric fitting...")
1543
1544 associations.deprojectFittedStars()
1545
1546 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
1547 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
1548 # them so carefully?
1549 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
1550
1551 if self.config.astrometryModel == "constrained":
1552 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
1553 sky_to_tan_projection,
1554 chipOrder=self.config.astrometryChipOrder,
1555 visitOrder=self.config.astrometryVisitOrder)
1556 elif self.config.astrometryModel == "simple":
1557 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
1558 sky_to_tan_projection,
1559 self.config.useInputWcs,
1560 nNotFit=0,
1561 order=self.config.astrometrySimpleOrder)
1562
1563 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
1564 # TODO DM-12446: turn this into a "butler save" somehow.
1565 # Save reference and measurement chi2 contributions for this data
1566 if self.config.writeChi2FilesInitialFinal:
1567 baseName = f"astrometry_initial_chi2-{dataName}"
1568 else:
1569 baseName = None
1570 if self.config.writeInitialModel:
1571 fullpath = self._getDebugPath_getDebugPath(f"initial_astrometry_model-{dataName}.txt")
1572 writeModel(model, fullpath, self.log)
1573 self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
1574
1575 def getChi2Name(whatToFit):
1576 if self.config.writeChi2FilesOuterLoop:
1577 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
1578 else:
1579 return None
1580
1581 if self.config.writeInitMatrix:
1582 dumpMatrixFile = self._getDebugPath_getDebugPath(f"astrometry_preinit-{dataName}")
1583 else:
1584 dumpMatrixFile = ""
1585 # The constrained model needs the visit transform fit first; the chip
1586 # transform is initialized from the detector's cameraGeom, so it's close.
1587 if self.config.astrometryModel == "constrained":
1588 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1589 self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit",
1590 writeChi2Name=getChi2Name("DistortionsVisit"))
1591 dumpMatrixFile = "" # so we don't redo the output on the next step
1592
1593 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
1594 self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Distortions",
1595 writeChi2Name=getChi2Name("Distortions"))
1596
1597 fit.minimize("Positions")
1598 self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Positions",
1599 writeChi2Name=getChi2Name("Positions"))
1600
1601 fit.minimize("Distortions Positions")
1602 self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions",
1603 writeChi2Name=getChi2Name("DistortionsPositions"))
1604
1605 chi2 = self._iterate_fit_iterate_fit(associations,
1606 fit,
1607 self.config.maxAstrometrySteps,
1608 "astrometry",
1609 "Distortions Positions",
1610 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance,
1611 doRankUpdate=self.config.astrometryDoRankUpdate,
1612 dataName=dataName)
1613
1614 add_measurement(self.jobjob, 'jointcal.astrometry_final_chi2', chi2.chi2)
1615 add_measurement(self.jobjob, 'jointcal.astrometry_final_ndof', chi2.ndof)
1616
1617 return Astrometry(fit, model, sky_to_tan_projection)
1618
1619 def _check_stars(self, associations):
1620 """Count measured and reference stars per ccd and warn/log them."""
1621 for ccdImage in associations.getCcdImageList():
1622 nMeasuredStars, nRefStars = ccdImage.countStars()
1623 self.log.debug("ccdImage %s has %s measured and %s reference stars",
1624 ccdImage.getName(), nMeasuredStars, nRefStars)
1625 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1626 self.log.warning("ccdImage %s has only %s measuredStars (desired %s)",
1627 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1628 if nRefStars < self.config.minRefStarsPerCcd:
1629 self.log.warning("ccdImage %s has only %s RefStars (desired %s)",
1630 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1631
1632 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1633 dataName="",
1634 sigmaRelativeTolerance=0,
1635 doRankUpdate=True,
1636 doLineSearch=False):
1637 """Run fitter.minimize up to max_steps times, returning the final chi2.
1638
1639 Parameters
1640 ----------
1641 associations : `lsst.jointcal.Associations`
1642 The star/reference star associations to fit.
1643 fitter : `lsst.jointcal.FitterBase`
1644 The fitter to use for minimization.
1645 max_steps : `int`
1646 Maximum number of steps to run outlier rejection before declaring
1647 convergence failure.
1648 name : {'photometry' or 'astrometry'}
1649 What type of data are we fitting (for logs and debugging files).
1650 whatToFit : `str`
1651 Passed to ``fitter.minimize()`` to define the parameters to fit.
1652 dataName : `str`, optional
1653 Descriptive name for this dataset (e.g. tract and filter),
1654 for debugging.
1655 sigmaRelativeTolerance : `float`, optional
1656 Convergence tolerance for the fractional change in the chi2 cut
1657 level for determining outliers. If set to zero, iterations will
1658 continue until there are no outliers.
1659 doRankUpdate : `bool`, optional
1660 Do an Eigen rank update during minimization, or recompute the full
1661 matrix and gradient?
1662 doLineSearch : `bool`, optional
1663 Do a line search for the optimum step during minimization?
1664
1665 Returns
1666 -------
1668 The final chi2 after the fit converges, or is forced to end.
1669
1670 Raises
1671 ------
1672 FloatingPointError
1673 Raised if the fitter fails with a non-finite value.
1674 RuntimeError
1675 Raised if the fitter fails for some other reason;
1676 log messages will provide further details.
1677 """
1678 if self.config.writeInitMatrix:
1679 dumpMatrixFile = self._getDebugPath_getDebugPath(f"{name}_postinit-{dataName}")
1680 else:
1681 dumpMatrixFile = ""
1682 oldChi2 = lsst.jointcal.Chi2Statistic()
1683 oldChi2.chi2 = float("inf")
1684 for i in range(max_steps):
1685 if self.config.writeChi2FilesOuterLoop:
1686 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
1687 else:
1688 writeChi2Name = None
1689 result = fitter.minimize(whatToFit,
1690 self.config.outlierRejectSigma,
1691 sigmaRelativeTolerance=sigmaRelativeTolerance,
1692 doRankUpdate=doRankUpdate,
1693 doLineSearch=doLineSearch,
1694 dumpMatrixFile=dumpMatrixFile)
1695 dumpMatrixFile = "" # clear it so we don't write the matrix again.
1696 chi2 = self._logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(),
1697 f"Fit iteration {i}", writeChi2Name=writeChi2Name)
1698
1699 if result == MinimizeResult.Converged:
1700 if doRankUpdate:
1701 self.log.debug("fit has converged - no more outliers - redo minimization "
1702 "one more time in case we have lost accuracy in rank update.")
1703 # Redo minimization one more time in case we have lost accuracy in rank update
1704 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma,
1705 sigmaRelativeTolerance=sigmaRelativeTolerance)
1706 chi2 = self._logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1707
1708 # log a message for a large final chi2, TODO: DM-15247 for something better
1709 if chi2.chi2/chi2.ndof >= 4.0:
1710 self.log.error("Potentially bad fit: High chi-squared/ndof.")
1711
1712 break
1713 elif result == MinimizeResult.Chi2Increased:
1714 self.log.warning("Still some outliers remaining but chi2 increased - retry")
1715 # Check whether the increase was large enough to cause trouble.
1716 chi2Ratio = chi2.chi2 / oldChi2.chi2
1717 if chi2Ratio > 1.5:
1718 self.log.warning('Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1719 chi2.chi2, oldChi2.chi2, chi2Ratio)
1720 # Based on a variety of HSC jointcal logs (see DM-25779), it
1721 # appears that chi2 increases more than a factor of ~2 always
1722 # result in the fit diverging rapidly and ending at chi2 > 1e10.
1723 # Using 10 as the "failure" threshold gives some room between
1724 # leaving a warning and bailing early.
1725 if chi2Ratio > 10:
1726 msg = ("Large chi2 increase between steps: fit likely cannot converge."
1727 " Try setting one or more of the `writeChi2*` config fields and looking"
1728 " at how individual star chi2-values evolve during the fit.")
1729 raise RuntimeError(msg)
1730 oldChi2 = chi2
1731 elif result == MinimizeResult.NonFinite:
1732 filename = self._getDebugPath_getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1733 # TODO DM-12446: turn this into a "butler save" somehow.
1734 fitter.saveChi2Contributions(filename+"{type}")
1735 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1736 raise FloatingPointError(msg.format(filename))
1737 elif result == MinimizeResult.Failed:
1738 raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1739 else:
1740 raise RuntimeError("Unxepected return code from minimize().")
1741 else:
1742 self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1743
1744 return chi2
1745
1746 def _make_output(self, ccdImageList, model, func):
1747 """Return the internal jointcal models converted to the afw
1748 structures that will be saved to disk.
1749
1750 Parameters
1751 ----------
1752 ccdImageList : `lsst.jointcal.CcdImageList`
1753 The list of CcdImages to get the output for.
1755 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`.
1756 func : `str`
1757 The name of the function to call on ``model`` to get the converted
1758 structure. Must accept an `lsst.jointcal.CcdImage`.
1759
1760 Returns
1761 -------
1762 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or
1763 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`]
1764 The data to be saved, keyed on (visit, detector).
1765 """
1766 output = {}
1767 for ccdImage in ccdImageList:
1768 ccd = ccdImage.ccdId
1769 visit = ccdImage.visit
1770 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd)
1771 output[(visit, ccd)] = getattr(model, func)(ccdImage)
1772 return output
1773
1774 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1775 """
1776 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1777
1778 Parameters
1779 ----------
1780 associations : `lsst.jointcal.Associations`
1781 The star/reference star associations to fit.
1783 The astrometric model that was fit.
1784 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1785 Dict of ccdImage identifiers to dataRefs that were fit.
1786 """
1787 ccdImageList = associations.getCcdImageList()
1788 output = self._make_output_make_output(ccdImageList, model, "makeSkyWcs")
1789 for key, skyWcs in output.items():
1790 dataRef = visit_ccd_to_dataRef[key]
1791 try:
1792 dataRef.put(skyWcs, 'jointcal_wcs')
1793 except pexExceptions.Exception as e:
1794 self.log.fatal('Failed to write updated Wcs: %s', str(e))
1795 raise e
1796
1797 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1798 """
1799 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1800
1801 Parameters
1802 ----------
1803 associations : `lsst.jointcal.Associations`
1804 The star/reference star associations to fit.
1806 The photoometric model that was fit.
1807 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1808 Dict of ccdImage identifiers to dataRefs that were fit.
1809 """
1810
1811 ccdImageList = associations.getCcdImageList()
1812 output = self._make_output_make_output(ccdImageList, model, "toPhotoCalib")
1813 for key, photoCalib in output.items():
1814 dataRef = visit_ccd_to_dataRef[key]
1815 try:
1816 dataRef.put(photoCalib, 'jointcal_photoCalib')
1817 except pexExceptions.Exception as e:
1818 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1819 raise e
1820
1821
1823 """Return an afw SourceTable to use as a base for creating the
1824 SourceCatalog to insert values from the dataFrame into.
1825
1826 Returns
1827 -------
1829 Table with schema and slots to use to make SourceCatalogs.
1830 """
1832 schema.addField("centroid_x", "D")
1833 schema.addField("centroid_y", "D")
1834 schema.addField("centroid_xErr", "F")
1835 schema.addField("centroid_yErr", "F")
1836 schema.addField("shape_xx", "D")
1837 schema.addField("shape_yy", "D")
1838 schema.addField("shape_xy", "D")
1839 schema.addField("flux_instFlux", "D")
1840 schema.addField("flux_instFluxErr", "D")
1841 table = lsst.afw.table.SourceTable.make(schema)
1842 table.defineCentroid("centroid")
1843 table.defineShape("shape")
1844 return table
1845
1846
1847def get_sourceTable_visit_columns(inColumns, config, sourceSelector):
1848 """
1849 Get the sourceTable_visit columns to load from the catalogs.
1850
1851 Parameters
1852 ----------
1853 inColumns : `list`
1854 List of columns known to be available in the sourceTable_visit.
1855 config : `JointcalConfig`
1856 A filled-in config to to help define column names.
1857 sourceSelector : `lsst.meas.algorithms.BaseSourceSelectorTask`
1858 A configured source selector to define column names to load.
1859
1860 Returns
1861 -------
1862 columns : `list`
1863 List of columns to read from sourceTable_visit.
1864 detectorColumn : `str`
1865 Name of the detector column.
1866 ixxColumns : `list`
1867 Name of the ixx/iyy/ixy columns.
1868 """
1869 if 'detector' in inColumns:
1870 # Default name for Gen3.
1871 detectorColumn = 'detector'
1872 else:
1873 # Default name for Gen2 and Gen2 conversions.
1874 detectorColumn = 'ccd'
1875
1876 columns = ['visit', detectorColumn,
1877 'sourceId', 'x', 'xErr', 'y', 'yErr',
1878 config.sourceFluxType + '_instFlux', config.sourceFluxType + '_instFluxErr']
1879
1880 if 'ixx' in inColumns:
1881 # New columns post-DM-31825
1882 ixxColumns = ['ixx', 'iyy', 'ixy']
1883 else:
1884 # Old columns pre-DM-31825
1885 ixxColumns = ['Ixx', 'Iyy', 'Ixy']
1886 columns.extend(ixxColumns)
1887
1888 if sourceSelector.config.doFlags:
1889 columns.extend(sourceSelector.config.flags.bad)
1890 if sourceSelector.config.doUnresolved:
1891 columns.append(sourceSelector.config.unresolved.name)
1892 if sourceSelector.config.doIsolated:
1893 columns.append(sourceSelector.config.isolated.parentName)
1894 columns.append(sourceSelector.config.isolated.nChildName)
1895
1896 return columns, detectorColumn, ixxColumns
1897
1898
1899def extract_detector_catalog_from_visit_catalog(table, visitCatalog, detectorId,
1900 detectorColumn, ixxColumns, sourceFluxType, log):
1901 """Return an afw SourceCatalog extracted from a visit-level dataframe,
1902 limited to just one detector.
1903
1904 Parameters
1905 ----------
1907 Table factory to use to make the SourceCatalog that will be
1908 populated with data from ``visitCatalog``.
1909 visitCatalog : `pandas.DataFrame`
1910 DataFrame to extract a detector catalog from.
1911 detectorId : `int`
1912 Numeric id of the detector to extract from ``visitCatalog``.
1913 detectorColumn : `str`
1914 Name of the detector column in the catalog.
1915 ixxColumns : `list` [`str`]
1916 Names of the ixx/iyy/ixy columns in the catalog.
1917 sourceFluxType : `str`
1918 Name of the catalog field to load instFluxes from.
1919 log : `lsst.log.Log`
1920 Logging instance to log to.
1921
1922 Returns
1923 -------
1924 catalog : `lsst.afw.table.SourceCatalog`, or `None`
1925 Detector-level catalog extracted from ``visitCatalog``, or `None`
1926 if there was no data to load.
1927 """
1928 # map from dataFrame column to afw table column
1929 mapping = {'x': 'centroid_x',
1930 'y': 'centroid_y',
1931 'xErr': 'centroid_xErr',
1932 'yErr': 'centroid_yErr',
1933 ixxColumns[0]: 'shape_xx',
1934 ixxColumns[1]: 'shape_yy',
1935 ixxColumns[2]: 'shape_xy',
1936 f'{sourceFluxType}_instFlux': 'flux_instFlux',
1937 f'{sourceFluxType}_instFluxErr': 'flux_instFluxErr',
1938 }
1939
1940 catalog = lsst.afw.table.SourceCatalog(table)
1941 matched = visitCatalog[detectorColumn] == detectorId
1942 n = sum(matched)
1943 if n == 0:
1944 return None
1945 catalog.resize(sum(matched))
1946 view = visitCatalog.loc[matched]
1947 catalog['id'] = view.index
1948 for dfCol, afwCol in mapping.items():
1949 catalog[afwCol] = view[dfCol]
1950
1951 log.debug("%d sources selected in visit %d detector %d",
1952 len(catalog),
1953 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first
1954 detectorId)
1955 return catalog
static std::shared_ptr< SourceTable > make(Schema const &schema, std::shared_ptr< IdFactory > const &idFactory)
static Schema makeMinimalSchema()
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:54
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:78
Interface between AstrometryFit and the combinations of Mappings from pixels to some tangent plane (a...
Handler of an actual image from a single CCD.
Definition: CcdImage.h:64
Base class for Chi2Statistic and Chi2List, to allow addEntry inside Fitter for either class.
Definition: Chi2.h:44
Simple structure to accumulate chi2 and ndof.
Definition: Chi2.h:52
A multi-component model, fitting mappings for sensors and visits simultaneously.
Base class for fitters.
Definition: FitterBase.h:53
A projection handler in which all CCDs from the same visit have the same tangent point.
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:45
A model where there is one independent transform per CcdImage.
def getTargetList(parsedCmd, **kwargs)
Definition: jointcal.py:78
def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, jointcalControl, camera)
Definition: jointcal.py:852
def _compute_proper_motion_epoch(self, ccdImageList)
Definition: jointcal.py:1197
def _get_refcat_coordinate_error_override(self, refCat, name)
Definition: jointcal.py:1150
def _getDebugPath(self, filename)
Definition: jointcal.py:1048
def _check_star_lists(self, associations, name)
Definition: jointcal.py:1361
def _make_one_input_data(self, visitRecord, catalog, detectorDict)
Definition: jointcal.py:925
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", sigmaRelativeTolerance=0, doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:1636
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1774
def _check_stars(self, associations)
Definition: jointcal.py:1619
def _prep_sky(self, associations, filters)
Definition: jointcal.py:1053
def _readDataId(self, butler, dataId)
Definition: jointcal.py:998
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: jointcal.py:694
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, tract="", match_cut=3.0, reject_bad_fluxes=False, *name="", refObjLoader=None, referenceSelector=None, fit_function=None, epoch=None)
Definition: jointcal.py:1218
def _put_metrics(self, butlerQC, job, outputRefs)
Definition: jointcal.py:723
def _build_ccdImage(self, data, associations, jointcalControl)
Definition: jointcal.py:953
def runDataRef(self, dataRefs)
Definition: jointcal.py:1076
def __init__(self, butler=None, initInputs=None, **kwargs)
Definition: jointcal.py:673
def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None)
Definition: jointcal.py:798
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:1413
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1797
def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None)
Definition: jointcal.py:1370
def _make_output(self, ccdImageList, model, func)
Definition: jointcal.py:1746
def _put_output(self, butlerQC, outputs, outputRefs, camera, setter)
Definition: jointcal.py:739
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, applyColorterms=False, epoch=None)
Definition: jointcal.py:1301
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:1519
def loadData(self, dataRefs, associations, jointcalControl)
Definition: jointcal.py:1024
def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
Definition: jointcal.py:142
def add_measurement(job, name, value)
Definition: jointcal.py:60
def get_sourceTable_visit_columns(inColumns, config, sourceSelector)
Definition: jointcal.py:1847
def lookupVisitRefCats(datasetType, registry, quantumDataId, collections)
Definition: jointcal.py:182
def extract_detector_catalog_from_visit_catalog(table, visitCatalog, detectorId, detectorColumn, ixxColumns, sourceFluxType, log)
Definition: jointcal.py:1900
def writeModel(model, filename, log)
Definition: jointcal.py:627
This is a virtual class that allows a lot of freedom in the choice of the projection from "Sky" (wher...