28 import astropy.units
as u
39 from lsst.obs.base
import Instrument
40 from lsst.pipe.tasks.colorterms
import ColortermLibrary
41 from lsst.verify
import Job, Measurement
44 ReferenceObjectLoader)
47 from .dataIds
import PerTractCcdDataIdContainer
52 __all__ = [
"JointcalConfig",
"JointcalRunner",
"JointcalTask"]
54 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
55 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
60 meas = Measurement(job.metrics[name], value)
61 job.measurements.insert(meas)
65 """Subclass of TaskRunner for jointcalTask (gen2)
67 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
68 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
69 single dataRef, are are called repeatedly). This class transforms the processed
70 arguments generated by the ArgumentParser into the arguments expected by
71 Jointcal.runDataRef().
73 See pipeBase.TaskRunner for more information.
79 Return a list of tuples per tract, each containing (dataRefs, kwargs).
81 Jointcal operates on lists of dataRefs simultaneously.
83 kwargs[
'butler'] = parsedCmd.butler
87 for ref
in parsedCmd.id.refList:
88 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
90 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
98 Arguments for Task.runDataRef()
103 if self.doReturnResults is False:
105 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
107 if self.doReturnResults is True:
109 - ``result``: the result of calling jointcal.runDataRef()
110 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
115 dataRefList, kwargs = args
116 butler = kwargs.pop(
'butler')
117 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
120 result = task.runDataRef(dataRefList, **kwargs)
121 exitStatus = result.exitStatus
122 job_path = butler.get(
'verify_job_filename')
123 result.job.write(job_path[0])
124 except Exception
as e:
129 eName = type(e).__name__
130 tract = dataRefList[0].dataId[
'tract']
131 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
134 kwargs[
'butler'] = butler
135 if self.doReturnResults:
136 return pipeBase.Struct(result=result, exitStatus=exitStatus)
138 return pipeBase.Struct(exitStatus=exitStatus)
142 """Lookup function that asserts/hopes that a static calibration dataset
143 exists in a particular collection, since this task can't provide a single
144 date/time to use to search for one properly.
146 This is mostly useful for the ``camera`` dataset, in cases where the task's
147 quantum dimensions do *not* include something temporal, like ``exposure``
152 datasetType : `lsst.daf.butler.DatasetType`
153 Type of dataset being searched for.
154 registry : `lsst.daf.butler.Registry`
155 Data repository registry to search.
156 quantumDataId : `lsst.daf.butler.DataCoordinate`
157 Data ID of the quantum this camera should match.
158 collections : `Iterable` [ `str` ]
159 Collections that should be searched - but this lookup function works
160 by ignoring this in favor of a more-or-less hard-coded value.
164 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
165 Iterator over dataset references; should have only one element.
169 This implementation duplicates one in fgcmcal, and is at least quite
170 similar to another in cp_pipe. This duplicate has the most documentation.
171 Fixing this is DM-29661.
173 instrument = Instrument.fromName(quantumDataId[
"instrument"], registry)
174 unboundedCollection = instrument.makeUnboundedCalibrationRunName()
175 return registry.queryDatasets(datasetType,
176 dataId=quantumDataId,
177 collections=[unboundedCollection],
182 """Lookup function that finds all refcats for all visits that overlap a
183 tract, rather than just the refcats that directly overlap the tract.
187 datasetType : `lsst.daf.butler.DatasetType`
188 Type of dataset being searched for.
189 registry : `lsst.daf.butler.Registry`
190 Data repository registry to search.
191 quantumDataId : `lsst.daf.butler.DataCoordinate`
192 Data ID of the quantum; expected to be something we can use as a
193 constraint to query for overlapping visits.
194 collections : `Iterable` [ `str` ]
195 Collections to search.
199 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
200 Iterator over refcat references.
209 for visit_data_id
in set(registry.queryDataIds(
"visit", dataId=quantumDataId).expanded()):
211 registry.queryDatasets(
213 collections=collections,
214 dataId=visit_data_id,
222 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter")):
223 """Middleware input/output connections for jointcal data."""
224 inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
225 doc=
"The camera instrument that took these observations.",
227 storageClass=
"Camera",
228 dimensions=(
"instrument",),
230 lookupFunction=lookupStaticCalibrations,
232 inputSourceTableVisit = pipeBase.connectionTypes.Input(
233 doc=
"Source table in parquet format, per visit",
234 name=
"sourceTable_visit",
235 storageClass=
"DataFrame",
236 dimensions=(
"instrument",
"visit"),
240 inputVisitSummary = pipeBase.connectionTypes.Input(
241 doc=(
"Per-visit consolidated exposure metadata built from calexps. "
242 "These catalogs use detector id for the id and must be sorted for "
243 "fast lookups of a detector."),
245 storageClass=
"ExposureCatalog",
246 dimensions=(
"instrument",
"visit"),
250 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
251 doc=
"The astrometry reference catalog to match to loaded input catalog sources.",
252 name=
"gaia_dr2_20200414",
253 storageClass=
"SimpleCatalog",
254 dimensions=(
"skypix",),
257 lookupFunction=lookupVisitRefCats,
259 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
260 doc=
"The photometry reference catalog to match to loaded input catalog sources.",
261 name=
"ps1_pv3_3pi_20170110",
262 storageClass=
"SimpleCatalog",
263 dimensions=(
"skypix",),
266 lookupFunction=lookupVisitRefCats,
269 outputWcs = pipeBase.connectionTypes.Output(
270 doc=(
"Per-tract, per-visit world coordinate systems derived from the fitted model."
271 " These catalogs only contain entries for detectors with an output, and use"
272 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
273 name=
"jointcalSkyWcsCatalog",
274 storageClass=
"ExposureCatalog",
275 dimensions=(
"instrument",
"visit",
"skymap",
"tract"),
278 outputPhotoCalib = pipeBase.connectionTypes.Output(
279 doc=(
"Per-tract, per-visit photometric calibrations derived from the fitted model."
280 " These catalogs only contain entries for detectors with an output, and use"
281 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
282 name=
"jointcalPhotoCalibCatalog",
283 storageClass=
"ExposureCatalog",
284 dimensions=(
"instrument",
"visit",
"skymap",
"tract"),
296 if not config.doAstrometry:
297 self.prerequisiteInputs.remove(
"astrometryRefCat")
298 self.outputs.remove(
"outputWcs")
299 if not config.doPhotometry:
300 self.prerequisiteInputs.remove(
"photometryRefCat")
301 self.outputs.remove(
"outputPhotoCalib")
305 pipelineConnections=JointcalTaskConnections):
306 """Configuration for JointcalTask"""
308 doAstrometry = pexConfig.Field(
309 doc=
"Fit astrometry and write the fitted result.",
313 doPhotometry = pexConfig.Field(
314 doc=
"Fit photometry and write the fitted result.",
318 coaddName = pexConfig.Field(
319 doc=
"Type of coadd, typically deep or goodSeeing",
324 sourceFluxType = pexConfig.Field(
326 doc=
"Source flux field to use in source selection and to get fluxes from the catalog.",
329 positionErrorPedestal = pexConfig.Field(
330 doc=
"Systematic term to apply to the measured position error (pixels)",
334 photometryErrorPedestal = pexConfig.Field(
335 doc=
"Systematic term to apply to the measured error on flux or magnitude as a "
336 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
341 matchCut = pexConfig.Field(
342 doc=
"Matching radius between fitted and reference stars (arcseconds)",
346 minMeasurements = pexConfig.Field(
347 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
351 minMeasuredStarsPerCcd = pexConfig.Field(
352 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
356 minRefStarsPerCcd = pexConfig.Field(
357 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
361 allowLineSearch = pexConfig.Field(
362 doc=
"Allow a line search during minimization, if it is reasonable for the model"
363 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
367 astrometrySimpleOrder = pexConfig.Field(
368 doc=
"Polynomial order for fitting the simple astrometry model.",
372 astrometryChipOrder = pexConfig.Field(
373 doc=
"Order of the per-chip transform for the constrained astrometry model.",
377 astrometryVisitOrder = pexConfig.Field(
378 doc=
"Order of the per-visit transform for the constrained astrometry model.",
382 useInputWcs = pexConfig.Field(
383 doc=
"Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
387 astrometryModel = pexConfig.ChoiceField(
388 doc=
"Type of model to fit to astrometry",
390 default=
"constrained",
391 allowed={
"simple":
"One polynomial per ccd",
392 "constrained":
"One polynomial per ccd, and one polynomial per visit"}
394 photometryModel = pexConfig.ChoiceField(
395 doc=
"Type of model to fit to photometry",
397 default=
"constrainedMagnitude",
398 allowed={
"simpleFlux":
"One constant zeropoint per ccd and visit, fitting in flux space.",
399 "constrainedFlux":
"Constrained zeropoint per ccd, and one polynomial per visit,"
400 " fitting in flux space.",
401 "simpleMagnitude":
"One constant zeropoint per ccd and visit,"
402 " fitting in magnitude space.",
403 "constrainedMagnitude":
"Constrained zeropoint per ccd, and one polynomial per visit,"
404 " fitting in magnitude space.",
407 applyColorTerms = pexConfig.Field(
408 doc=
"Apply photometric color terms to reference stars?"
409 "Requires that colorterms be set to a ColortermLibrary",
413 colorterms = pexConfig.ConfigField(
414 doc=
"Library of photometric reference catalog name to color term dict.",
415 dtype=ColortermLibrary,
417 photometryVisitOrder = pexConfig.Field(
418 doc=
"Order of the per-visit polynomial transform for the constrained photometry model.",
422 photometryDoRankUpdate = pexConfig.Field(
423 doc=(
"Do the rank update step during minimization. "
424 "Skipping this can help deal with models that are too non-linear."),
428 astrometryDoRankUpdate = pexConfig.Field(
429 doc=(
"Do the rank update step during minimization (should not change the astrometry fit). "
430 "Skipping this can help deal with models that are too non-linear."),
434 outlierRejectSigma = pexConfig.Field(
435 doc=
"How many sigma to reject outliers at during minimization.",
439 maxPhotometrySteps = pexConfig.Field(
440 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
444 maxAstrometrySteps = pexConfig.Field(
445 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
449 astrometryRefObjLoader = pexConfig.ConfigurableField(
450 target=LoadIndexedReferenceObjectsTask,
451 doc=
"Reference object loader for astrometric fit",
453 photometryRefObjLoader = pexConfig.ConfigurableField(
454 target=LoadIndexedReferenceObjectsTask,
455 doc=
"Reference object loader for photometric fit",
457 sourceSelector = sourceSelectorRegistry.makeField(
458 doc=
"How to select sources for cross-matching",
461 astrometryReferenceSelector = pexConfig.ConfigurableField(
462 target=ReferenceSourceSelectorTask,
463 doc=
"How to down-select the loaded astrometry reference catalog.",
465 photometryReferenceSelector = pexConfig.ConfigurableField(
466 target=ReferenceSourceSelectorTask,
467 doc=
"How to down-select the loaded photometry reference catalog.",
469 astrometryReferenceErr = pexConfig.Field(
470 doc=(
"Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
471 "If None, then raise an exception if the reference catalog is missing coordinate errors. "
472 "If specified, overrides any existing `coord_*Err` values."),
479 writeInitMatrix = pexConfig.Field(
481 doc=(
"Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
482 "Output files will be written to `config.debugOutputPath` and will "
483 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. "
484 "Note that these files are the dense versions of the matrix, and so may be very large."),
487 writeChi2FilesInitialFinal = pexConfig.Field(
489 doc=(
"Write .csv files containing the contributions to chi2 for the initialization and final fit. "
490 "Output files will be written to `config.debugOutputPath` and will "
491 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER1."),
494 writeChi2FilesOuterLoop = pexConfig.Field(
496 doc=(
"Write .csv files containing the contributions to chi2 for the outer fit loop. "
497 "Output files will be written to `config.debugOutputPath` and will "
498 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."),
501 writeInitialModel = pexConfig.Field(
503 doc=(
"Write the pre-initialization model to text files, for debugging. "
504 "Output files will be written to `config.debugOutputPath` and will be "
505 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."),
508 debugOutputPath = pexConfig.Field(
511 doc=(
"Path to write debug output files to. Used by "
512 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
514 detailedProfile = pexConfig.Field(
517 doc=
"Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
523 msg =
"applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
524 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
526 msg = (
"Only doing astrometry, but Colorterms are not applied for astrometry;"
527 "applyColorTerms=True will be ignored.")
536 self.
sourceSelectorsourceSelector[
'science'].doSignalToNoise =
True
539 self.
sourceSelectorsourceSelector[
'science'].signalToNoise.minimum = 10.
541 fluxField = f
"slot_{self.sourceFluxType}Flux_instFlux"
542 self.
sourceSelectorsourceSelector[
'science'].signalToNoise.fluxField = fluxField
543 self.
sourceSelectorsourceSelector[
'science'].signalToNoise.errField = fluxField +
"Err"
549 badFlags = [
'base_PixelFlags_flag_edge',
'base_PixelFlags_flag_saturated',
550 'base_PixelFlags_flag_interpolatedCenter',
'base_SdssCentroid_flag',
551 'base_PsfFlux_flag',
'base_PixelFlags_flag_suspectCenter']
563 """Write model to outfile."""
564 with open(filename,
"w")
as file:
565 file.write(repr(model))
566 log.info(
"Wrote %s to file: %s", model, filename)
569 @dataclasses.dataclass
571 """The input data jointcal needs for each detector/visit."""
573 """The visit identifier of this exposure."""
575 """The catalog derived from this exposure."""
577 """The VisitInfo of this exposure."""
579 """The detector of this exposure."""
581 """The photometric calibration of this exposure."""
583 """The WCS of this exposure."""
585 """The bounding box of this exposure."""
587 """The filter of this exposure."""
591 """Astrometricly and photometricly calibrate across multiple visits of the
596 butler : `lsst.daf.persistence.Butler`
597 The butler is passed to the refObjLoader constructor in case it is
598 needed. Ignored if the refObjLoader argument provides a loader directly.
599 Used to initialize the astrometry and photometry refObjLoaders.
600 initInputs : `dict`, optional
601 Dictionary used to initialize PipelineTasks (empty for jointcal).
604 ConfigClass = JointcalConfig
605 RunnerClass = JointcalRunner
606 _DefaultName =
"jointcal"
608 def __init__(self, butler=None, initInputs=None, **kwargs):
610 self.makeSubtask(
"sourceSelector")
611 if self.config.doAstrometry:
612 if initInputs
is None:
614 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
615 self.makeSubtask(
"astrometryReferenceSelector")
618 if self.config.doPhotometry:
619 if initInputs
is None:
621 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
622 self.makeSubtask(
"photometryReferenceSelector")
627 self.
jobjob = Job.load_metrics_package(subset=
'jointcal')
632 inputs = butlerQC.get(inputRefs)
634 tract = butlerQC.quantum.dataId[
'tract']
635 if self.config.doAstrometry:
637 dataIds=[ref.datasetRef.dataId
638 for ref
in inputRefs.astrometryRefCat],
639 refCats=inputs.pop(
'astrometryRefCat'),
640 config=self.config.astrometryRefObjLoader,
642 if self.config.doPhotometry:
644 dataIds=[ref.datasetRef.dataId
645 for ref
in inputRefs.photometryRefCat],
646 refCats=inputs.pop(
'photometryRefCat'),
647 config=self.config.photometryRefObjLoader,
649 outputs = self.
runrun(**inputs, tract=tract)
650 if self.config.doAstrometry:
651 self.
_put_output_put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
652 inputs[
'inputCamera'],
"setWcs")
653 if self.config.doPhotometry:
654 self.
_put_output_put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
655 inputs[
'inputCamera'],
"setPhotoCalib")
657 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
658 """Persist the output datasets to their appropriate datarefs.
662 butlerQC : `ButlerQuantumContext`
663 A butler which is specialized to operate in the context of a
664 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
665 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or
666 `dict` [`tuple, `lsst.afw.image.PhotoCalib`]
667 The fitted objects to persist.
668 outputRefs : `list` [`OutputQuantizedConnection`]
669 The DatasetRefs to persist the data to.
670 camera : `lsst.afw.cameraGeom.Camera`
671 The camera for this instrument, to get detector ids from.
673 The method to call on the ExposureCatalog to set each output.
676 schema.addField(
'visit', type=
'I', doc=
'Visit number')
678 def new_catalog(visit, size):
679 """Return an catalog ready to be filled with appropriate output."""
682 catalog[
'visit'] = visit
684 metadata.add(
"COMMENT",
"Catalog id is detector id, sorted.")
685 metadata.add(
"COMMENT",
"Only detectors with data have entries.")
689 detectors_per_visit = collections.defaultdict(int)
692 detectors_per_visit[key[0]] += 1
694 for ref
in outputRefs:
695 visit = ref.dataId[
'visit']
696 catalog = new_catalog(visit, detectors_per_visit[visit])
699 for detector
in camera:
700 detectorId = detector.getId()
701 key = (ref.dataId[
'visit'], detectorId)
702 if key
not in outputs:
704 self.log.debug(
"No %s output for detector %s in visit %s",
705 setter[3:], detectorId, visit)
708 catalog[i].setId(detectorId)
709 getattr(catalog[i], setter)(outputs[key])
713 butlerQC.put(catalog, ref)
714 self.log.info(
"Wrote %s detectors to %s", i, ref)
716 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
722 sourceFluxField =
"flux"
726 oldWcsList, bands = self.
_load_data_load_data(inputSourceTableVisit,
732 boundingCircle, center, radius, defaultFilter = self.
_prep_sky_prep_sky(associations, bands)
735 if self.config.doAstrometry:
739 referenceSelector=self.astrometryReferenceSelector,
743 astrometry_output = self.
_make_output_make_output(associations.getCcdImageList(),
747 astrometry_output =
None
749 if self.config.doPhotometry:
753 referenceSelector=self.photometryReferenceSelector,
757 reject_bad_fluxes=
True)
758 photometry_output = self.
_make_output_make_output(associations.getCcdImageList(),
762 photometry_output =
None
764 return pipeBase.Struct(outputWcs=astrometry_output,
765 outputPhotoCalib=photometry_output,
770 def _make_schema_table(self):
771 """Return an afw SourceTable to use as a base for creating the
772 SourceCatalog to insert values from the dataFrame into.
776 table : `lsst.afw.table.SourceTable`
777 Table with schema and slots to use to make SourceCatalogs.
780 schema.addField(
"centroid_x",
"D")
781 schema.addField(
"centroid_y",
"D")
782 schema.addField(
"centroid_xErr",
"F")
783 schema.addField(
"centroid_yErr",
"F")
784 schema.addField(
"shape_xx",
"D")
785 schema.addField(
"shape_yy",
"D")
786 schema.addField(
"shape_xy",
"D")
787 schema.addField(
"flux_instFlux",
"D")
788 schema.addField(
"flux_instFluxErr",
"D")
790 table.defineCentroid(
"centroid")
791 table.defineShape(
"shape")
794 def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId):
795 """Return an afw SourceCatalog extracted from a visit-level dataframe,
796 limited to just one detector.
800 table : `lsst.afw.table.SourceTable`
801 Table factory to use to make the SourceCatalog that will be
802 populated with data from ``visitCatalog``.
803 visitCatalog : `pandas.DataFrame`
804 DataFrame to extract a detector catalog from.
806 Numeric id of the detector to extract from ``visitCatalog``.
810 catalog : `lsst.afw.table.SourceCatalog`
811 Detector-level catalog extracted from ``visitCatalog``.
814 mapping = {
'sourceId':
'id',
817 'xErr':
'centroid_xErr',
818 'yErr':
'centroid_yErr',
822 f
'{self.config.sourceFluxType}_instFlux':
'flux_instFlux',
823 f
'{self.config.sourceFluxType}_instFluxErr':
'flux_instFluxErr',
831 detector_column =
"detector" if "detector" in visitCatalog.columns
else "ccd"
833 matched = visitCatalog[detector_column] == detectorId
834 catalog.resize(sum(matched))
835 view = visitCatalog.loc[matched]
836 for dfCol, afwCol
in mapping.items():
837 catalog[afwCol] = view[dfCol]
839 self.log.debug(
"%d sources selected in visit %d detector %d",
841 view[
'visit'].iloc[0],
845 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
846 jointcalControl, camera):
847 """Read the data that jointcal needs to run. (Gen3 version)
849 Modifies ``associations`` in-place with the loaded data.
853 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
854 References to visit-level DataFrames to load the catalog data from.
855 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
856 Visit-level exposure summary catalog with metadata.
857 associations : `lsst.jointcal.Associations`
858 Object to add the loaded data to by constructing new CcdImages.
859 jointcalControl : `jointcal.JointcalControl`
860 Control object for C++ associations management.
861 camera : `lsst.afw.cameraGeom.Camera`
862 Camera object for detector geometry.
866 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`]
867 The original WCS of the input data, to aid in writing tests.
868 bands : `list` [`str`]
869 The filter bands of each input dataset.
873 load_cat_prof_file =
'jointcal_load_data.prof' if self.config.detailedProfile
else ''
874 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
877 catalogMap = {ref.dataId[
'visit']: i
for i, ref
in enumerate(inputSourceTableVisit)}
878 detectorDict = {detector.getId(): detector
for detector
in camera}
880 for visitSummaryRef
in inputVisitSummary:
881 visitSummary = visitSummaryRef.get()
882 visitCatalog = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId[
'visit']]].get()
883 selected = self.sourceSelector.
run(visitCatalog)
886 detectors = {id: index
for index, id
in enumerate(visitSummary[
'id'])}
887 for id, index
in detectors.items():
889 data = self.
_make_one_input_data_make_one_input_data(visitSummary[index], catalog, detectorDict)
890 result = self.
_build_ccdImage_build_ccdImage(data, associations, jointcalControl)
891 if result
is not None:
892 oldWcsList.append(result.wcs)
894 filters.append(data.filter)
895 if len(filters) == 0:
896 raise RuntimeError(
"No data to process: did source selector remove all sources?")
897 filters = collections.Counter(filters)
899 return oldWcsList, filters
901 def _make_one_input_data(self, visitRecord, catalog, detectorDict):
902 """Return a data structure for this detector+visit."""
905 visitInfo=visitRecord.getVisitInfo(),
906 detector=detectorDict[visitRecord.getId()],
907 photoCalib=visitRecord.getPhotoCalib(),
908 wcs=visitRecord.getWcs(),
909 bbox=visitRecord.getBBox(),
913 physical=visitRecord[
'physical_filter']))
918 def _getMetadataName(self):
922 def _makeArgumentParser(cls):
923 """Create an argument parser"""
924 parser = pipeBase.ArgumentParser(name=cls.
_DefaultName_DefaultName)
925 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
926 ContainerClass=PerTractCcdDataIdContainer)
929 def _build_ccdImage(self, data, associations, jointcalControl):
931 Extract the necessary things from this catalog+metadata to add a new
936 data : `JointcalInputData`
937 The loaded input data.
938 associations : `lsst.jointcal.Associations`
939 Object to add the info to, to construct a new CcdImage
940 jointcalControl : `jointcal.JointcalControl`
941 Control object for associations management
947 The TAN WCS of this image, read from the calexp
948 (`lsst.afw.geom.SkyWcs`).
950 A key to identify this dataRef by its visit and ccd ids
953 if there are no sources in the loaded catalog.
955 if len(data.catalog) == 0:
956 self.log.warn(
"No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
959 associations.createCcdImage(data.catalog,
963 data.filter.physicalLabel,
967 data.detector.getId(),
970 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key'))
971 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
972 return Result(data.wcs, Key(data.visit, data.detector.getId()))
974 def _readDataId(self, butler, dataId):
975 """Read all of the data for one dataId from the butler. (gen2 version)"""
977 if "visit" in dataId.keys():
978 visit = dataId[
"visit"]
980 visit = butler.getButler().queryMetadata(
"calexp", (
"visit"), butler.dataId)[0]
981 detector = butler.get(
'calexp_detector', dataId=dataId)
983 catalog = butler.get(
'src',
984 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
986 goodSrc = self.sourceSelector.
run(catalog)
987 self.log.debug(
"%d sources selected in visit %d detector %d",
988 len(goodSrc.sourceCat),
992 catalog=goodSrc.sourceCat,
993 visitInfo=butler.get(
'calexp_visitInfo', dataId=dataId),
995 photoCalib=butler.get(
'calexp_photoCalib', dataId=dataId),
996 wcs=butler.get(
'calexp_wcs', dataId=dataId),
997 bbox=butler.get(
'calexp_bbox', dataId=dataId),
998 filter=butler.get(
'calexp_filterLabel', dataId=dataId))
1000 def loadData(self, dataRefs, associations, jointcalControl):
1001 """Read the data that jointcal needs to run. (Gen2 version)"""
1002 visit_ccd_to_dataRef = {}
1005 load_cat_prof_file =
'jointcal_loadData.prof' if self.config.detailedProfile
else ''
1006 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1008 camera = dataRefs[0].get(
'camera', immediate=
True)
1010 for dataRef
in dataRefs:
1011 data = self.
_readDataId_readDataId(dataRef.getButler(), dataRef.dataId)
1012 result = self.
_build_ccdImage_build_ccdImage(data, associations, jointcalControl)
1015 oldWcsList.append(result.wcs)
1016 visit_ccd_to_dataRef[result.key] = dataRef
1017 filters.append(data.filter)
1018 if len(filters) == 0:
1019 raise RuntimeError(
"No data to process: did source selector remove all sources?")
1020 filters = collections.Counter(filters)
1022 return oldWcsList, filters, visit_ccd_to_dataRef
1024 def _getDebugPath(self, filename):
1025 """Constructs a path to filename using the configured debug path.
1027 return os.path.join(self.config.debugOutputPath, filename)
1029 def _prep_sky(self, associations, filters):
1030 """Prepare on-sky and other data that must be computed after data has
1033 associations.computeCommonTangentPoint()
1035 boundingCircle = associations.computeBoundingCircle()
1037 radius =
lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
1039 self.log.info(f
"Data has center={center} with radius={radius.asDegrees()} degrees.")
1042 defaultFilter = filters.most_common(1)[0][0]
1043 self.log.debug(
"Using '%s' filter for reference flux", defaultFilter.physicalLabel)
1045 return boundingCircle, center, radius, defaultFilter
1047 @pipeBase.timeMethod
1050 Jointly calibrate the astrometry and photometry across a set of images.
1052 NOTE: this is for gen2 middleware only.
1056 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
1057 List of data references to the exposures to be fit.
1061 result : `lsst.pipe.base.Struct`
1062 Struct of metadata from the fit, containing:
1065 The provided data references that were fit (with updated WCSs)
1067 The original WCS from each dataRef
1069 Dictionary of internally-computed metrics for testing/validation.
1071 if len(dataRefs) == 0:
1072 raise ValueError(
'Need a non-empty list of data references!')
1076 sourceFluxField =
"slot_%sFlux" % (self.config.sourceFluxType,)
1080 oldWcsList, filters, visit_ccd_to_dataRef = self.
loadDataloadData(dataRefs,
1084 boundingCircle, center, radius, defaultFilter = self.
_prep_sky_prep_sky(associations, filters)
1087 tract = dataRefs[0].dataId[
'tract']
1089 if self.config.doAstrometry:
1093 referenceSelector=self.astrometryReferenceSelector,
1101 if self.config.doPhotometry:
1105 referenceSelector=self.photometryReferenceSelector,
1109 reject_bad_fluxes=
True)
1114 return pipeBase.Struct(dataRefs=dataRefs,
1115 oldWcsList=oldWcsList,
1119 defaultFilter=defaultFilter,
1121 exitStatus=exitStatus)
1123 def _get_refcat_coordinate_error_override(self, refCat, name):
1124 """Check whether we should override the refcat coordinate errors, and
1125 return the overridden error if necessary.
1129 refCat : `lsst.afw.table.SimpleCatalog`
1130 The reference catalog to check for a ``coord_raErr`` field.
1132 Whether we are doing "astrometry" or "photometry".
1136 refCoordErr : `float`
1137 The refcat coordinate error to use, or NaN if we are not overriding
1142 lsst.pex.config.FieldValidationError
1143 Raised if the refcat does not contain coordinate errors and
1144 ``config.astrometryReferenceErr`` is not set.
1148 if name.lower() ==
"photometry":
1149 if 'coord_raErr' not in refCat.schema:
1154 if self.config.astrometryReferenceErr
is None and 'coord_raErr' not in refCat.schema:
1155 msg = (
"Reference catalog does not contain coordinate errors, "
1156 "and config.astrometryReferenceErr not supplied.")
1157 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
1161 if self.config.astrometryReferenceErr
is not None and 'coord_raErr' in refCat.schema:
1162 self.log.warn(
"Overriding reference catalog coordinate errors with %f/coordinate [mas]",
1163 self.config.astrometryReferenceErr)
1165 if self.config.astrometryReferenceErr
is None:
1168 return self.config.astrometryReferenceErr
1170 def _compute_proper_motion_epoch(self, ccdImageList):
1171 """Return the proper motion correction epoch of the provided images.
1175 ccdImageList : `list` [`lsst.jointcal.CcdImage`]
1176 The images to compute the appropriate epoch for.
1180 epoch : `astropy.time.Time`
1181 The date to use for proper motion corrections.
1183 mjds = [ccdImage.getMjd()
for ccdImage
in ccdImageList]
1184 return astropy.time.Time(np.mean(mjds), format=
'mjd', scale=
"tai")
1186 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
1187 tract="", match_cut=3.0,
1188 reject_bad_fluxes=False, *,
1189 name="", refObjLoader=None, referenceSelector=None,
1190 fit_function=None, epoch=None):
1191 """Load reference catalog, perform the fit, and return the result.
1195 associations : `lsst.jointcal.Associations`
1196 The star/reference star associations to fit.
1197 defaultFilter : `lsst.afw.image.FilterLabel`
1198 filter to load from reference catalog.
1199 center : `lsst.geom.SpherePoint`
1200 ICRS center of field to load from reference catalog.
1201 radius : `lsst.geom.Angle`
1202 On-sky radius to load from reference catalog.
1204 Name of thing being fit: "astrometry" or "photometry".
1205 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1206 Reference object loader to use to load a reference catalog.
1207 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1208 Selector to use to pick objects from the loaded reference catalog.
1209 fit_function : callable
1210 Function to call to perform fit (takes Associations object).
1211 tract : `str`, optional
1212 Name of tract currently being fit.
1213 match_cut : `float`, optional
1214 Radius in arcseconds to find cross-catalog matches to during
1215 associations.associateCatalogs.
1216 reject_bad_fluxes : `bool`, optional
1217 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
1218 epoch : `astropy.time.Time`, optional
1219 Epoch to which to correct refcat proper motion and parallax,
1220 or `None` to not apply such corrections.
1224 result : `Photometry` or `Astrometry`
1225 Result of `fit_function()`
1227 self.log.info(
"====== Now processing %s...", name)
1230 associations.associateCatalogs(match_cut)
1232 associations.fittedStarListSize())
1234 applyColorterms =
False if name.lower() ==
"astrometry" else self.config.applyColorTerms
1236 center, radius, defaultFilter,
1237 applyColorterms=applyColorterms,
1241 associations.collectRefStars(refCat,
1242 self.config.matchCut*lsst.geom.arcseconds,
1244 refCoordinateErr=refCoordErr,
1245 rejectBadFluxes=reject_bad_fluxes)
1247 associations.refStarListSize())
1249 associations.prepareFittedStars(self.config.minMeasurements)
1253 associations.nFittedStarsWithAssociatedRefStar())
1255 associations.fittedStarListSize())
1257 associations.nCcdImagesValidForFit())
1259 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if self.config.detailedProfile
else ''
1260 dataName =
"{}_{}".format(tract, defaultFilter.physicalLabel)
1261 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1262 result = fit_function(associations, dataName)
1265 if self.config.writeChi2FilesInitialFinal:
1266 baseName = self.
_getDebugPath_getDebugPath(f
"{name}_final_chi2-{dataName}")
1267 result.fit.saveChi2Contributions(baseName+
"{type}")
1268 self.log.info(
"Wrote chi2 contributions files: %s", baseName)
1272 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1273 applyColorterms=False, epoch=None):
1274 """Load the necessary reference catalog sources, convert fluxes to
1275 correct units, and apply color term corrections if requested.
1279 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1280 The reference catalog loader to use to get the data.
1281 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1282 Source selector to apply to loaded reference catalog.
1283 center : `lsst.geom.SpherePoint`
1284 The center around which to load sources.
1285 radius : `lsst.geom.Angle`
1286 The radius around ``center`` to load sources in.
1287 filterLabel : `lsst.afw.image.FilterLabel`
1288 The camera filter to load fluxes for.
1289 applyColorterms : `bool`
1290 Apply colorterm corrections to the refcat for ``filterName``?
1291 epoch : `astropy.time.Time`, optional
1292 Epoch to which to correct refcat proper motion and parallax,
1293 or `None` to not apply such corrections.
1297 refCat : `lsst.afw.table.SimpleCatalog`
1298 The loaded reference catalog.
1300 The name of the reference catalog flux field appropriate for ``filterName``.
1302 skyCircle = refObjLoader.loadSkyCircle(center,
1304 filterLabel.bandLabel,
1307 selected = referenceSelector.run(skyCircle.refCat)
1309 if not selected.sourceCat.isContiguous():
1310 refCat = selected.sourceCat.copy(deep=
True)
1312 refCat = selected.sourceCat
1315 refCatName = refObjLoader.ref_dataset_name
1316 self.log.info(
"Applying color terms for physical filter=%r reference catalog=%s",
1317 filterLabel.physicalLabel, refCatName)
1318 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1322 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1323 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1325 refCat[skyCircle.fluxField+
'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1327 return refCat, skyCircle.fluxField
1329 def _check_star_lists(self, associations, name):
1331 if associations.nCcdImagesValidForFit() == 0:
1332 raise RuntimeError(
'No images in the ccdImageList!')
1333 if associations.fittedStarListSize() == 0:
1334 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
1335 if associations.refStarListSize() == 0:
1336 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
1338 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1339 """Compute chi2, log it, validate the model, and return chi2.
1343 associations : `lsst.jointcal.Associations`
1344 The star/reference star associations to fit.
1345 fit : `lsst.jointcal.FitterBase`
1346 The fitter to use for minimization.
1347 model : `lsst.jointcal.Model`
1348 The model being fit.
1350 Label to describe the chi2 (e.g. "Initialized", "Final").
1351 writeChi2Name : `str`, optional
1352 Filename prefix to write the chi2 contributions to.
1353 Do not supply an extension: an appropriate one will be added.
1357 chi2: `lsst.jointcal.Chi2Accumulator`
1358 The chi2 object for the current fitter and model.
1363 Raised if chi2 is infinite or NaN.
1365 Raised if the model is not valid.
1367 if writeChi2Name
is not None:
1369 fit.saveChi2Contributions(fullpath+
"{type}")
1370 self.log.info(
"Wrote chi2 contributions files: %s", fullpath)
1372 chi2 = fit.computeChi2()
1373 self.log.info(
"%s %s", chi2Label, chi2)
1375 if not np.isfinite(chi2.chi2):
1376 raise FloatingPointError(f
'{chi2Label} chi2 is invalid: {chi2}')
1377 if not model.validate(associations.getCcdImageList(), chi2.ndof):
1378 raise ValueError(
"Model is not valid: check log messages for warnings.")
1381 def _fit_photometry(self, associations, dataName=None):
1383 Fit the photometric data.
1387 associations : `lsst.jointcal.Associations`
1388 The star/reference star associations to fit.
1390 Name of the data being processed (e.g. "1234_HSC-Y"), for
1391 identifying debugging files.
1395 fit_result : `namedtuple`
1396 fit : `lsst.jointcal.PhotometryFit`
1397 The photometric fitter used to perform the fit.
1398 model : `lsst.jointcal.PhotometryModel`
1399 The photometric model that was fit.
1401 self.log.info(
"=== Starting photometric fitting...")
1404 if self.config.photometryModel ==
"constrainedFlux":
1407 visitOrder=self.config.photometryVisitOrder,
1408 errorPedestal=self.config.photometryErrorPedestal)
1410 doLineSearch = self.config.allowLineSearch
1411 elif self.config.photometryModel ==
"constrainedMagnitude":
1414 visitOrder=self.config.photometryVisitOrder,
1415 errorPedestal=self.config.photometryErrorPedestal)
1417 doLineSearch = self.config.allowLineSearch
1418 elif self.config.photometryModel ==
"simpleFlux":
1420 errorPedestal=self.config.photometryErrorPedestal)
1421 doLineSearch =
False
1422 elif self.config.photometryModel ==
"simpleMagnitude":
1424 errorPedestal=self.config.photometryErrorPedestal)
1425 doLineSearch =
False
1430 if self.config.writeChi2FilesInitialFinal:
1431 baseName = f
"photometry_initial_chi2-{dataName}"
1434 if self.config.writeInitialModel:
1435 fullpath = self.
_getDebugPath_getDebugPath(f
"initial_photometry_model-{dataName}.txt")
1437 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialized", writeChi2Name=baseName)
1439 def getChi2Name(whatToFit):
1440 if self.config.writeChi2FilesOuterLoop:
1441 return f
"photometry_init-%s_chi2-{dataName}" % whatToFit
1447 if self.config.writeInitMatrix:
1448 dumpMatrixFile = self.
_getDebugPath_getDebugPath(f
"photometry_preinit-{dataName}")
1451 if self.config.photometryModel.startswith(
"constrained"):
1454 fit.minimize(
"ModelVisit", dumpMatrixFile=dumpMatrixFile)
1455 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize ModelVisit",
1456 writeChi2Name=getChi2Name(
"ModelVisit"))
1459 fit.minimize(
"Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1461 writeChi2Name=getChi2Name(
"Model"))
1463 fit.minimize(
"Fluxes")
1464 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize Fluxes",
1465 writeChi2Name=getChi2Name(
"Fluxes"))
1467 fit.minimize(
"Model Fluxes", doLineSearch=doLineSearch)
1468 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize ModelFluxes",
1469 writeChi2Name=getChi2Name(
"ModelFluxes"))
1471 model.freezeErrorTransform()
1472 self.log.debug(
"Photometry error scales are frozen.")
1476 self.config.maxPhotometrySteps,
1479 doRankUpdate=self.config.photometryDoRankUpdate,
1480 doLineSearch=doLineSearch,
1487 def _fit_astrometry(self, associations, dataName=None):
1489 Fit the astrometric data.
1493 associations : `lsst.jointcal.Associations`
1494 The star/reference star associations to fit.
1496 Name of the data being processed (e.g. "1234_HSC-Y"), for
1497 identifying debugging files.
1501 fit_result : `namedtuple`
1502 fit : `lsst.jointcal.AstrometryFit`
1503 The astrometric fitter used to perform the fit.
1504 model : `lsst.jointcal.AstrometryModel`
1505 The astrometric model that was fit.
1506 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
1507 The model for the sky to tangent plane projection that was used in the fit.
1510 self.log.info(
"=== Starting astrometric fitting...")
1512 associations.deprojectFittedStars()
1519 if self.config.astrometryModel ==
"constrained":
1521 sky_to_tan_projection,
1522 chipOrder=self.config.astrometryChipOrder,
1523 visitOrder=self.config.astrometryVisitOrder)
1524 elif self.config.astrometryModel ==
"simple":
1526 sky_to_tan_projection,
1527 self.config.useInputWcs,
1529 order=self.config.astrometrySimpleOrder)
1534 if self.config.writeChi2FilesInitialFinal:
1535 baseName = f
"astrometry_initial_chi2-{dataName}"
1538 if self.config.writeInitialModel:
1539 fullpath = self.
_getDebugPath_getDebugPath(f
"initial_astrometry_model-{dataName}.txt")
1541 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initial", writeChi2Name=baseName)
1543 def getChi2Name(whatToFit):
1544 if self.config.writeChi2FilesOuterLoop:
1545 return f
"astrometry_init-%s_chi2-{dataName}" % whatToFit
1549 if self.config.writeInitMatrix:
1550 dumpMatrixFile = self.
_getDebugPath_getDebugPath(f
"astrometry_preinit-{dataName}")
1555 if self.config.astrometryModel ==
"constrained":
1556 fit.minimize(
"DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1557 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize DistortionsVisit",
1558 writeChi2Name=getChi2Name(
"DistortionsVisit"))
1561 fit.minimize(
"Distortions", dumpMatrixFile=dumpMatrixFile)
1562 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize Distortions",
1563 writeChi2Name=getChi2Name(
"Distortions"))
1565 fit.minimize(
"Positions")
1566 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize Positions",
1567 writeChi2Name=getChi2Name(
"Positions"))
1569 fit.minimize(
"Distortions Positions")
1570 self.
_logChi2AndValidate_logChi2AndValidate(associations, fit, model,
"Initialize DistortionsPositions",
1571 writeChi2Name=getChi2Name(
"DistortionsPositions"))
1575 self.config.maxAstrometrySteps,
1577 "Distortions Positions",
1578 doRankUpdate=self.config.astrometryDoRankUpdate,
1584 return Astrometry(fit, model, sky_to_tan_projection)
1586 def _check_stars(self, associations):
1587 """Count measured and reference stars per ccd and warn/log them."""
1588 for ccdImage
in associations.getCcdImageList():
1589 nMeasuredStars, nRefStars = ccdImage.countStars()
1590 self.log.debug(
"ccdImage %s has %s measured and %s reference stars",
1591 ccdImage.getName(), nMeasuredStars, nRefStars)
1592 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1593 self.log.warn(
"ccdImage %s has only %s measuredStars (desired %s)",
1594 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1595 if nRefStars < self.config.minRefStarsPerCcd:
1596 self.log.warn(
"ccdImage %s has only %s RefStars (desired %s)",
1597 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1599 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1602 doLineSearch=False):
1603 """Run fitter.minimize up to max_steps times, returning the final chi2.
1607 associations : `lsst.jointcal.Associations`
1608 The star/reference star associations to fit.
1609 fitter : `lsst.jointcal.FitterBase`
1610 The fitter to use for minimization.
1612 Maximum number of steps to run outlier rejection before declaring
1613 convergence failure.
1614 name : {'photometry' or 'astrometry'}
1615 What type of data are we fitting (for logs and debugging files).
1617 Passed to ``fitter.minimize()`` to define the parameters to fit.
1618 dataName : `str`, optional
1619 Descriptive name for this dataset (e.g. tract and filter),
1621 doRankUpdate : `bool`, optional
1622 Do an Eigen rank update during minimization, or recompute the full
1623 matrix and gradient?
1624 doLineSearch : `bool`, optional
1625 Do a line search for the optimum step during minimization?
1629 chi2: `lsst.jointcal.Chi2Statistic`
1630 The final chi2 after the fit converges, or is forced to end.
1635 Raised if the fitter fails with a non-finite value.
1637 Raised if the fitter fails for some other reason;
1638 log messages will provide further details.
1640 if self.config.writeInitMatrix:
1641 dumpMatrixFile = self.
_getDebugPath_getDebugPath(f
"{name}_postinit-{dataName}")
1645 oldChi2.chi2 = float(
"inf")
1646 for i
in range(max_steps):
1647 if self.config.writeChi2FilesOuterLoop:
1648 writeChi2Name = f
"{name}_iterate_{i}_chi2-{dataName}"
1650 writeChi2Name =
None
1651 result = fitter.minimize(whatToFit,
1652 self.config.outlierRejectSigma,
1653 doRankUpdate=doRankUpdate,
1654 doLineSearch=doLineSearch,
1655 dumpMatrixFile=dumpMatrixFile)
1657 chi2 = self.
_logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(),
1658 f
"Fit iteration {i}", writeChi2Name=writeChi2Name)
1660 if result == MinimizeResult.Converged:
1662 self.log.debug(
"fit has converged - no more outliers - redo minimization "
1663 "one more time in case we have lost accuracy in rank update.")
1665 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
1666 chi2 = self.
_logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(),
"Fit completed")
1669 if chi2.chi2/chi2.ndof >= 4.0:
1670 self.log.error(
"Potentially bad fit: High chi-squared/ndof.")
1673 elif result == MinimizeResult.Chi2Increased:
1674 self.log.warn(
"Still some outliers remaining but chi2 increased - retry")
1676 chi2Ratio = chi2.chi2 / oldChi2.chi2
1678 self.log.warn(
'Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1679 chi2.chi2, oldChi2.chi2, chi2Ratio)
1686 msg = (
"Large chi2 increase between steps: fit likely cannot converge."
1687 " Try setting one or more of the `writeChi2*` config fields and looking"
1688 " at how individual star chi2-values evolve during the fit.")
1689 raise RuntimeError(msg)
1691 elif result == MinimizeResult.NonFinite:
1692 filename = self.
_getDebugPath_getDebugPath(
"{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1694 fitter.saveChi2Contributions(filename+
"{type}")
1695 msg =
"Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1696 raise FloatingPointError(msg.format(filename))
1697 elif result == MinimizeResult.Failed:
1698 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
1700 raise RuntimeError(
"Unxepected return code from minimize().")
1702 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
1706 def _make_output(self, ccdImageList, model, func):
1707 """Return the internal jointcal models converted to the afw
1708 structures that will be saved to disk.
1712 ccdImageList : `lsst.jointcal.CcdImageList`
1713 The list of CcdImages to get the output for.
1714 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel`
1715 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`.
1717 The name of the function to call on ``model`` to get the converted
1718 structure. Must accept an `lsst.jointcal.CcdImage`.
1722 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or
1723 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`]
1724 The data to be saved, keyed on (visit, detector).
1727 for ccdImage
in ccdImageList:
1728 ccd = ccdImage.ccdId
1729 visit = ccdImage.visit
1730 self.log.debug(
"%s for visit: %d, ccd: %d", func, visit, ccd)
1731 output[(visit, ccd)] = getattr(model, func)(ccdImage)
1734 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1736 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1740 associations : `lsst.jointcal.Associations`
1741 The star/reference star associations to fit.
1742 model : `lsst.jointcal.AstrometryModel`
1743 The astrometric model that was fit.
1744 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1745 Dict of ccdImage identifiers to dataRefs that were fit.
1747 ccdImageList = associations.getCcdImageList()
1748 output = self.
_make_output_make_output(ccdImageList, model,
"makeSkyWcs")
1749 for key, skyWcs
in output.items():
1750 dataRef = visit_ccd_to_dataRef[key]
1752 dataRef.put(skyWcs,
'jointcal_wcs')
1753 except pexExceptions.Exception
as e:
1754 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
1757 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1759 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1763 associations : `lsst.jointcal.Associations`
1764 The star/reference star associations to fit.
1765 model : `lsst.jointcal.PhotometryModel`
1766 The photoometric model that was fit.
1767 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1768 Dict of ccdImage identifiers to dataRefs that were fit.
1771 ccdImageList = associations.getCcdImageList()
1772 output = self.
_make_output_make_output(ccdImageList, model,
"toPhotoCalib")
1773 for key, photoCalib
in output.items():
1774 dataRef = visit_ccd_to_dataRef[key]
1776 dataRef.put(photoCalib,
'jointcal_photoCalib')
1777 except pexExceptions.Exception
as e:
1778 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
static Schema makeMinimalSchema()
static std::shared_ptr< SourceTable > make(Schema const &schema, std::shared_ptr< IdFactory > const &idFactory)
static Schema makeMinimalSchema()
The class that implements the relations between MeasuredStar and FittedStar.
Class that handles the astrometric least squares problem.
Simple structure to accumulate chi2 and ndof.
A multi-component model, fitting mappings for sensors and visits simultaneously.
A projection handler in which all CCDs from the same visit have the same tangent point.
Class that handles the photometric least squares problem.
A model where there is one independent transform per CcdImage.
def getTargetList(parsedCmd, **kwargs)
def __init__(self, *config=None)
def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, jointcalControl, camera)
def _compute_proper_motion_epoch(self, ccdImageList)
def _get_refcat_coordinate_error_override(self, refCat, name)
def _getDebugPath(self, filename)
def _check_star_lists(self, associations, name)
def _make_one_input_data(self, visitRecord, catalog, detectorDict)
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def _check_stars(self, associations)
def _prep_sky(self, associations, filters)
def _readDataId(self, butler, dataId)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, tract="", match_cut=3.0, reject_bad_fluxes=False, *name="", refObjLoader=None, referenceSelector=None, fit_function=None, epoch=None)
def _build_ccdImage(self, data, associations, jointcalControl)
def runDataRef(self, dataRefs)
def __init__(self, butler=None, initInputs=None, **kwargs)
def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId)
def _make_schema_table(self)
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None)
def _fit_photometry(self, associations, dataName=None)
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None)
def _make_output(self, ccdImageList, model, func)
def _put_output(self, butlerQC, outputs, outputRefs, camera, setter)
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, applyColorterms=False, epoch=None)
def _fit_astrometry(self, associations, dataName=None)
def loadData(self, dataRefs, associations, jointcalControl)
def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
def add_measurement(job, name, value)
def lookupVisitRefCats(datasetType, registry, quantumDataId, collections)
def writeModel(model, filename, log)