28import astropy.units
as u
33import lsst.pipe.base
as pipeBase
39from lsst.pipe.base
import Instrument
40from lsst.pipe.tasks.colorterms
import ColortermLibrary
41from lsst.verify
import Job, Measurement
44 LoadIndexedReferenceObjectsConfig)
50__all__ = [
"JointcalConfig",
"JointcalTask"]
52Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
53Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
58 meas = Measurement(job.metrics[name], value)
59 job.measurements.insert(meas)
63 """Lookup function that asserts/hopes that a static calibration dataset
64 exists in a particular collection, since this task can
't provide a single
65 date/time to use to search for one properly.
67 This
is mostly useful
for the ``camera`` dataset,
in cases where the task
's
68 quantum dimensions do *not* include something temporal, like ``exposure``
73 datasetType : `lsst.daf.butler.DatasetType`
74 Type of dataset being searched
for.
75 registry : `lsst.daf.butler.Registry`
76 Data repository registry to search.
77 quantumDataId : `lsst.daf.butler.DataCoordinate`
78 Data ID of the quantum this camera should match.
79 collections : `Iterable` [ `str` ]
80 Collections that should be searched - but this lookup function works
81 by ignoring this
in favor of a more-
or-less hard-coded value.
85 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
86 Iterator over dataset references; should have only one element.
90 This implementation duplicates one
in fgcmcal,
and is at least quite
91 similar to another
in cp_pipe. This duplicate has the most documentation.
92 Fixing this
is DM-29661.
94 instrument = Instrument.fromName(quantumDataId["instrument"], registry)
95 unboundedCollection = instrument.makeUnboundedCalibrationRunName()
96 return registry.queryDatasets(datasetType,
98 collections=[unboundedCollection],
103 """Lookup function that finds all refcats for all visits that overlap a
104 tract, rather than just the refcats that directly overlap the tract.
108 datasetType : `lsst.daf.butler.DatasetType`
109 Type of dataset being searched for.
110 registry : `lsst.daf.butler.Registry`
111 Data repository registry to search.
112 quantumDataId : `lsst.daf.butler.DataCoordinate`
113 Data ID of the quantum; expected to be something we can use
as a
114 constraint to query
for overlapping visits.
115 collections : `Iterable` [ `str` ]
116 Collections to search.
120 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
121 Iterator over refcat references.
130 for visit_data_id
in set(registry.queryDataIds(
"visit", dataId=quantumDataId).expanded()):
132 registry.queryDatasets(
134 collections=collections,
135 dataId=visit_data_id,
143 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter")):
144 """Middleware input/output connections for jointcal data."""
145 inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
146 doc=
"The camera instrument that took these observations.",
148 storageClass=
"Camera",
149 dimensions=(
"instrument",),
151 lookupFunction=lookupStaticCalibrations,
153 inputSourceTableVisit = pipeBase.connectionTypes.Input(
154 doc=
"Source table in parquet format, per visit",
155 name=
"sourceTable_visit",
156 storageClass=
"DataFrame",
157 dimensions=(
"instrument",
"visit"),
161 inputVisitSummary = pipeBase.connectionTypes.Input(
162 doc=(
"Per-visit consolidated exposure metadata built from calexps. "
163 "These catalogs use detector id for the id and must be sorted for "
164 "fast lookups of a detector."),
166 storageClass=
"ExposureCatalog",
167 dimensions=(
"instrument",
"visit"),
171 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
172 doc=
"The astrometry reference catalog to match to loaded input catalog sources.",
173 name=
"gaia_dr2_20200414",
174 storageClass=
"SimpleCatalog",
175 dimensions=(
"skypix",),
178 lookupFunction=lookupVisitRefCats,
180 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
181 doc=
"The photometry reference catalog to match to loaded input catalog sources.",
182 name=
"ps1_pv3_3pi_20170110",
183 storageClass=
"SimpleCatalog",
184 dimensions=(
"skypix",),
187 lookupFunction=lookupVisitRefCats,
190 outputWcs = pipeBase.connectionTypes.Output(
191 doc=(
"Per-tract, per-visit world coordinate systems derived from the fitted model."
192 " These catalogs only contain entries for detectors with an output, and use"
193 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
194 name=
"jointcalSkyWcsCatalog",
195 storageClass=
"ExposureCatalog",
196 dimensions=(
"instrument",
"visit",
"skymap",
"tract"),
199 outputPhotoCalib = pipeBase.connectionTypes.Output(
200 doc=(
"Per-tract, per-visit photometric calibrations derived from the fitted model."
201 " These catalogs only contain entries for detectors with an output, and use"
202 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
203 name=
"jointcalPhotoCalibCatalog",
204 storageClass=
"ExposureCatalog",
205 dimensions=(
"instrument",
"visit",
"skymap",
"tract"),
213 for name
in (
"astrometry",
"photometry"):
214 vars()[f
"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output(
215 doc=f
"The number of cross-matched fittedStars for {name}",
216 name=f
"metricvalue_jointcal_{name}_matched_fittedStars",
217 storageClass=
"MetricValue",
218 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
220 vars()[f
"{name}_collected_refStars"] = pipeBase.connectionTypes.Output(
221 doc=f
"The number of {name} reference stars drawn from the reference catalog, before matching.",
222 name=f
"metricvalue_jointcal_{name}_collected_refStars",
223 storageClass=
"MetricValue",
224 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
226 vars()[f
"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output(
227 doc=f
"The number of {name} reference stars matched to fittedStars.",
228 name=f
"metricvalue_jointcal_{name}_prepared_refStars",
229 storageClass=
"MetricValue",
230 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
232 vars()[f
"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output(
233 doc=f
"The number of cross-matched fittedStars after cleanup, for {name}.",
234 name=f
"metricvalue_jointcal_{name}_prepared_fittedStars",
235 storageClass=
"MetricValue",
236 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
238 vars()[f
"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output(
239 doc=f
"The number of ccdImages that will be fit for {name}, after cleanup.",
240 name=f
"metricvalue_jointcal_{name}_prepared_ccdImages",
241 storageClass=
"MetricValue",
242 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
244 vars()[f
"{name}_final_chi2"] = pipeBase.connectionTypes.Output(
245 doc=f
"The final chi2 of the {name} fit.",
246 name=f
"metricvalue_jointcal_{name}_final_chi2",
247 storageClass=
"MetricValue",
248 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
250 vars()[f
"{name}_final_ndof"] = pipeBase.connectionTypes.Output(
251 doc=f
"The number of degrees of freedom of the fitted {name}.",
252 name=f
"metricvalue_jointcal_{name}_final_ndof",
253 storageClass=
"MetricValue",
254 dimensions=(
"skymap",
"tract",
"instrument",
"physical_filter"),
265 if not config.doAstrometry:
266 self.prerequisiteInputs.remove(
"astrometryRefCat")
267 self.outputs.remove(
"outputWcs")
268 for key
in list(self.outputs):
269 if "metricvalue_jointcal_astrometry" in key:
270 self.outputs.remove(key)
271 if not config.doPhotometry:
272 self.prerequisiteInputs.remove(
"photometryRefCat")
273 self.outputs.remove(
"outputPhotoCalib")
274 for key
in list(self.outputs):
275 if "metricvalue_jointcal_photometry" in key:
276 self.outputs.remove(key)
280 pipelineConnections=JointcalTaskConnections):
281 """Configuration for JointcalTask"""
283 doAstrometry = pexConfig.Field(
284 doc=
"Fit astrometry and write the fitted result.",
288 doPhotometry = pexConfig.Field(
289 doc=
"Fit photometry and write the fitted result.",
293 sourceFluxType = pexConfig.Field(
295 doc=
"Source flux field to use in source selection and to get fluxes from the catalog.",
296 default=
'apFlux_12_0'
298 positionErrorPedestal = pexConfig.Field(
299 doc=
"Systematic term to apply to the measured position error (pixels)",
303 photometryErrorPedestal = pexConfig.Field(
304 doc=
"Systematic term to apply to the measured error on flux or magnitude as a "
305 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
310 matchCut = pexConfig.Field(
311 doc=
"Matching radius between fitted and reference stars (arcseconds)",
315 minMeasurements = pexConfig.Field(
316 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
320 minMeasuredStarsPerCcd = pexConfig.Field(
321 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
325 minRefStarsPerCcd = pexConfig.Field(
326 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
330 allowLineSearch = pexConfig.Field(
331 doc=
"Allow a line search during minimization, if it is reasonable for the model"
332 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
336 astrometrySimpleOrder = pexConfig.Field(
337 doc=
"Polynomial order for fitting the simple astrometry model.",
341 astrometryChipOrder = pexConfig.Field(
342 doc=
"Order of the per-chip transform for the constrained astrometry model.",
346 astrometryVisitOrder = pexConfig.Field(
347 doc=
"Order of the per-visit transform for the constrained astrometry model.",
351 useInputWcs = pexConfig.Field(
352 doc=
"Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
356 astrometryModel = pexConfig.ChoiceField(
357 doc=
"Type of model to fit to astrometry",
359 default=
"constrained",
360 allowed={
"simple":
"One polynomial per ccd",
361 "constrained":
"One polynomial per ccd, and one polynomial per visit"}
363 photometryModel = pexConfig.ChoiceField(
364 doc=
"Type of model to fit to photometry",
366 default=
"constrainedMagnitude",
367 allowed={
"simpleFlux":
"One constant zeropoint per ccd and visit, fitting in flux space.",
368 "constrainedFlux":
"Constrained zeropoint per ccd, and one polynomial per visit,"
369 " fitting in flux space.",
370 "simpleMagnitude":
"One constant zeropoint per ccd and visit,"
371 " fitting in magnitude space.",
372 "constrainedMagnitude":
"Constrained zeropoint per ccd, and one polynomial per visit,"
373 " fitting in magnitude space.",
376 applyColorTerms = pexConfig.Field(
377 doc=
"Apply photometric color terms to reference stars?"
378 "Requires that colorterms be set to a ColortermLibrary",
382 colorterms = pexConfig.ConfigField(
383 doc=
"Library of photometric reference catalog name to color term dict.",
384 dtype=ColortermLibrary,
386 photometryVisitOrder = pexConfig.Field(
387 doc=
"Order of the per-visit polynomial transform for the constrained photometry model.",
391 photometryDoRankUpdate = pexConfig.Field(
392 doc=(
"Do the rank update step during minimization. "
393 "Skipping this can help deal with models that are too non-linear."),
397 astrometryDoRankUpdate = pexConfig.Field(
398 doc=(
"Do the rank update step during minimization (should not change the astrometry fit). "
399 "Skipping this can help deal with models that are too non-linear."),
403 outlierRejectSigma = pexConfig.Field(
404 doc=
"How many sigma to reject outliers at during minimization.",
408 astrometryOutlierRelativeTolerance = pexConfig.Field(
409 doc=(
"Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will "
410 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set "
411 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002"
412 "as a balance between a shorter minimization runtime and achieving a final fitted model that is"
413 "close to the solution found when removing all outliers."),
417 maxPhotometrySteps = pexConfig.Field(
418 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
422 maxAstrometrySteps = pexConfig.Field(
423 doc=
"Maximum number of minimize iterations to take when fitting astrometry.",
427 astrometryRefObjLoader = pexConfig.ConfigField(
428 dtype=LoadIndexedReferenceObjectsConfig,
429 doc=
"Reference object loader for astrometric fit",
431 photometryRefObjLoader = pexConfig.ConfigField(
432 dtype=LoadIndexedReferenceObjectsConfig,
433 doc=
"Reference object loader for photometric fit",
435 sourceSelector = sourceSelectorRegistry.makeField(
436 doc=
"How to select sources for cross-matching",
439 astrometryReferenceSelector = pexConfig.ConfigurableField(
440 target=ReferenceSourceSelectorTask,
441 doc=
"How to down-select the loaded astrometry reference catalog.",
443 photometryReferenceSelector = pexConfig.ConfigurableField(
444 target=ReferenceSourceSelectorTask,
445 doc=
"How to down-select the loaded photometry reference catalog.",
447 astrometryReferenceErr = pexConfig.Field(
448 doc=(
"Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
449 "If None, then raise an exception if the reference catalog is missing coordinate errors. "
450 "If specified, overrides any existing `coord_*Err` values."),
457 writeInitMatrix = pexConfig.Field(
459 doc=(
"Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
460 "Output files will be written to `config.debugOutputPath` and will "
461 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. "
462 "Note that these files are the dense versions of the matrix, and so may be very large."),
465 writeChi2FilesInitialFinal = pexConfig.Field(
467 doc=(
"Write .csv files containing the contributions to chi2 for the initialization and final fit. "
468 "Output files will be written to `config.debugOutputPath` and will "
469 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."),
472 writeChi2FilesOuterLoop = pexConfig.Field(
474 doc=(
"Write .csv files containing the contributions to chi2 for the outer fit loop. "
475 "Output files will be written to `config.debugOutputPath` and will "
476 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."),
479 writeInitialModel = pexConfig.Field(
481 doc=(
"Write the pre-initialization model to text files, for debugging. "
482 "Output files will be written to `config.debugOutputPath` and will be "
483 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."),
486 debugOutputPath = pexConfig.Field(
489 doc=(
"Path to write debug output files to. Used by "
490 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
492 detailedProfile = pexConfig.Field(
495 doc=
"Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
501 msg =
"applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
502 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
504 msg = (
"Only doing astrometry, but Colorterms are not applied for astrometry;"
505 "applyColorTerms=True will be ignored.")
506 lsst.log.warning(msg)
518 self.
sourceSelector[
"science"].signalToNoise.fluxField = f
"{self.sourceFluxType}_instFlux"
519 self.
sourceSelector[
"science"].signalToNoise.errField = f
"{self.sourceFluxType}_instFluxErr"
522 self.
sourceSelector[
"science"].isolated.parentName =
"parentSourceId"
523 self.
sourceSelector[
"science"].isolated.nChildName =
"deblend_nChild"
527 badFlags = [
"pixelFlags_edge",
528 "pixelFlags_saturated",
529 "pixelFlags_interpolatedCenter",
530 "pixelFlags_interpolated",
531 "pixelFlags_crCenter",
533 "hsmPsfMoments_flag",
534 f
"{self.sourceFluxType}_flag",
545 """Write model to outfile."""
546 with open(filename,
"w")
as file:
547 file.write(repr(model))
548 log.info(
"Wrote %s to file: %s", model, filename)
551@dataclasses.dataclass
553 """The input data jointcal needs for each detector/visit."""
555 """The visit identifier of this exposure."""
557 """The catalog derived from this exposure."""
559 """The VisitInfo of this exposure."""
561 """The detector of this exposure."""
563 """The photometric calibration of this exposure."""
565 """The WCS of this exposure."""
567 """The bounding box of this exposure."""
569 """The filter of this exposure."""
573 """Astrometricly and photometricly calibrate across multiple visits of the
577 ConfigClass = JointcalConfig
578 _DefaultName = "jointcal"
582 self.makeSubtask(
"sourceSelector")
583 if self.config.doAstrometry:
584 self.makeSubtask(
"astrometryReferenceSelector")
587 if self.config.doPhotometry:
588 self.makeSubtask(
"photometryReferenceSelector")
593 self.
job = Job.load_metrics_package(subset=
'jointcal')
598 inputs = butlerQC.get(inputRefs)
600 tract = butlerQC.quantum.dataId[
'tract']
601 if self.config.doAstrometry:
603 dataIds=[ref.datasetRef.dataId
604 for ref
in inputRefs.astrometryRefCat],
605 refCats=inputs.pop(
'astrometryRefCat'),
606 config=self.config.astrometryRefObjLoader,
608 if self.config.doPhotometry:
610 dataIds=[ref.datasetRef.dataId
611 for ref
in inputRefs.photometryRefCat],
612 refCats=inputs.pop(
'photometryRefCat'),
613 config=self.config.photometryRefObjLoader,
615 outputs = self.
run(**inputs, tract=tract)
617 if self.config.doAstrometry:
618 self.
_put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
619 inputs[
'inputCamera'],
"setWcs")
620 if self.config.doPhotometry:
621 self.
_put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
622 inputs[
'inputCamera'],
"setPhotoCalib")
624 def _put_metrics(self, butlerQC, job, outputRefs):
625 """Persist all measured metrics stored in a job.
629 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
630 A butler which is specialized to operate
in the context of a
631 `lsst.daf.butler.Quantum`; This
is the input to `runQuantum`.
632 job : `lsst.verify.job.Job`
633 Measurements of metrics to persist.
634 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
635 The DatasetRefs to persist the data to.
637 for key
in job.measurements.keys():
638 butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace(
'jointcal.',
'')))
640 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
641 """Persist the output datasets to their appropriate datarefs.
645 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
646 A butler which is specialized to operate
in the context of a
647 `lsst.daf.butler.Quantum`; This
is the input to `runQuantum`.
650 The fitted objects to persist.
651 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
652 The DatasetRefs to persist the data to.
654 The camera
for this instrument, to get detector ids
from.
656 The method to call on the ExposureCatalog to set each output.
659 schema.addField('visit', type=
'L', doc=
'Visit number')
661 def new_catalog(visit, size):
662 """Return an catalog ready to be filled with appropriate output."""
665 catalog[
'visit'] = visit
667 metadata.add(
"COMMENT",
"Catalog id is detector id, sorted.")
668 metadata.add(
"COMMENT",
"Only detectors with data have entries.")
672 detectors_per_visit = collections.defaultdict(int)
675 detectors_per_visit[key[0]] += 1
677 for ref
in outputRefs:
678 visit = ref.dataId[
'visit']
679 catalog = new_catalog(visit, detectors_per_visit[visit])
682 for detector
in camera:
683 detectorId = detector.getId()
684 key = (ref.dataId[
'visit'], detectorId)
685 if key
not in outputs:
687 self.log.debug(
"No %s output for detector %s in visit %s",
688 setter[3:], detectorId, visit)
691 catalog[i].setId(detectorId)
692 getattr(catalog[i], setter)(outputs[key])
696 butlerQC.put(catalog, ref)
697 self.log.info(
"Wrote %s detectors to %s", i, ref)
699 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
705 sourceFluxField =
"flux"
709 oldWcsList, bands = self.
_load_data(inputSourceTableVisit,
715 boundingCircle, center, radius, defaultFilter, epoch = self.
_prep_sky(associations, bands)
717 if self.config.doAstrometry:
721 referenceSelector=self.astrometryReferenceSelector,
725 astrometry_output = self.
_make_output(associations.getCcdImageList(),
729 astrometry_output =
None
731 if self.config.doPhotometry:
735 referenceSelector=self.photometryReferenceSelector,
739 reject_bad_fluxes=
True)
740 photometry_output = self.
_make_output(associations.getCcdImageList(),
744 photometry_output =
None
746 return pipeBase.Struct(outputWcs=astrometry_output,
747 outputPhotoCalib=photometry_output,
752 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
753 jointcalControl, camera):
754 """Read the data that jointcal needs to run.
756 Modifies ``associations`` in-place
with the loaded data.
760 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
761 References to visit-level DataFrames to load the catalog data
from.
762 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
763 Visit-level exposure summary catalog
with metadata.
765 Object to add the loaded data to by constructing new CcdImages.
767 Control object
for C++ associations management.
769 Camera object
for detector geometry.
774 The original WCS of the input data, to aid
in writing tests.
775 bands : `list` [`str`]
776 The filter bands of each input dataset.
780 load_cat_profile_file = 'jointcal_load_data.prof' if self.config.detailedProfile
else ''
781 with pipeBase.cmdLineTask.profile(load_cat_profile_file):
784 catalogMap = {ref.dataId[
'visit']: i
for i, ref
in enumerate(inputSourceTableVisit)}
785 detectorDict = {detector.getId(): detector
for detector
in camera}
789 for visitSummaryRef
in inputVisitSummary:
790 visitSummary = visitSummaryRef.get()
792 dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId[
'visit']]]
794 inColumns = dataRef.get(component=
'columns')
798 visitCatalog = dataRef.get(parameters={
'columns': columns})
800 selected = self.sourceSelector.
run(visitCatalog)
803 detectors = {id: index
for index, id
in enumerate(visitSummary[
'id'])}
804 for id, index
in detectors.items():
810 self.config.sourceFluxType,
816 if result
is not None:
817 oldWcsList.append(result.wcs)
819 filters.append(data.filter)
820 if len(filters) == 0:
821 raise RuntimeError(
"No data to process: did source selector remove all sources?")
822 filters = collections.Counter(filters)
824 return oldWcsList, filters
826 def _make_one_input_data(self, visitRecord, catalog, detectorDict):
827 """Return a data structure for this detector+visit."""
830 visitInfo=visitRecord.getVisitInfo(),
831 detector=detectorDict[visitRecord.getId()],
832 photoCalib=visitRecord.getPhotoCalib(),
833 wcs=visitRecord.getWcs(),
834 bbox=visitRecord.getBBox(),
838 physical=visitRecord[
'physical_filter']))
843 def _getMetadataName(self):
846 def _build_ccdImage(self, data, associations, jointcalControl):
848 Extract the necessary things from this catalog+metadata to add a new
853 data : `JointcalInputData`
854 The loaded input data.
856 Object to add the info to, to construct a new CcdImage
858 Control object
for associations management
864 The TAN WCS of this image, read
from the calexp
867 A key to identify this dataRef by its visit
and ccd ids
870 if there are no sources
in the loaded catalog.
872 if len(data.catalog) == 0:
873 self.log.warning(
"No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
876 associations.createCcdImage(data.catalog,
880 data.filter.physicalLabel,
884 data.detector.getId(),
887 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key'))
888 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
889 return Result(data.wcs, Key(data.visit, data.detector.getId()))
891 def _getDebugPath(self, filename):
892 """Constructs a path to filename using the configured debug path.
894 return os.path.join(self.config.debugOutputPath, filename)
896 def _prep_sky(self, associations, filters):
897 """Prepare on-sky and other data that must be computed after data has
900 associations.computeCommonTangentPoint()
902 boundingCircle = associations.computeBoundingCircle()
904 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
906 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.")
909 defaultFilter = filters.most_common(1)[0][0]
910 self.log.debug(
"Using '%s' filter for reference flux", defaultFilter.physicalLabel)
914 associations.setEpoch(epoch.jyear)
916 return boundingCircle, center, radius, defaultFilter, epoch
918 def _get_refcat_coordinate_error_override(self, refCat, name):
919 """Check whether we should override the refcat coordinate errors, and
920 return the overridden error
if necessary.
925 The reference catalog to check
for a ``coord_raErr`` field.
927 Whether we are doing
"astrometry" or "photometry".
931 refCoordErr : `float`
932 The refcat coordinate error to use,
or NaN
if we are
not overriding
937 lsst.pex.config.FieldValidationError
938 Raised
if the refcat does
not contain coordinate errors
and
939 ``config.astrometryReferenceErr``
is not set.
943 if name.lower() ==
"photometry":
944 if 'coord_raErr' not in refCat.schema:
949 if self.config.astrometryReferenceErr
is None and 'coord_raErr' not in refCat.schema:
950 msg = (
"Reference catalog does not contain coordinate errors, "
951 "and config.astrometryReferenceErr not supplied.")
952 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
956 if self.config.astrometryReferenceErr
is not None and 'coord_raErr' in refCat.schema:
957 self.log.warning(
"Overriding reference catalog coordinate errors with %f/coordinate [mas]",
958 self.config.astrometryReferenceErr)
960 if self.config.astrometryReferenceErr
is None:
963 return self.config.astrometryReferenceErr
965 def _compute_proper_motion_epoch(self, ccdImageList):
966 """Return the proper motion correction epoch of the provided images.
971 The images to compute the appropriate epoch for.
975 epoch : `astropy.time.Time`
976 The date to use
for proper motion corrections.
978 return astropy.time.Time(np.mean([ccdImage.getEpoch()
for ccdImage
in ccdImageList]),
982 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
983 tract="", match_cut=3.0,
984 reject_bad_fluxes=False, *,
985 name="", refObjLoader=None, referenceSelector=None,
986 fit_function=None, epoch=None):
987 """Load reference catalog, perform the fit, and return the result.
992 The star/reference star associations to fit.
994 filter to load from reference catalog.
996 ICRS center of field to load
from reference catalog.
998 On-sky radius to load
from reference catalog.
1000 Name of thing being fit:
"astrometry" or "photometry".
1001 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`
1002 Reference object loader to use to load a reference catalog.
1003 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1004 Selector to use to pick objects
from the loaded reference catalog.
1005 fit_function : callable
1006 Function to call to perform fit (takes Associations object).
1007 tract : `str`, optional
1008 Name of tract currently being fit.
1009 match_cut : `float`, optional
1010 Radius
in arcseconds to find cross-catalog matches to during
1011 associations.associateCatalogs.
1012 reject_bad_fluxes : `bool`, optional
1013 Reject refCat sources
with NaN/inf flux
or NaN/0 fluxErr.
1014 epoch : `astropy.time.Time`, optional
1015 Epoch to which to correct refcat proper motion
and parallax,
1016 or `
None` to
not apply such corrections.
1020 result : `Photometry`
or `Astrometry`
1021 Result of `fit_function()`
1023 self.log.info("====== Now processing %s...", name)
1026 associations.associateCatalogs(match_cut)
1028 associations.fittedStarListSize())
1030 applyColorterms =
False if name.lower() ==
"astrometry" else self.config.applyColorTerms
1032 center, radius, defaultFilter,
1033 applyColorterms=applyColorterms,
1037 associations.collectRefStars(refCat,
1038 self.config.matchCut*lsst.geom.arcseconds,
1040 refCoordinateErr=refCoordErr,
1041 rejectBadFluxes=reject_bad_fluxes)
1043 associations.refStarListSize())
1045 associations.prepareFittedStars(self.config.minMeasurements)
1049 associations.nFittedStarsWithAssociatedRefStar())
1051 associations.fittedStarListSize())
1053 associations.nCcdImagesValidForFit())
1055 fit_profile_file =
'jointcal_fit_%s.prof'%name
if self.config.detailedProfile
else ''
1056 dataName =
"{}_{}".format(tract, defaultFilter.physicalLabel)
1057 with pipeBase.cmdLineTask.profile(fit_profile_file):
1058 result = fit_function(associations, dataName)
1061 if self.config.writeChi2FilesInitialFinal:
1062 baseName = self.
_getDebugPath(f
"{name}_final_chi2-{dataName}")
1063 result.fit.saveChi2Contributions(baseName+
"{type}")
1064 self.log.info(
"Wrote chi2 contributions files: %s", baseName)
1068 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1069 applyColorterms=False, epoch=None):
1070 """Load the necessary reference catalog sources, convert fluxes to
1071 correct units, and apply color term corrections
if requested.
1075 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`
1076 The reference catalog loader to use to get the data.
1077 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1078 Source selector to apply to loaded reference catalog.
1080 The center around which to load sources.
1082 The radius around ``center`` to load sources
in.
1084 The camera filter to load fluxes
for.
1085 applyColorterms : `bool`
1086 Apply colorterm corrections to the refcat
for ``filterName``?
1087 epoch : `astropy.time.Time`, optional
1088 Epoch to which to correct refcat proper motion
and parallax,
1089 or `
None` to
not apply such corrections.
1094 The loaded reference catalog.
1096 The name of the reference catalog flux field appropriate
for ``filterName``.
1098 skyCircle = refObjLoader.loadSkyCircle(center,
1100 filterLabel.bandLabel,
1103 selected = referenceSelector.run(skyCircle.refCat)
1105 if not selected.sourceCat.isContiguous():
1106 refCat = selected.sourceCat.copy(deep=
True)
1108 refCat = selected.sourceCat
1111 refCatName = self.config.connections.photometryRefCat
1112 self.log.info(
"Applying color terms for physical filter=%r reference catalog=%s",
1113 filterLabel.physicalLabel, refCatName)
1114 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1118 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1119 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1121 refCat[skyCircle.fluxField+
'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1123 return refCat, skyCircle.fluxField
1125 def _check_star_lists(self, associations, name):
1127 if associations.nCcdImagesValidForFit() == 0:
1128 raise RuntimeError(
'No images in the ccdImageList!')
1129 if associations.fittedStarListSize() == 0:
1130 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
1131 if associations.refStarListSize() == 0:
1132 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
1134 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1135 """Compute chi2, log it, validate the model, and return chi2.
1140 The star/reference star associations to fit.
1142 The fitter to use for minimization.
1143 model : `lsst.jointcal.Model`
1144 The model being fit.
1146 Label to describe the chi2 (e.g.
"Initialized",
"Final").
1147 writeChi2Name : `str`, optional
1148 Filename prefix to write the chi2 contributions to.
1149 Do
not supply an extension: an appropriate one will be added.
1154 The chi2 object
for the current fitter
and model.
1159 Raised
if chi2
is infinite
or NaN.
1161 Raised
if the model
is not valid.
1163 if writeChi2Name
is not None:
1165 fit.saveChi2Contributions(fullpath+
"{type}")
1166 self.log.info(
"Wrote chi2 contributions files: %s", fullpath)
1168 chi2 = fit.computeChi2()
1169 self.log.info(
"%s %s", chi2Label, chi2)
1171 if not np.isfinite(chi2.chi2):
1172 raise FloatingPointError(f
'{chi2Label} chi2 is invalid: {chi2}')
1173 if not model.validate(associations.getCcdImageList(), chi2.ndof):
1174 raise ValueError(
"Model is not valid: check log messages for warnings.")
1177 def _fit_photometry(self, associations, dataName=None):
1179 Fit the photometric data.
1184 The star/reference star associations to fit.
1186 Name of the data being processed (e.g. "1234_HSC-Y"),
for
1187 identifying debugging files.
1191 fit_result : `namedtuple`
1193 The photometric fitter used to perform the fit.
1195 The photometric model that was fit.
1197 self.log.info("=== Starting photometric fitting...")
1200 if self.config.photometryModel ==
"constrainedFlux":
1203 visitOrder=self.config.photometryVisitOrder,
1204 errorPedestal=self.config.photometryErrorPedestal)
1206 doLineSearch = self.config.allowLineSearch
1207 elif self.config.photometryModel ==
"constrainedMagnitude":
1210 visitOrder=self.config.photometryVisitOrder,
1211 errorPedestal=self.config.photometryErrorPedestal)
1213 doLineSearch = self.config.allowLineSearch
1214 elif self.config.photometryModel ==
"simpleFlux":
1216 errorPedestal=self.config.photometryErrorPedestal)
1217 doLineSearch =
False
1218 elif self.config.photometryModel ==
"simpleMagnitude":
1220 errorPedestal=self.config.photometryErrorPedestal)
1221 doLineSearch =
False
1226 if self.config.writeChi2FilesInitialFinal:
1227 baseName = f
"photometry_initial_chi2-{dataName}"
1230 if self.config.writeInitialModel:
1231 fullpath = self.
_getDebugPath(f
"initial_photometry_model-{dataName}.txt")
1235 def getChi2Name(whatToFit):
1236 if self.config.writeChi2FilesOuterLoop:
1237 return f
"photometry_init-%s_chi2-{dataName}" % whatToFit
1243 if self.config.writeInitMatrix:
1244 dumpMatrixFile = self.
_getDebugPath(f
"photometry_preinit-{dataName}")
1247 if self.config.photometryModel.startswith(
"constrained"):
1250 fit.minimize(
"ModelVisit", dumpMatrixFile=dumpMatrixFile)
1252 writeChi2Name=getChi2Name(
"ModelVisit"))
1255 fit.minimize(
"Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1257 writeChi2Name=getChi2Name(
"Model"))
1259 fit.minimize(
"Fluxes")
1261 writeChi2Name=getChi2Name(
"Fluxes"))
1263 fit.minimize(
"Model Fluxes", doLineSearch=doLineSearch)
1265 writeChi2Name=getChi2Name(
"ModelFluxes"))
1267 model.freezeErrorTransform()
1268 self.log.debug(
"Photometry error scales are frozen.")
1272 self.config.maxPhotometrySteps,
1275 doRankUpdate=self.config.photometryDoRankUpdate,
1276 doLineSearch=doLineSearch,
1283 def _fit_astrometry(self, associations, dataName=None):
1285 Fit the astrometric data.
1290 The star/reference star associations to fit.
1292 Name of the data being processed (e.g. "1234_HSC-Y"),
for
1293 identifying debugging files.
1297 fit_result : `namedtuple`
1299 The astrometric fitter used to perform the fit.
1301 The astrometric model that was fit.
1303 The model
for the sky to tangent plane projection that was used
in the fit.
1306 self.log.info("=== Starting astrometric fitting...")
1308 associations.deprojectFittedStars()
1315 if self.config.astrometryModel ==
"constrained":
1317 sky_to_tan_projection,
1318 chipOrder=self.config.astrometryChipOrder,
1319 visitOrder=self.config.astrometryVisitOrder)
1320 elif self.config.astrometryModel ==
"simple":
1322 sky_to_tan_projection,
1323 self.config.useInputWcs,
1325 order=self.config.astrometrySimpleOrder)
1330 if self.config.writeChi2FilesInitialFinal:
1331 baseName = f
"astrometry_initial_chi2-{dataName}"
1334 if self.config.writeInitialModel:
1335 fullpath = self.
_getDebugPath(f
"initial_astrometry_model-{dataName}.txt")
1339 def getChi2Name(whatToFit):
1340 if self.config.writeChi2FilesOuterLoop:
1341 return f
"astrometry_init-%s_chi2-{dataName}" % whatToFit
1345 if self.config.writeInitMatrix:
1346 dumpMatrixFile = self.
_getDebugPath(f
"astrometry_preinit-{dataName}")
1351 if self.config.astrometryModel ==
"constrained":
1352 fit.minimize(
"DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1354 writeChi2Name=getChi2Name(
"DistortionsVisit"))
1357 fit.minimize(
"Distortions", dumpMatrixFile=dumpMatrixFile)
1359 writeChi2Name=getChi2Name(
"Distortions"))
1361 fit.minimize(
"Positions")
1363 writeChi2Name=getChi2Name(
"Positions"))
1365 fit.minimize(
"Distortions Positions")
1367 writeChi2Name=getChi2Name(
"DistortionsPositions"))
1371 self.config.maxAstrometrySteps,
1373 "Distortions Positions",
1374 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance,
1375 doRankUpdate=self.config.astrometryDoRankUpdate,
1381 return Astrometry(fit, model, sky_to_tan_projection)
1383 def _check_stars(self, associations):
1384 """Count measured and reference stars per ccd and warn/log them."""
1385 for ccdImage
in associations.getCcdImageList():
1386 nMeasuredStars, nRefStars = ccdImage.countStars()
1387 self.log.debug(
"ccdImage %s has %s measured and %s reference stars",
1388 ccdImage.getName(), nMeasuredStars, nRefStars)
1389 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1390 self.log.warning(
"ccdImage %s has only %s measuredStars (desired %s)",
1391 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1392 if nRefStars < self.config.minRefStarsPerCcd:
1393 self.log.warning(
"ccdImage %s has only %s RefStars (desired %s)",
1394 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1396 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1398 sigmaRelativeTolerance=0,
1400 doLineSearch=False):
1401 """Run fitter.minimize up to max_steps times, returning the final chi2.
1406 The star/reference star associations to fit.
1408 The fitter to use for minimization.
1410 Maximum number of steps to run outlier rejection before declaring
1411 convergence failure.
1412 name : {
'photometry' or 'astrometry'}
1413 What type of data are we fitting (
for logs
and debugging files).
1415 Passed to ``fitter.minimize()`` to define the parameters to fit.
1416 dataName : `str`, optional
1417 Descriptive name
for this dataset (e.g. tract
and filter),
1419 sigmaRelativeTolerance : `float`, optional
1420 Convergence tolerance
for the fractional change
in the chi2 cut
1421 level
for determining outliers. If set to zero, iterations will
1422 continue until there are no outliers.
1423 doRankUpdate : `bool`, optional
1424 Do an Eigen rank update during minimization,
or recompute the full
1425 matrix
and gradient?
1426 doLineSearch : `bool`, optional
1427 Do a line search
for the optimum step during minimization?
1432 The final chi2 after the fit converges,
or is forced to end.
1437 Raised
if the fitter fails
with a non-finite value.
1439 Raised
if the fitter fails
for some other reason;
1440 log messages will provide further details.
1442 if self.config.writeInitMatrix:
1443 dumpMatrixFile = self.
_getDebugPath(f
"{name}_postinit-{dataName}")
1447 oldChi2.chi2 = float(
"inf")
1448 for i
in range(max_steps):
1449 if self.config.writeChi2FilesOuterLoop:
1450 writeChi2Name = f
"{name}_iterate_{i}_chi2-{dataName}"
1452 writeChi2Name =
None
1453 result = fitter.minimize(whatToFit,
1454 self.config.outlierRejectSigma,
1455 sigmaRelativeTolerance=sigmaRelativeTolerance,
1456 doRankUpdate=doRankUpdate,
1457 doLineSearch=doLineSearch,
1458 dumpMatrixFile=dumpMatrixFile)
1461 f
"Fit iteration {i}", writeChi2Name=writeChi2Name)
1463 if result == MinimizeResult.Converged:
1465 self.log.debug(
"fit has converged - no more outliers - redo minimization "
1466 "one more time in case we have lost accuracy in rank update.")
1468 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma,
1469 sigmaRelativeTolerance=sigmaRelativeTolerance)
1473 if chi2.chi2/chi2.ndof >= 4.0:
1474 self.log.error(
"Potentially bad fit: High chi-squared/ndof.")
1477 elif result == MinimizeResult.Chi2Increased:
1478 self.log.warning(
"Still some outliers remaining but chi2 increased - retry")
1480 chi2Ratio = chi2.chi2 / oldChi2.chi2
1482 self.log.warning(
'Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1483 chi2.chi2, oldChi2.chi2, chi2Ratio)
1490 msg = (
"Large chi2 increase between steps: fit likely cannot converge."
1491 " Try setting one or more of the `writeChi2*` config fields and looking"
1492 " at how individual star chi2-values evolve during the fit.")
1493 raise RuntimeError(msg)
1495 elif result == MinimizeResult.NonFinite:
1496 filename = self.
_getDebugPath(
"{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1498 fitter.saveChi2Contributions(filename+
"{type}")
1499 msg =
"Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1500 raise FloatingPointError(msg.format(filename))
1501 elif result == MinimizeResult.Failed:
1502 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
1504 raise RuntimeError(
"Unxepected return code from minimize().")
1506 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
1510 def _make_output(self, ccdImageList, model, func):
1511 """Return the internal jointcal models converted to the afw
1512 structures that will be saved to disk.
1517 The list of CcdImages to get the output for.
1521 The name of the function to call on ``model`` to get the converted
1528 The data to be saved, keyed on (visit, detector).
1531 for ccdImage
in ccdImageList:
1532 ccd = ccdImage.ccdId
1533 visit = ccdImage.visit
1534 self.log.debug(
"%s for visit: %d, ccd: %d", func, visit, ccd)
1535 output[(visit, ccd)] = getattr(model, func)(ccdImage)
1538 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1540 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1545 The star/reference star associations to fit.
1547 The astrometric model that was fit.
1548 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1549 Dict of ccdImage identifiers to dataRefs that were fit.
1551 ccdImageList = associations.getCcdImageList()
1552 output = self._make_output(ccdImageList, model, "makeSkyWcs")
1553 for key, skyWcs
in output.items():
1554 dataRef = visit_ccd_to_dataRef[key]
1556 dataRef.put(skyWcs,
'jointcal_wcs')
1557 except pexExceptions.Exception
as e:
1558 self.log.fatal(
'Failed to write updated Wcs: %s',
str(e))
1561 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1563 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1568 The star/reference star associations to fit.
1570 The photoometric model that was fit.
1571 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1572 Dict of ccdImage identifiers to dataRefs that were fit.
1575 ccdImageList = associations.getCcdImageList()
1576 output = self._make_output(ccdImageList, model, "toPhotoCalib")
1577 for key, photoCalib
in output.items():
1578 dataRef = visit_ccd_to_dataRef[key]
1580 dataRef.put(photoCalib,
'jointcal_photoCalib')
1581 except pexExceptions.Exception
as e:
1582 self.log.fatal(
'Failed to write updated PhotoCalib: %s',
str(e))
1587 """Return an afw SourceTable to use as a base for creating the
1588 SourceCatalog to insert values from the dataFrame into.
1593 Table
with schema
and slots to use to make SourceCatalogs.
1596 schema.addField("centroid_x",
"D")
1597 schema.addField(
"centroid_y",
"D")
1598 schema.addField(
"centroid_xErr",
"F")
1599 schema.addField(
"centroid_yErr",
"F")
1600 schema.addField(
"shape_xx",
"D")
1601 schema.addField(
"shape_yy",
"D")
1602 schema.addField(
"shape_xy",
"D")
1603 schema.addField(
"flux_instFlux",
"D")
1604 schema.addField(
"flux_instFluxErr",
"D")
1606 table.defineCentroid(
"centroid")
1607 table.defineShape(
"shape")
1613 Get the sourceTable_visit columns to load from the catalogs.
1618 List of columns known to be available
in the sourceTable_visit.
1619 config : `JointcalConfig`
1620 A filled-
in config to to help define column names.
1621 sourceSelector : `lsst.meas.algorithms.BaseSourceSelectorTask`
1622 A configured source selector to define column names to load.
1627 List of columns to read
from sourceTable_visit.
1628 detectorColumn : `str`
1629 Name of the detector column.
1631 Name of the ixx/iyy/ixy columns.
1633 if 'detector' in inColumns:
1635 detectorColumn =
'detector'
1638 detectorColumn =
'ccd'
1640 columns = [
'visit', detectorColumn,
1641 'sourceId',
'x',
'xErr',
'y',
'yErr',
1642 config.sourceFluxType +
'_instFlux', config.sourceFluxType +
'_instFluxErr']
1644 if 'ixx' in inColumns:
1646 ixxColumns = [
'ixx',
'iyy',
'ixy']
1649 ixxColumns = [
'Ixx',
'Iyy',
'Ixy']
1650 columns.extend(ixxColumns)
1652 if sourceSelector.config.doFlags:
1653 columns.extend(sourceSelector.config.flags.bad)
1654 if sourceSelector.config.doUnresolved:
1655 columns.append(sourceSelector.config.unresolved.name)
1656 if sourceSelector.config.doIsolated:
1657 columns.append(sourceSelector.config.isolated.parentName)
1658 columns.append(sourceSelector.config.isolated.nChildName)
1660 return columns, detectorColumn, ixxColumns
1664 detectorColumn, ixxColumns, sourceFluxType, log):
1665 """Return an afw SourceCatalog extracted from a visit-level dataframe,
1666 limited to just one detector.
1671 Table factory to use to make the SourceCatalog that will be
1672 populated with data
from ``visitCatalog``.
1673 visitCatalog : `pandas.DataFrame`
1674 DataFrame to extract a detector catalog
from.
1676 Numeric id of the detector to extract
from ``visitCatalog``.
1677 detectorColumn : `str`
1678 Name of the detector column
in the catalog.
1679 ixxColumns : `list` [`str`]
1680 Names of the ixx/iyy/ixy columns
in the catalog.
1681 sourceFluxType : `str`
1682 Name of the catalog field to load instFluxes
from.
1684 Logging instance to log to.
1689 Detector-level catalog extracted
from ``visitCatalog``,
or `
None`
1690 if there was no data to load.
1693 mapping = {
'x':
'centroid_x',
1695 'xErr':
'centroid_xErr',
1696 'yErr':
'centroid_yErr',
1697 ixxColumns[0]:
'shape_xx',
1698 ixxColumns[1]:
'shape_yy',
1699 ixxColumns[2]:
'shape_xy',
1700 f
'{sourceFluxType}_instFlux':
'flux_instFlux',
1701 f
'{sourceFluxType}_instFluxErr':
'flux_instFluxErr',
1705 matched = visitCatalog[detectorColumn] == detectorId
1709 catalog.resize(sum(matched))
1710 view = visitCatalog.loc[matched]
1711 catalog[
'id'] = view.index
1712 for dfCol, afwCol
in mapping.items():
1713 catalog[afwCol] = view[dfCol]
1715 log.debug(
"%d sources selected in visit %d detector %d",
1717 view[
'visit'].iloc[0],
static Schema makeMinimalSchema()
static std::shared_ptr< SourceTable > make(Schema const &schema, std::shared_ptr< IdFactory > const &idFactory)
static Schema makeMinimalSchema()
The class that implements the relations between MeasuredStar and FittedStar.
Class that handles the astrometric least squares problem.
Interface between AstrometryFit and the combinations of Mappings from pixels to some tangent plane (a...
Handler of an actual image from a single CCD.
Base class for Chi2Statistic and Chi2List, to allow addEntry inside Fitter for either class.
Simple structure to accumulate chi2 and ndof.
A multi-component model, fitting mappings for sensors and visits simultaneously.
A projection handler in which all CCDs from the same visit have the same tangent point.
Class that handles the photometric least squares problem.
A model where there is one independent transform per CcdImage.
def __init__(self, *config=None)
def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, jointcalControl, camera)
def _compute_proper_motion_epoch(self, ccdImageList)
def _get_refcat_coordinate_error_override(self, refCat, name)
def _getDebugPath(self, filename)
def _check_star_lists(self, associations, name)
def _make_one_input_data(self, visitRecord, catalog, detectorDict)
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", sigmaRelativeTolerance=0, doRankUpdate=True, doLineSearch=False)
def _check_stars(self, associations)
def _prep_sky(self, associations, filters)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, tract="", match_cut=3.0, reject_bad_fluxes=False, *name="", refObjLoader=None, referenceSelector=None, fit_function=None, epoch=None)
def _put_metrics(self, butlerQC, job, outputRefs)
def _build_ccdImage(self, data, associations, jointcalControl)
def __init__(self, **kwargs)
def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None)
def _fit_photometry(self, associations, dataName=None)
def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None)
def _make_output(self, ccdImageList, model, func)
def _put_output(self, butlerQC, outputs, outputRefs, camera, setter)
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, applyColorterms=False, epoch=None)
def _fit_astrometry(self, associations, dataName=None)
def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
def add_measurement(job, name, value)
def get_sourceTable_visit_columns(inColumns, config, sourceSelector)
def lookupVisitRefCats(datasetType, registry, quantumDataId, collections)
def extract_detector_catalog_from_visit_catalog(table, visitCatalog, detectorId, detectorColumn, ixxColumns, sourceFluxType, log)
def writeModel(model, filename, log)
This is a virtual class that allows a lot of freedom in the choice of the projection from "Sky" (wher...