lsst.fgcmcal  22.0.1-7-g65f59fa+54b92689ce
fgcmBuildStarsBase.py
Go to the documentation of this file.
1 # This file is part of fgcmcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """Base class for BuildStars using src tables or sourceTable_visit tables.
22 """
23 
24 import os
25 import sys
26 import traceback
27 import abc
28 
29 import numpy as np
30 
31 import lsst.daf.persistence as dafPersist
32 import lsst.pex.config as pexConfig
33 import lsst.pipe.base as pipeBase
34 import lsst.afw.table as afwTable
35 import lsst.geom as geom
36 from lsst.daf.base import PropertyList
37 from lsst.daf.base.dateTime import DateTime
38 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
39 
40 from .utilities import computeApertureRadiusFromDataRef
41 from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
42 
43 import fgcm
44 
45 REFSTARS_FORMAT_VERSION = 1
46 
47 __all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']
48 
49 
50 class FgcmBuildStarsConfigBase(pexConfig.Config):
51  """Base config for FgcmBuildStars tasks"""
52 
53  instFluxField = pexConfig.Field(
54  doc=("Faull name of the source instFlux field to use, including 'instFlux'. "
55  "The associated flag will be implicitly included in badFlags"),
56  dtype=str,
57  default='slot_CalibFlux_instFlux',
58  )
59  minPerBand = pexConfig.Field(
60  doc="Minimum observations per band",
61  dtype=int,
62  default=2,
63  )
64  matchRadius = pexConfig.Field(
65  doc="Match radius (arcseconds)",
66  dtype=float,
67  default=1.0,
68  )
69  isolationRadius = pexConfig.Field(
70  doc="Isolation radius (arcseconds)",
71  dtype=float,
72  default=2.0,
73  )
74  densityCutNside = pexConfig.Field(
75  doc="Density cut healpix nside",
76  dtype=int,
77  default=128,
78  )
79  densityCutMaxPerPixel = pexConfig.Field(
80  doc="Density cut number of stars per pixel",
81  dtype=int,
82  default=1000,
83  )
84  matchNside = pexConfig.Field(
85  doc="Healpix Nside for matching",
86  dtype=int,
87  default=4096,
88  )
89  coarseNside = pexConfig.Field(
90  doc="Healpix coarse Nside for partitioning matches",
91  dtype=int,
92  default=8,
93  )
94  # The following config will not be necessary after Gen2 retirement.
95  # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band'
96  # which is easiest to access in the config file.
97  physicalFilterMap = pexConfig.DictField(
98  doc="Mapping from 'physicalFilter' to band.",
99  keytype=str,
100  itemtype=str,
101  default={},
102  )
103  requiredBands = pexConfig.ListField(
104  doc="Bands required for each star",
105  dtype=str,
106  default=(),
107  )
108  primaryBands = pexConfig.ListField(
109  doc=("Bands for 'primary' star matches. "
110  "A star must be observed in one of these bands to be considered "
111  "as a calibration star."),
112  dtype=str,
113  default=None
114  )
115  visitDataRefName = pexConfig.Field(
116  doc="dataRef name for the 'visit' field, usually 'visit'.",
117  dtype=str,
118  default="visit"
119  )
120  ccdDataRefName = pexConfig.Field(
121  doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
122  dtype=str,
123  default="ccd"
124  )
125  doApplyWcsJacobian = pexConfig.Field(
126  doc="Apply the jacobian of the WCS to the star observations prior to fit?",
127  dtype=bool,
128  default=True
129  )
130  doModelErrorsWithBackground = pexConfig.Field(
131  doc="Model flux errors with background term?",
132  dtype=bool,
133  default=True
134  )
135  psfCandidateName = pexConfig.Field(
136  doc="Name of field with psf candidate flag for propagation",
137  dtype=str,
138  default="calib_psf_candidate"
139  )
140  doSubtractLocalBackground = pexConfig.Field(
141  doc=("Subtract the local background before performing calibration? "
142  "This is only supported for circular aperture calibration fluxes."),
143  dtype=bool,
144  default=False
145  )
146  localBackgroundFluxField = pexConfig.Field(
147  doc="Full name of the local background instFlux field to use.",
148  dtype=str,
149  default='base_LocalBackground_instFlux'
150  )
151  sourceSelector = sourceSelectorRegistry.makeField(
152  doc="How to select sources",
153  default="science"
154  )
155  apertureInnerInstFluxField = pexConfig.Field(
156  doc=("Full name of instFlux field that contains inner aperture "
157  "flux for aperture correction proxy"),
158  dtype=str,
159  default='base_CircularApertureFlux_12_0_instFlux'
160  )
161  apertureOuterInstFluxField = pexConfig.Field(
162  doc=("Full name of instFlux field that contains outer aperture "
163  "flux for aperture correction proxy"),
164  dtype=str,
165  default='base_CircularApertureFlux_17_0_instFlux'
166  )
167  doReferenceMatches = pexConfig.Field(
168  doc="Match reference catalog as additional constraint on calibration",
169  dtype=bool,
170  default=True,
171  )
172  fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
173  target=FgcmLoadReferenceCatalogTask,
174  doc="FGCM reference object loader",
175  )
176  nVisitsPerCheckpoint = pexConfig.Field(
177  doc="Number of visits read between checkpoints",
178  dtype=int,
179  default=500,
180  )
181 
182  def setDefaults(self):
183  sourceSelector = self.sourceSelectorsourceSelector["science"]
184  sourceSelector.setDefaults()
185 
186  sourceSelector.doFlags = True
187  sourceSelector.doUnresolved = True
188  sourceSelector.doSignalToNoise = True
189  sourceSelector.doIsolated = True
190 
191  sourceSelector.signalToNoise.minimum = 10.0
192  sourceSelector.signalToNoise.maximum = 1000.0
193 
194  # FGCM operates on unresolved sources, and this setting is
195  # appropriate for the current base_ClassificationExtendedness
196  sourceSelector.unresolved.maximum = 0.5
197 
198 
199 class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
200  """Subclass of TaskRunner for FgcmBuildStars tasks
201 
202  fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of
203  arguments, one of which is the butler (for persistence and mapper data),
204  and a list of dataRefs extracted from the command line. Note that FGCM
205  runs on a large set of dataRefs, and not on single dataRef/tract/patch.
206  This class transforms the process arguments generated by the ArgumentParser
207  into the arguments expected by FgcmBuildStarsTask.run(). This runner does
208  not use any parallelization.
209  """
210  @staticmethod
211  def getTargetList(parsedCmd):
212  """
213  Return a list with one element: a tuple with the butler and
214  list of dataRefs
215  """
216  # we want to combine the butler with any (or no!) dataRefs
217  return [(parsedCmd.butler, parsedCmd.id.refList)]
218 
219  def __call__(self, args):
220  """
221  Parameters
222  ----------
223  args: `tuple` with (butler, dataRefList)
224 
225  Returns
226  -------
227  exitStatus: `list` with `lsst.pipe.base.Struct`
228  exitStatus (0: success; 1: failure)
229  """
230  butler, dataRefList = args
231 
232  task = self.TaskClass(config=self.config, log=self.log)
233 
234  exitStatus = 0
235  if self.doRaise:
236  task.runDataRef(butler, dataRefList)
237  else:
238  try:
239  task.runDataRef(butler, dataRefList)
240  except Exception as e:
241  exitStatus = 1
242  task.log.fatal("Failed: %s" % e)
243  if not isinstance(e, pipeBase.TaskError):
244  traceback.print_exc(file=sys.stderr)
245 
246  task.writeMetadata(butler)
247 
248  # The task does not return any results:
249  return [pipeBase.Struct(exitStatus=exitStatus)]
250 
251  def run(self, parsedCmd):
252  """
253  Run the task, with no multiprocessing
254 
255  Parameters
256  ----------
257  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
258  """
259 
260  resultList = []
261 
262  if self.precall(parsedCmd):
263  targetList = self.getTargetListgetTargetList(parsedCmd)
264  resultList = self(targetList[0])
265 
266  return resultList
267 
268 
269 class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC):
270  """
271  Base task to build stars for FGCM global calibration
272 
273  Parameters
274  ----------
275  butler : `lsst.daf.persistence.Butler`
276  """
277  def __init__(self, initInputs=None, butler=None, **kwargs):
278  super().__init__(**kwargs)
279 
280  self.makeSubtask("sourceSelector")
281  # Only log warning and fatal errors from the sourceSelector
282  self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
283 
284  # no saving of metadata for now
285  def _getMetadataName(self):
286  return None
287 
288  @pipeBase.timeMethod
289  def runDataRef(self, butler, dataRefs):
290  """
291  Cross-match and make star list for FGCM Input
292 
293  Parameters
294  ----------
295  butler: `lsst.daf.persistence.Butler`
296  dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
297  Source data references for the input visits.
298 
299  Raises
300  ------
301  RuntimeErrror: Raised if `config.doReferenceMatches` is set and
302  an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
303  fails if the calibFlux is not a CircularAperture flux.
304  """
305  datasetType = dataRefs[0].butlerSubset.datasetType
306  self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
307 
308  if self.config.doReferenceMatches:
309  self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
310  # Ensure that we have a LUT
311  if not butler.datasetExists('fgcmLookUpTable'):
312  raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
313  # Compute aperture radius if necessary. This is useful to do now before
314  # any heavy lifting has happened (fail early).
315  calibFluxApertureRadius = None
316  if self.config.doSubtractLocalBackground:
317  try:
318  calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
319  self.config.instFluxField)
320  except RuntimeError as e:
321  raise RuntimeError("Could not determine aperture radius from %s. "
322  "Cannot use doSubtractLocalBackground." %
323  (self.config.instFluxField)) from e
324 
325  camera = butler.get('camera')
326  groupedDataRefs = self._findAndGroupDataRefsGen2_findAndGroupDataRefsGen2(butler, camera, dataRefs)
327 
328  # Make the visit catalog if necessary
329  # First check if the visit catalog is in the _current_ path
330  # We cannot use Gen2 datasetExists() because that checks all parent
331  # directories as well, which would make recovering from faults
332  # and fgcmcal reruns impossible.
333  visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
334  filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
335  if os.path.exists(filename):
336  # This file exists and we should continue processing
337  inVisitCat = visitCatDataRef.get()
338  if len(inVisitCat) != len(groupedDataRefs):
339  raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
340  "number of visits. Cannot continue.")
341  else:
342  inVisitCat = None
343 
344  visitCat = self.fgcmMakeVisitCatalogfgcmMakeVisitCatalog(camera, groupedDataRefs,
345  visitCatDataRef=visitCatDataRef,
346  inVisitCat=inVisitCat)
347 
348  # Persist the visitCat as a checkpoint file.
349  visitCatDataRef.put(visitCat)
350 
351  starObsDataRef = butler.dataRef('fgcmStarObservations')
352  filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
353  if os.path.exists(filename):
354  inStarObsCat = starObsDataRef.get()
355  else:
356  inStarObsCat = None
357 
358  rad = calibFluxApertureRadius
359  sourceSchemaDataRef = butler.dataRef('src_schema')
360  sourceSchema = sourceSchemaDataRef.get('src_schema', immediate=True).schema
361  fgcmStarObservationCat = self.fgcmMakeAllStarObservationsfgcmMakeAllStarObservations(groupedDataRefs,
362  visitCat,
363  sourceSchema,
364  camera,
365  calibFluxApertureRadius=rad,
366  starObsDataRef=starObsDataRef,
367  visitCatDataRef=visitCatDataRef,
368  inStarObsCat=inStarObsCat)
369  visitCatDataRef.put(visitCat)
370  starObsDataRef.put(fgcmStarObservationCat)
371 
372  # Always do the matching.
373  if self.config.doReferenceMatches:
374  lutDataRef = butler.dataRef('fgcmLookUpTable')
375  else:
376  lutDataRef = None
377  fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStarsfgcmMatchStars(visitCat,
378  fgcmStarObservationCat,
379  lutDataRef=lutDataRef)
380 
381  # Persist catalogs via the butler
382  butler.put(fgcmStarIdCat, 'fgcmStarIds')
383  butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
384  if fgcmRefCat is not None:
385  butler.put(fgcmRefCat, 'fgcmReferenceStars')
386 
387  @abc.abstractmethod
388  def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs):
389  """
390  Find and group dataRefs (by visit); Gen2 only.
391 
392  Parameters
393  ----------
394  butler : `lsst.daf.persistence.Butler`
395  Gen2 butler.
396  camera : `lsst.afw.cameraGeom.Camera`
397  Camera from the butler.
398  dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
399  Data references for the input visits.
400 
401  Returns
402  -------
403  groupedDataRefs : `dict` [`int`, `list`]
404  Dictionary with sorted visit keys, and `list`s of
405  `lsst.daf.persistence.ButlerDataRef`
406  """
407  raise NotImplementedError("_findAndGroupDataRefsGen2 not implemented.")
408 
409  @abc.abstractmethod
410  def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
411  sourceSchema,
412  camera,
413  calibFluxApertureRadius=None,
414  visitCatDataRef=None,
415  starObsDataRef=None,
416  inStarObsCat=None):
417  """
418  Compile all good star observations from visits in visitCat. Checkpoint files
419  will be stored if both visitCatDataRef and starObsDataRef are not None.
420 
421  Parameters
422  ----------
423  groupedDataRefs : `dict` of `list`s
424  Lists of `~lsst.daf.persistence.ButlerDataRef` or
425  `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit.
426  visitCat : `~afw.table.BaseCatalog`
427  Catalog with visit data for FGCM
428  sourceSchema : `~lsst.afw.table.Schema`
429  Schema for the input src catalogs.
430  camera : `~lsst.afw.cameraGeom.Camera`
431  calibFluxApertureRadius : `float`, optional
432  Aperture radius for calibration flux.
433  visitCatDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional
434  Dataref to write visitCat for checkpoints
435  starObsDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional
436  Dataref to write the star observation catalog for checkpoints.
437  inStarObsCat : `~afw.table.BaseCatalog`
438  Input observation catalog. If this is incomplete, observations
439  will be appended from when it was cut off.
440 
441  Returns
442  -------
443  fgcmStarObservations : `afw.table.BaseCatalog`
444  Full catalog of good observations.
445 
446  Raises
447  ------
448  RuntimeError: Raised if doSubtractLocalBackground is True and
449  calibFluxApertureRadius is not set.
450  """
451  raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")
452 
453  def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None,
454  visitCatDataRef=None, inVisitCat=None):
455  """
456  Make a visit catalog with all the keys from each visit
457 
458  Parameters
459  ----------
460  camera: `lsst.afw.cameraGeom.Camera`
461  Camera from the butler
462  groupedDataRefs: `dict`
463  Dictionary with visit keys, and `list`s of
464  `lsst.daf.persistence.ButlerDataRef`
465  bkgDataRefDict: `dict`, optional
466  Dictionary of gen3 dataRefHandles for background info.
467  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
468  Dataref to write visitCat for checkpoints
469  inVisitCat: `afw.table.BaseCatalog`, optional
470  Input (possibly incomplete) visit catalog
471 
472  Returns
473  -------
474  visitCat: `afw.table.BaseCatalog`
475  """
476 
477  self.log.info("Assembling visitCatalog from %d %ss" %
478  (len(groupedDataRefs), self.config.visitDataRefName))
479 
480  nCcd = len(camera)
481 
482  if inVisitCat is None:
483  schema = self._makeFgcmVisitSchema_makeFgcmVisitSchema(nCcd)
484 
485  visitCat = afwTable.BaseCatalog(schema)
486  visitCat.reserve(len(groupedDataRefs))
487  visitCat.resize(len(groupedDataRefs))
488 
489  visitCat['visit'] = list(groupedDataRefs.keys())
490  visitCat['used'] = 0
491  visitCat['sources_read'] = False
492  else:
493  visitCat = inVisitCat
494 
495  # No matter what, fill the catalog. This will check if it was
496  # already read.
497  self._fillVisitCatalog_fillVisitCatalog(visitCat, groupedDataRefs,
498  bkgDataRefDict=bkgDataRefDict,
499  visitCatDataRef=visitCatDataRef)
500 
501  return visitCat
502 
503  def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None,
504  visitCatDataRef=None):
505  """
506  Fill the visit catalog with visit metadata
507 
508  Parameters
509  ----------
510  visitCat : `afw.table.BaseCatalog`
511  Visit catalog. See _makeFgcmVisitSchema() for schema definition.
512  groupedDataRefs : `dict`
513  Dictionary with visit keys, and `list`s of
514  `lsst.daf.persistence.ButlerDataRef` or
515  `lsst.daf.butler.DeferredDatasetHandle`
516  visitCatDataRef : `lsst.daf.persistence.ButlerDataRef`, optional
517  Dataref to write ``visitCat`` for checkpoints. Gen2 only.
518  bkgDataRefDict : `dict`, optional
519  Dictionary of Gen3 `lsst.daf.butler.DeferredDatasetHandle`
520  for background info.
521  """
522  bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
523 
524  for i, visit in enumerate(groupedDataRefs):
525  # We don't use the bypasses since we need the psf info which does
526  # not have a bypass
527  # TODO: When DM-15500 is implemented in the Gen3 Butler, this
528  # can be fixed
529 
530  # Do not read those that have already been read
531  if visitCat['used'][i]:
532  continue
533 
534  if (i % self.config.nVisitsPerCheckpoint) == 0:
535  self.log.info("Retrieving metadata for %s %d (%d/%d)" %
536  (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
537  # Save checkpoint if desired
538  if visitCatDataRef is not None:
539  visitCatDataRef.put(visitCat)
540 
541  dataRef = groupedDataRefs[visit][0]
542  if isinstance(dataRef, dafPersist.ButlerDataRef):
543  # Gen2: calexp dataRef
544  # The first dataRef in the group will be the reference ccd (if available)
545  exp = dataRef.get(datasetType='calexp_sub', bbox=bbox)
546  visitInfo = exp.getInfo().getVisitInfo()
547  label = dataRef.get(datasetType='calexp_filterLabel')
548  physicalFilter = label.physicalLabel
549  psf = exp.getPsf()
550  psfSigma = psf.computeShape().getDeterminantRadius()
551  else:
552  # Gen3: use the visitSummary dataRef
553  summary = dataRef.get()
554 
555  summaryRow = summary.find(self.config.referenceCCD)
556  if summaryRow is None:
557  # Take the first available ccd if reference isn't available
558  summaryRow = summary[0]
559 
560  summaryDetector = summaryRow['id']
561  visitInfo = summaryRow.getVisitInfo()
562  physicalFilter = summaryRow['physical_filter']
563  # Compute the median psf sigma if possible
564  goodSigma, = np.where(summary['psfSigma'] > 0)
565  if goodSigma.size > 2:
566  psfSigma = np.median(summary['psfSigma'][goodSigma])
567  elif goodSigma > 0:
568  psfSigma = np.mean(summary['psfSigma'][goodSigma])
569  else:
570  psfSigma = 0.0
571 
572  rec = visitCat[i]
573  rec['visit'] = visit
574  rec['physicalFilter'] = physicalFilter
575  # TODO DM-26991: when gen2 is removed, gen3 workflow will make it
576  # much easier to get the wcs's necessary to recompute the pointing
577  # ra/dec at the center of the camera.
578  radec = visitInfo.getBoresightRaDec()
579  rec['telra'] = radec.getRa().asDegrees()
580  rec['teldec'] = radec.getDec().asDegrees()
581  rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
582  rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
583  rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
584  rec['exptime'] = visitInfo.getExposureTime()
585  # convert from Pa to millibar
586  # Note that I don't know if this unit will need to be per-camera config
587  rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
588  # Flag to signify if this is a "deep" field. Not currently used
589  rec['deepFlag'] = 0
590  # Relative flat scaling (1.0 means no relative scaling)
591  rec['scaling'][:] = 1.0
592  # Median delta aperture, to be measured from stars
593  rec['deltaAper'] = 0.0
594  rec['psfSigma'] = psfSigma
595 
596  if self.config.doModelErrorsWithBackground:
597  foundBkg = False
598  if isinstance(dataRef, dafPersist.ButlerDataRef):
599  # Gen2-style dataRef
600  det = dataRef.dataId[self.config.ccdDataRefName]
601  if dataRef.datasetExists(datasetType='calexpBackground'):
602  bgList = dataRef.get(datasetType='calexpBackground')
603  foundBkg = True
604  else:
605  # Gen3-style dataRef
606  try:
607  # Use the same detector used from the summary.
608  bkgRef = bkgDataRefDict[(visit, summaryDetector)]
609  bgList = bkgRef.get()
610  foundBkg = True
611  except KeyError:
612  pass
613 
614  if foundBkg:
615  bgStats = (bg[0].getStatsImage().getImage().array
616  for bg in bgList)
617  rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
618  else:
619  self.log.warn('Sky background not found for visit %d / ccd %d' %
620  (visit, det))
621  rec['skyBackground'] = -1.0
622  else:
623  rec['skyBackground'] = -1.0
624 
625  rec['used'] = 1
626 
627  def _makeSourceMapper(self, sourceSchema):
628  """
629  Make a schema mapper for fgcm sources
630 
631  Parameters
632  ----------
633  sourceSchema: `afwTable.Schema`
634  Default source schema from the butler
635 
636  Returns
637  -------
638  sourceMapper: `afwTable.schemaMapper`
639  Mapper to the FGCM source schema
640  """
641 
642  # create a mapper to the preferred output
643  sourceMapper = afwTable.SchemaMapper(sourceSchema)
644 
645  # map to ra/dec
646  sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
647  sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
648  sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
649  sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
650  # Add the mapping if the field exists in the input catalog.
651  # If the field does not exist, simply add it (set to False).
652  # This field is not required for calibration, but is useful
653  # to collate if available.
654  try:
655  sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
656  'psf_candidate')
657  except LookupError:
658  sourceMapper.editOutputSchema().addField(
659  "psf_candidate", type='Flag',
660  doc=("Flag set if the source was a candidate for PSF determination, "
661  "as determined by the star selector."))
662 
663  # and add the fields we want
664  sourceMapper.editOutputSchema().addField(
665  "visit", type=np.int32, doc="Visit number")
666  sourceMapper.editOutputSchema().addField(
667  "ccd", type=np.int32, doc="CCD number")
668  sourceMapper.editOutputSchema().addField(
669  "instMag", type=np.float32, doc="Instrumental magnitude")
670  sourceMapper.editOutputSchema().addField(
671  "instMagErr", type=np.float32, doc="Instrumental magnitude error")
672  sourceMapper.editOutputSchema().addField(
673  "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
674  sourceMapper.editOutputSchema().addField(
675  "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset")
676 
677  return sourceMapper
678 
679  def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None):
680  """
681  Use FGCM code to match observations into unique stars.
682 
683  Parameters
684  ----------
685  visitCat: `afw.table.BaseCatalog`
686  Catalog with visit data for fgcm
687  obsCat: `afw.table.BaseCatalog`
688  Full catalog of star observations for fgcm
689  lutDataRef: `lsst.daf.persistence.ButlerDataRef` or
690  `lsst.daf.butler.DeferredDatasetHandle`, optional
691  Data reference to fgcm look-up table (used if matching reference stars).
692 
693  Returns
694  -------
695  fgcmStarIdCat: `afw.table.BaseCatalog`
696  Catalog of unique star identifiers and index keys
697  fgcmStarIndicesCat: `afwTable.BaseCatalog`
698  Catalog of unique star indices
699  fgcmRefCat: `afw.table.BaseCatalog`
700  Catalog of matched reference stars.
701  Will be None if `config.doReferenceMatches` is False.
702  """
703  # get filter names into a numpy array...
704  # This is the type that is expected by the fgcm code
705  visitFilterNames = np.zeros(len(visitCat), dtype='a30')
706  for i in range(len(visitCat)):
707  visitFilterNames[i] = visitCat[i]['physicalFilter']
708 
709  # match to put filterNames with observations
710  visitIndex = np.searchsorted(visitCat['visit'],
711  obsCat['visit'])
712 
713  obsFilterNames = visitFilterNames[visitIndex]
714 
715  if self.config.doReferenceMatches:
716  # Get the reference filter names, using the LUT
717  lutCat = lutDataRef.get()
718 
719  stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
720  zip(lutCat[0]['physicalFilters'].split(','),
721  lutCat[0]['stdPhysicalFilters'].split(','))}
722  stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
723  zip(lutCat[0]['stdPhysicalFilters'].split(','),
724  lutCat[0]['lambdaStdFilter'])}
725 
726  del lutCat
727 
728  referenceFilterNames = self._getReferenceFilterNames_getReferenceFilterNames(visitCat,
729  stdFilterDict,
730  stdLambdaDict)
731  self.log.info("Using the following reference filters: %s" %
732  (', '.join(referenceFilterNames)))
733 
734  else:
735  # This should be an empty list
736  referenceFilterNames = []
737 
738  # make the fgcm starConfig dict
739  starConfig = {'logger': self.log,
740  'filterToBand': self.config.physicalFilterMap,
741  'requiredBands': self.config.requiredBands,
742  'minPerBand': self.config.minPerBand,
743  'matchRadius': self.config.matchRadius,
744  'isolationRadius': self.config.isolationRadius,
745  'matchNSide': self.config.matchNside,
746  'coarseNSide': self.config.coarseNside,
747  'densNSide': self.config.densityCutNside,
748  'densMaxPerPixel': self.config.densityCutMaxPerPixel,
749  'primaryBands': self.config.primaryBands,
750  'referenceFilterNames': referenceFilterNames}
751 
752  # initialize the FgcmMakeStars object
753  fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
754 
755  # make the primary stars
756  # note that the ra/dec native Angle format is radians
757  # We determine the conversion from the native units (typically
758  # radians) to degrees for the first observation. This allows us
759  # to treate ra/dec as numpy arrays rather than Angles, which would
760  # be approximately 600x slower.
761  conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
762  fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
763  obsCat['dec'] * conv,
764  filterNameArray=obsFilterNames,
765  bandSelected=False)
766 
767  # and match all the stars
768  fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
769  obsCat['dec'] * conv,
770  obsFilterNames)
771 
772  if self.config.doReferenceMatches:
773  fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
774 
775  # now persist
776 
777  objSchema = self._makeFgcmObjSchema_makeFgcmObjSchema()
778 
779  # make catalog and records
780  fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
781  fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
782  for i in range(fgcmMakeStars.objIndexCat.size):
783  fgcmStarIdCat.addNew()
784 
785  # fill the catalog
786  fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
787  fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
788  fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
789  fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
790  fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
791 
792  obsSchema = self._makeFgcmObsSchema_makeFgcmObsSchema()
793 
794  fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
795  fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
796  for i in range(fgcmMakeStars.obsIndexCat.size):
797  fgcmStarIndicesCat.addNew()
798 
799  fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
800 
801  if self.config.doReferenceMatches:
802  refSchema = self._makeFgcmRefSchema_makeFgcmRefSchema(len(referenceFilterNames))
803 
804  fgcmRefCat = afwTable.BaseCatalog(refSchema)
805  fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
806 
807  for i in range(fgcmMakeStars.referenceCat.size):
808  fgcmRefCat.addNew()
809 
810  fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
811  fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
812  fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
813 
814  md = PropertyList()
815  md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
816  md.set("FILTERNAMES", referenceFilterNames)
817  fgcmRefCat.setMetadata(md)
818 
819  else:
820  fgcmRefCat = None
821 
822  return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
823 
824  def _makeFgcmVisitSchema(self, nCcd):
825  """
826  Make a schema for an fgcmVisitCatalog
827 
828  Parameters
829  ----------
830  nCcd: `int`
831  Number of CCDs in the camera
832 
833  Returns
834  -------
835  schema: `afwTable.Schema`
836  """
837 
838  schema = afwTable.Schema()
839  schema.addField('visit', type=np.int32, doc="Visit number")
840  schema.addField('physicalFilter', type=str, size=30, doc="Physical filter")
841  schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
842  schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
843  schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
844  schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
845  schema.addField('mjd', type=np.float64, doc="MJD of visit")
846  schema.addField('exptime', type=np.float32, doc="Exposure time")
847  schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
848  schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
849  schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
850  schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
851  # the following field is not used yet
852  schema.addField('deepFlag', type=np.int32, doc="Deep observation")
853  schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
854  size=nCcd)
855  schema.addField('used', type=np.int32, doc="This visit has been ingested.")
856  schema.addField('sources_read', type='Flag', doc="This visit had sources read.")
857 
858  return schema
859 
860  def _makeFgcmObjSchema(self):
861  """
862  Make a schema for the objIndexCat from fgcmMakeStars
863 
864  Returns
865  -------
866  schema: `afwTable.Schema`
867  """
868 
869  objSchema = afwTable.Schema()
870  objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
871  # Will investigate making these angles...
872  objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
873  objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
874  objSchema.addField('obsArrIndex', type=np.int32,
875  doc='Index in obsIndexTable for first observation')
876  objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
877 
878  return objSchema
879 
880  def _makeFgcmObsSchema(self):
881  """
882  Make a schema for the obsIndexCat from fgcmMakeStars
883 
884  Returns
885  -------
886  schema: `afwTable.Schema`
887  """
888 
889  obsSchema = afwTable.Schema()
890  obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
891 
892  return obsSchema
893 
894  def _makeFgcmRefSchema(self, nReferenceBands):
895  """
896  Make a schema for the referenceCat from fgcmMakeStars
897 
898  Parameters
899  ----------
900  nReferenceBands: `int`
901  Number of reference bands
902 
903  Returns
904  -------
905  schema: `afwTable.Schema`
906  """
907 
908  refSchema = afwTable.Schema()
909  refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
910  refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
911  size=nReferenceBands)
912  refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
913  size=nReferenceBands)
914 
915  return refSchema
916 
917  def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
918  """
919  Get the reference filter names, in wavelength order, from the visitCat and
920  information from the look-up-table.
921 
922  Parameters
923  ----------
924  visitCat: `afw.table.BaseCatalog`
925  Catalog with visit data for FGCM
926  stdFilterDict: `dict`
927  Mapping of filterName to stdFilterName from LUT
928  stdLambdaDict: `dict`
929  Mapping of stdFilterName to stdLambda from LUT
930 
931  Returns
932  -------
933  referenceFilterNames: `list`
934  Wavelength-ordered list of reference filter names
935  """
936 
937  # Find the unique list of filter names in visitCat
938  filterNames = np.unique(visitCat.asAstropy()['physicalFilter'])
939 
940  # Find the unique list of "standard" filters
941  stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
942 
943  # And sort these by wavelength
944  referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
945 
946  return referenceFilterNames
def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None)
def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict)
def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None, visitCatDataRef=None)
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, sourceSchema, camera, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
def __init__(self, initInputs=None, butler=None, **kwargs)
def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None, visitCatDataRef=None, inVisitCat=None)
def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs)
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:807