lsst.fgcmcal  20.0.0-5-gfcebe35+1741cdab20
fgcmBuildStarsBase.py
Go to the documentation of this file.
1 # This file is part of fgcmcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """Base class for BuildStars using src tables or sourceTable_visit tables.
22 """
23 
24 import os
25 import sys
26 import traceback
27 import abc
28 
29 import numpy as np
30 
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 import lsst.afw.table as afwTable
34 import lsst.geom as geom
35 from lsst.daf.base import PropertyList
36 from lsst.daf.base.dateTime import DateTime
37 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
38 
39 from .utilities import computeApertureRadiusFromDataRef
40 from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask
41 
42 import fgcm
43 
44 REFSTARS_FORMAT_VERSION = 1
45 
46 __all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']
47 
48 
49 class FgcmBuildStarsConfigBase(pexConfig.Config):
50  """Base config for FgcmBuildStars tasks"""
51 
52  instFluxField = pexConfig.Field(
53  doc=("Faull name of the source instFlux field to use, including 'instFlux'. "
54  "The associated flag will be implicitly included in badFlags"),
55  dtype=str,
56  default='slot_CalibFlux_instFlux',
57  )
58  minPerBand = pexConfig.Field(
59  doc="Minimum observations per band",
60  dtype=int,
61  default=2,
62  )
63  matchRadius = pexConfig.Field(
64  doc="Match radius (arcseconds)",
65  dtype=float,
66  default=1.0,
67  )
68  isolationRadius = pexConfig.Field(
69  doc="Isolation radius (arcseconds)",
70  dtype=float,
71  default=2.0,
72  )
73  densityCutNside = pexConfig.Field(
74  doc="Density cut healpix nside",
75  dtype=int,
76  default=128,
77  )
78  densityCutMaxPerPixel = pexConfig.Field(
79  doc="Density cut number of stars per pixel",
80  dtype=int,
81  default=1000,
82  )
83  matchNside = pexConfig.Field(
84  doc="Healpix Nside for matching",
85  dtype=int,
86  default=4096,
87  )
88  coarseNside = pexConfig.Field(
89  doc="Healpix coarse Nside for partitioning matches",
90  dtype=int,
91  default=8,
92  )
93  filterMap = pexConfig.DictField(
94  doc="Mapping from 'filterName' to band.",
95  keytype=str,
96  itemtype=str,
97  default={},
98  )
99  requiredBands = pexConfig.ListField(
100  doc="Bands required for each star",
101  dtype=str,
102  default=(),
103  )
104  primaryBands = pexConfig.ListField(
105  doc=("Bands for 'primary' star matches. "
106  "A star must be observed in one of these bands to be considered "
107  "as a calibration star."),
108  dtype=str,
109  default=None
110  )
111  visitDataRefName = pexConfig.Field(
112  doc="dataRef name for the 'visit' field, usually 'visit'.",
113  dtype=str,
114  default="visit"
115  )
116  ccdDataRefName = pexConfig.Field(
117  doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",
118  dtype=str,
119  default="ccd"
120  )
121  doApplyWcsJacobian = pexConfig.Field(
122  doc="Apply the jacobian of the WCS to the star observations prior to fit?",
123  dtype=bool,
124  default=True
125  )
126  psfCandidateName = pexConfig.Field(
127  doc="Name of field with psf candidate flag for propagation",
128  dtype=str,
129  default="calib_psf_candidate"
130  )
131  doSubtractLocalBackground = pexConfig.Field(
132  doc=("Subtract the local background before performing calibration? "
133  "This is only supported for circular aperture calibration fluxes."),
134  dtype=bool,
135  default=False
136  )
137  localBackgroundFluxField = pexConfig.Field(
138  doc="Full name of the local background instFlux field to use.",
139  dtype=str,
140  default='base_LocalBackground_instFlux'
141  )
142  sourceSelector = sourceSelectorRegistry.makeField(
143  doc="How to select sources",
144  default="science"
145  )
146  apertureInnerInstFluxField = pexConfig.Field(
147  doc=("Full name of instFlux field that contains inner aperture "
148  "flux for aperture correction proxy"),
149  dtype=str,
150  default='base_CircularApertureFlux_12_0_instFlux'
151  )
152  apertureOuterInstFluxField = pexConfig.Field(
153  doc=("Full name of instFlux field that contains outer aperture "
154  "flux for aperture correction proxy"),
155  dtype=str,
156  default='base_CircularApertureFlux_17_0_instFlux'
157  )
158  doReferenceMatches = pexConfig.Field(
159  doc="Match reference catalog as additional constraint on calibration",
160  dtype=bool,
161  default=True,
162  )
163  fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(
164  target=FgcmLoadReferenceCatalogTask,
165  doc="FGCM reference object loader",
166  )
167  nVisitsPerCheckpoint = pexConfig.Field(
168  doc="Number of visits read between checkpoints",
169  dtype=int,
170  default=500,
171  )
172 
173  def setDefaults(self):
174  sourceSelector = self.sourceSelector["science"]
175  sourceSelector.setDefaults()
176 
177  sourceSelector.doFlags = True
178  sourceSelector.doUnresolved = True
179  sourceSelector.doSignalToNoise = True
180  sourceSelector.doIsolated = True
181 
182  sourceSelector.signalToNoise.minimum = 10.0
183  sourceSelector.signalToNoise.maximum = 1000.0
184 
185  # FGCM operates on unresolved sources, and this setting is
186  # appropriate for the current base_ClassificationExtendedness
187  sourceSelector.unresolved.maximum = 0.5
188 
189 
190 class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):
191  """Subclass of TaskRunner for FgcmBuildStars tasks
192 
193  fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of
194  arguments, one of which is the butler (for persistence and mapper data),
195  and a list of dataRefs extracted from the command line. Note that FGCM
196  runs on a large set of dataRefs, and not on single dataRef/tract/patch.
197  This class transforms the process arguments generated by the ArgumentParser
198  into the arguments expected by FgcmBuildStarsTask.run(). This runner does
199  not use any parallelization.
200  """
201  @staticmethod
202  def getTargetList(parsedCmd):
203  """
204  Return a list with one element: a tuple with the butler and
205  list of dataRefs
206  """
207  # we want to combine the butler with any (or no!) dataRefs
208  return [(parsedCmd.butler, parsedCmd.id.refList)]
209 
210  def __call__(self, args):
211  """
212  Parameters
213  ----------
214  args: `tuple` with (butler, dataRefList)
215 
216  Returns
217  -------
218  exitStatus: `list` with `lsst.pipe.base.Struct`
219  exitStatus (0: success; 1: failure)
220  """
221  butler, dataRefList = args
222 
223  task = self.TaskClass(config=self.config, log=self.log)
224 
225  exitStatus = 0
226  if self.doRaise:
227  task.runDataRef(butler, dataRefList)
228  else:
229  try:
230  task.runDataRef(butler, dataRefList)
231  except Exception as e:
232  exitStatus = 1
233  task.log.fatal("Failed: %s" % e)
234  if not isinstance(e, pipeBase.TaskError):
235  traceback.print_exc(file=sys.stderr)
236 
237  task.writeMetadata(butler)
238 
239  # The task does not return any results:
240  return [pipeBase.Struct(exitStatus=exitStatus)]
241 
242  def run(self, parsedCmd):
243  """
244  Run the task, with no multiprocessing
245 
246  Parameters
247  ----------
248  parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line
249  """
250 
251  resultList = []
252 
253  if self.precall(parsedCmd):
254  targetList = self.getTargetList(parsedCmd)
255  resultList = self(targetList[0])
256 
257  return resultList
258 
259 
260 class FgcmBuildStarsBaseTask(pipeBase.CmdLineTask, abc.ABC):
261  """
262  Base task to build stars for FGCM global calibration
263 
264  Parameters
265  ----------
266  butler : `lsst.daf.persistence.Butler`
267  """
268  def __init__(self, butler=None, **kwargs):
269  pipeBase.CmdLineTask.__init__(self, **kwargs)
270  self.makeSubtask("sourceSelector")
271  # Only log warning and fatal errors from the sourceSelector
272  self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)
273 
274  # no saving of metadata for now
275  def _getMetadataName(self):
276  return None
277 
278  @pipeBase.timeMethod
279  def runDataRef(self, butler, dataRefs):
280  """
281  Cross-match and make star list for FGCM Input
282 
283  Parameters
284  ----------
285  butler: `lsst.daf.persistence.Butler`
286  dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
287  Source data references for the input visits.
288 
289  Raises
290  ------
291  RuntimeErrror: Raised if `config.doReferenceMatches` is set and
292  an fgcmLookUpTable is not available, or if computeFluxApertureRadius()
293  fails if the calibFlux is not a CircularAperture flux.
294  """
295  datasetType = dataRefs[0].butlerSubset.datasetType
296  self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)
297 
298  if self.config.doReferenceMatches:
299  # Ensure that we have a LUT
300  if not butler.datasetExists('fgcmLookUpTable'):
301  raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")
302  # Compute aperture radius if necessary. This is useful to do now before
303  # any heavy lifting has happened (fail early).
304  calibFluxApertureRadius = None
305  if self.config.doSubtractLocalBackground:
306  try:
307  calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],
308  self.config.instFluxField)
309  except RuntimeError as e:
310  raise RuntimeError("Could not determine aperture radius from %s. "
311  "Cannot use doSubtractLocalBackground." %
312  (self.config.instFluxField)) from e
313 
314  groupedDataRefs = self.findAndGroupDataRefs(butler, dataRefs)
315 
316  camera = butler.get('camera')
317 
318  # Make the visit catalog if necessary
319  # First check if the visit catalog is in the _current_ path
320  # We cannot use Gen2 datasetExists() because that checks all parent
321  # directories as well, which would make recovering from faults
322  # and fgcmcal reruns impossible.
323  visitCatDataRef = butler.dataRef('fgcmVisitCatalog')
324  filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]
325  if os.path.exists(filename):
326  # This file exists and we should continue processing
327  inVisitCat = visitCatDataRef.get()
328  if len(inVisitCat) != len(groupedDataRefs):
329  raise RuntimeError("Existing visitCatalog found, but has an inconsistent "
330  "number of visits. Cannot continue.")
331  else:
332  inVisitCat = None
333 
334  visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs,
335  visitCatDataRef=visitCatDataRef,
336  inVisitCat=inVisitCat)
337 
338  # Persist the visitCat as a checkpoint file.
339  visitCatDataRef.put(visitCat)
340 
341  starObsDataRef = butler.dataRef('fgcmStarObservations')
342  filename = starObsDataRef.get('fgcmStarObservations_filename')[0]
343  if os.path.exists(filename):
344  inStarObsCat = starObsDataRef.get()
345  else:
346  inStarObsCat = None
347 
348  rad = calibFluxApertureRadius
349  fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs,
350  visitCat,
351  calibFluxApertureRadius=rad,
352  starObsDataRef=starObsDataRef,
353  visitCatDataRef=visitCatDataRef,
354  inStarObsCat=inStarObsCat)
355  visitCatDataRef.put(visitCat)
356  starObsDataRef.put(fgcmStarObservationCat)
357 
358  # Always do the matching.
359  fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(butler,
360  visitCat,
361  fgcmStarObservationCat)
362 
363  # Persist catalogs via the butler
364  butler.put(fgcmStarIdCat, 'fgcmStarIds')
365  butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
366  if fgcmRefCat is not None:
367  butler.put(fgcmRefCat, 'fgcmReferenceStars')
368 
369  @abc.abstractmethod
370  def findAndGroupDataRefs(self, butler, dataRefs):
371  """
372  Find and group dataRefs (by visit).
373 
374  Parameters
375  ----------
376  butler: `lsst.daf.persistence.Butler`
377  dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
378  Data references for the input visits.
379 
380  Returns
381  -------
382  groupedDataRefs: `dict` [`int`, `list`]
383  Dictionary with visit keys, and `list`s of `lsst.daf.persistence.ButlerDataRef`
384  """
385  raise NotImplementedError("findAndGroupDataRefs not implemented.")
386 
387  @abc.abstractmethod
388  def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
389  calibFluxApertureRadius=None,
390  visitCatDataRef=None,
391  starObsDataRef=None,
392  inStarObsCat=None):
393  """
394  Compile all good star observations from visits in visitCat. Checkpoint files
395  will be stored if both visitCatDataRef and starObsDataRef are not None.
396 
397  Parameters
398  ----------
399  groupedDataRefs: `dict` of `list`s
400  Lists of `lsst.daf.persistence.ButlerDataRef`, grouped by visit.
401  visitCat: `afw.table.BaseCatalog`
402  Catalog with visit data for FGCM
403  calibFluxApertureRadius: `float`, optional
404  Aperture radius for calibration flux.
405  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
406  Dataref to write visitCat for checkpoints
407  starObsDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
408  Dataref to write the star observation catalog for checkpoints.
409  inStarObsCat: `afw.table.BaseCatalog`
410  Input observation catalog. If this is incomplete, observations
411  will be appended from when it was cut off.
412 
413  Returns
414  -------
415  fgcmStarObservations: `afw.table.BaseCatalog`
416  Full catalog of good observations.
417 
418  Raises
419  ------
420  RuntimeError: Raised if doSubtractLocalBackground is True and
421  calibFluxApertureRadius is not set.
422  """
423  raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")
424 
425  def fgcmMakeVisitCatalog(self, camera, groupedDataRefs,
426  visitCatDataRef=None, inVisitCat=None):
427  """
428  Make a visit catalog with all the keys from each visit
429 
430  Parameters
431  ----------
432  camera: `lsst.afw.cameraGeom.Camera`
433  Camera from the butler
434  groupedDataRefs: `dict`
435  Dictionary with visit keys, and `list`s of
436  `lsst.daf.persistence.ButlerDataRef`
437  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
438  Dataref to write visitCat for checkpoints
439  inVisitCat: `afw.table.BaseCatalog`
440  Input (possibly incomplete) visit catalog
441 
442  Returns
443  -------
444  visitCat: `afw.table.BaseCatalog`
445  """
446 
447  self.log.info("Assembling visitCatalog from %d %ss" %
448  (len(groupedDataRefs), self.config.visitDataRefName))
449 
450  nCcd = len(camera)
451 
452  if inVisitCat is None:
453  schema = self._makeFgcmVisitSchema(nCcd)
454 
455  visitCat = afwTable.BaseCatalog(schema)
456  visitCat.reserve(len(groupedDataRefs))
457 
458  for i, visit in enumerate(sorted(groupedDataRefs)):
459  rec = visitCat.addNew()
460  rec['visit'] = visit
461  rec['used'] = 0
462  rec['sources_read'] = 0
463  else:
464  visitCat = inVisitCat
465 
466  # No matter what, fill the catalog. This will check if it was
467  # already read.
468  self._fillVisitCatalog(visitCat, groupedDataRefs,
469  visitCatDataRef=visitCatDataRef)
470 
471  return visitCat
472 
473  def _fillVisitCatalog(self, visitCat, groupedDataRefs,
474  visitCatDataRef=None):
475  """
476  Fill the visit catalog with visit metadata
477 
478  Parameters
479  ----------
480  visitCat: `afw.table.BaseCatalog`
481  Catalog with schema from _makeFgcmVisitSchema()
482  groupedDataRefs: `dict`
483  Dictionary with visit keys, and `list`s of
484  `lsst.daf.persistence.ButlerDataRef
485  visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
486  Dataref to write visitCat for checkpoints
487  """
488 
489  bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
490 
491  for i, visit in enumerate(sorted(groupedDataRefs)):
492  # We don't use the bypasses since we need the psf info which does
493  # not have a bypass
494  # TODO: When DM-15500 is implemented in the Gen3 Butler, this
495  # can be fixed
496 
497  # Do not read those that have already been read
498  if visitCat['used'][i]:
499  continue
500 
501  if (i % self.config.nVisitsPerCheckpoint) == 0:
502  self.log.info("Retrieving metadata for %s %d (%d/%d)" %
503  (self.config.visitDataRefName, visit, i, len(groupedDataRefs)))
504  # Save checkpoint if desired
505  if visitCatDataRef is not None:
506  visitCatDataRef.put(visitCat)
507 
508  # Note that the reference ccd is first in the list (if available).
509 
510  # The first dataRef in the group will be the reference ccd (if available)
511  dataRef = groupedDataRefs[visit][0]
512 
513  exp = dataRef.get(datasetType='calexp_sub', bbox=bbox,
514  flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
515 
516  visitInfo = exp.getInfo().getVisitInfo()
517  f = exp.getFilter()
518  psf = exp.getPsf()
519 
520  rec = visitCat[i]
521  rec['visit'] = visit
522  rec['filtername'] = f.getName()
523  radec = visitInfo.getBoresightRaDec()
524  rec['telra'] = radec.getRa().asDegrees()
525  rec['teldec'] = radec.getDec().asDegrees()
526  rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
527  rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
528  rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
529  rec['exptime'] = visitInfo.getExposureTime()
530  # convert from Pa to millibar
531  # Note that I don't know if this unit will need to be per-camera config
532  rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
533  # Flag to signify if this is a "deep" field. Not currently used
534  rec['deepFlag'] = 0
535  # Relative flat scaling (1.0 means no relative scaling)
536  rec['scaling'][:] = 1.0
537  # Median delta aperture, to be measured from stars
538  rec['deltaAper'] = 0.0
539 
540  rec['psfSigma'] = psf.computeShape().getDeterminantRadius()
541 
542  if dataRef.datasetExists(datasetType='calexpBackground'):
543  # Get background for reference CCD
544  # This approximation is good enough for now
545  bgStats = (bg[0].getStatsImage().getImage().array
546  for bg in dataRef.get(datasetType='calexpBackground'))
547  rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)
548  else:
549  self.log.warn('Sky background not found for visit %d / ccd %d' %
550  (visit, dataRef.dataId[self.config.ccdDataRefName]))
551  rec['skyBackground'] = -1.0
552 
553  rec['used'] = 1
554 
555  def _makeSourceMapper(self, sourceSchema):
556  """
557  Make a schema mapper for fgcm sources
558 
559  Parameters
560  ----------
561  sourceSchema: `afwTable.Schema`
562  Default source schema from the butler
563 
564  Returns
565  -------
566  sourceMapper: `afwTable.schemaMapper`
567  Mapper to the FGCM source schema
568  """
569 
570  # create a mapper to the preferred output
571  sourceMapper = afwTable.SchemaMapper(sourceSchema)
572 
573  # map to ra/dec
574  sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
575  sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
576  sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
577  sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
578  # Add the mapping if the field exists in the input catalog.
579  # If the field does not exist, simply add it (set to False).
580  # This field is not required for calibration, but is useful
581  # to collate if available.
582  try:
583  sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),
584  'psf_candidate')
585  except LookupError:
586  sourceMapper.editOutputSchema().addField(
587  "psf_candidate", type='Flag',
588  doc=("Flag set if the source was a candidate for PSF determination, "
589  "as determined by the star selector."))
590 
591  # and add the fields we want
592  sourceMapper.editOutputSchema().addField(
593  "visit", type=np.int32, doc="Visit number")
594  sourceMapper.editOutputSchema().addField(
595  "ccd", type=np.int32, doc="CCD number")
596  sourceMapper.editOutputSchema().addField(
597  "instMag", type=np.float32, doc="Instrumental magnitude")
598  sourceMapper.editOutputSchema().addField(
599  "instMagErr", type=np.float32, doc="Instrumental magnitude error")
600  sourceMapper.editOutputSchema().addField(
601  "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian")
602  sourceMapper.editOutputSchema().addField(
603  "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset")
604 
605  return sourceMapper
606 
607  def fgcmMatchStars(self, butler, visitCat, obsCat):
608  """
609  Use FGCM code to match observations into unique stars.
610 
611  Parameters
612  ----------
613  butler: `lsst.daf.persistence.Butler`
614  visitCat: `afw.table.BaseCatalog`
615  Catalog with visit data for fgcm
616  obsCat: `afw.table.BaseCatalog`
617  Full catalog of star observations for fgcm
618 
619  Returns
620  -------
621  fgcmStarIdCat: `afw.table.BaseCatalog`
622  Catalog of unique star identifiers and index keys
623  fgcmStarIndicesCat: `afwTable.BaseCatalog`
624  Catalog of unique star indices
625  fgcmRefCat: `afw.table.BaseCatalog`
626  Catalog of matched reference stars.
627  Will be None if `config.doReferenceMatches` is False.
628  """
629 
630  if self.config.doReferenceMatches:
631  # Make a subtask for reference loading
632  self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler)
633 
634  # get filter names into a numpy array...
635  # This is the type that is expected by the fgcm code
636  visitFilterNames = np.zeros(len(visitCat), dtype='a10')
637  for i in range(len(visitCat)):
638  visitFilterNames[i] = visitCat[i]['filtername']
639 
640  # match to put filterNames with observations
641  visitIndex = np.searchsorted(visitCat['visit'],
642  obsCat['visit'])
643 
644  obsFilterNames = visitFilterNames[visitIndex]
645 
646  if self.config.doReferenceMatches:
647  # Get the reference filter names, using the LUT
648  lutCat = butler.get('fgcmLookUpTable')
649 
650  stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in
651  zip(lutCat[0]['filterNames'].split(','),
652  lutCat[0]['stdFilterNames'].split(','))}
653  stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in
654  zip(lutCat[0]['stdFilterNames'].split(','),
655  lutCat[0]['lambdaStdFilter'])}
656 
657  del lutCat
658 
659  referenceFilterNames = self._getReferenceFilterNames(visitCat,
660  stdFilterDict,
661  stdLambdaDict)
662  self.log.info("Using the following reference filters: %s" %
663  (', '.join(referenceFilterNames)))
664 
665  else:
666  # This should be an empty list
667  referenceFilterNames = []
668 
669  # make the fgcm starConfig dict
670 
671  starConfig = {'logger': self.log,
672  'filterToBand': self.config.filterMap,
673  'requiredBands': self.config.requiredBands,
674  'minPerBand': self.config.minPerBand,
675  'matchRadius': self.config.matchRadius,
676  'isolationRadius': self.config.isolationRadius,
677  'matchNSide': self.config.matchNside,
678  'coarseNSide': self.config.coarseNside,
679  'densNSide': self.config.densityCutNside,
680  'densMaxPerPixel': self.config.densityCutMaxPerPixel,
681  'primaryBands': self.config.primaryBands,
682  'referenceFilterNames': referenceFilterNames}
683 
684  # initialize the FgcmMakeStars object
685  fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
686 
687  # make the primary stars
688  # note that the ra/dec native Angle format is radians
689  # We determine the conversion from the native units (typically
690  # radians) to degrees for the first observation. This allows us
691  # to treate ra/dec as numpy arrays rather than Angles, which would
692  # be approximately 600x slower.
693  conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
694  fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
695  obsCat['dec'] * conv,
696  filterNameArray=obsFilterNames,
697  bandSelected=False)
698 
699  # and match all the stars
700  fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
701  obsCat['dec'] * conv,
702  obsFilterNames)
703 
704  if self.config.doReferenceMatches:
705  fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
706 
707  # now persist
708 
709  objSchema = self._makeFgcmObjSchema()
710 
711  # make catalog and records
712  fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
713  fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
714  for i in range(fgcmMakeStars.objIndexCat.size):
715  fgcmStarIdCat.addNew()
716 
717  # fill the catalog
718  fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
719  fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
720  fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
721  fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex']
722  fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
723 
724  obsSchema = self._makeFgcmObsSchema()
725 
726  fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
727  fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
728  for i in range(fgcmMakeStars.obsIndexCat.size):
729  fgcmStarIndicesCat.addNew()
730 
731  fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex']
732 
733  if self.config.doReferenceMatches:
734  refSchema = self._makeFgcmRefSchema(len(referenceFilterNames))
735 
736  fgcmRefCat = afwTable.BaseCatalog(refSchema)
737  fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
738 
739  for i in range(fgcmMakeStars.referenceCat.size):
740  fgcmRefCat.addNew()
741 
742  fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
743  fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
744  fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr']
745 
746  md = PropertyList()
747  md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
748  md.set("FILTERNAMES", referenceFilterNames)
749  fgcmRefCat.setMetadata(md)
750 
751  else:
752  fgcmRefCat = None
753 
754  return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
755 
756  def _makeFgcmVisitSchema(self, nCcd):
757  """
758  Make a schema for an fgcmVisitCatalog
759 
760  Parameters
761  ----------
762  nCcd: `int`
763  Number of CCDs in the camera
764 
765  Returns
766  -------
767  schema: `afwTable.Schema`
768  """
769 
770  schema = afwTable.Schema()
771  schema.addField('visit', type=np.int32, doc="Visit number")
772  # Note that the FGCM code currently handles filternames up to 2 characters long
773  schema.addField('filtername', type=str, size=10, doc="Filter name")
774  schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
775  schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
776  schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)")
777  schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
778  schema.addField('mjd', type=np.float64, doc="MJD of visit")
779  schema.addField('exptime', type=np.float32, doc="Exposure time")
780  schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
781  schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)")
782  schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
783  schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)")
784  # the following field is not used yet
785  schema.addField('deepFlag', type=np.int32, doc="Deep observation")
786  schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment",
787  size=nCcd)
788  schema.addField('used', type=np.int32, doc="This visit has been ingested.")
789  schema.addField('sources_read', type='Flag', doc="This visit had sources read.")
790 
791  return schema
792 
793  def _makeFgcmObjSchema(self):
794  """
795  Make a schema for the objIndexCat from fgcmMakeStars
796 
797  Returns
798  -------
799  schema: `afwTable.Schema`
800  """
801 
802  objSchema = afwTable.Schema()
803  objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
804  # Will investigate making these angles...
805  objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
806  objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
807  objSchema.addField('obsArrIndex', type=np.int32,
808  doc='Index in obsIndexTable for first observation')
809  objSchema.addField('nObs', type=np.int32, doc='Total number of observations')
810 
811  return objSchema
812 
813  def _makeFgcmObsSchema(self):
814  """
815  Make a schema for the obsIndexCat from fgcmMakeStars
816 
817  Returns
818  -------
819  schema: `afwTable.Schema`
820  """
821 
822  obsSchema = afwTable.Schema()
823  obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table')
824 
825  return obsSchema
826 
827  def _makeFgcmRefSchema(self, nReferenceBands):
828  """
829  Make a schema for the referenceCat from fgcmMakeStars
830 
831  Parameters
832  ----------
833  nReferenceBands: `int`
834  Number of reference bands
835 
836  Returns
837  -------
838  schema: `afwTable.Schema`
839  """
840 
841  refSchema = afwTable.Schema()
842  refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
843  refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)',
844  size=nReferenceBands)
845  refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array',
846  size=nReferenceBands)
847 
848  return refSchema
849 
850  def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict):
851  """
852  Get the reference filter names, in wavelength order, from the visitCat and
853  information from the look-up-table.
854 
855  Parameters
856  ----------
857  visitCat: `afw.table.BaseCatalog`
858  Catalog with visit data for FGCM
859  stdFilterDict: `dict`
860  Mapping of filterName to stdFilterName from LUT
861  stdLambdaDict: `dict`
862  Mapping of stdFilterName to stdLambda from LUT
863 
864  Returns
865  -------
866  referenceFilterNames: `list`
867  Wavelength-ordered list of reference filter names
868  """
869 
870  # Find the unique list of filter names in visitCat
871  filterNames = np.unique(visitCat.asAstropy()['filtername'])
872 
873  # Find the unique list of "standard" filters
874  stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames}
875 
876  # And sort these by wavelength
877  referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get)
878 
879  return referenceFilterNames
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsRunner
Definition: fgcmBuildStarsBase.py:190
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.setDefaults
def setDefaults(self)
Definition: fgcmBuildStarsBase.py:173
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask
Definition: fgcmBuildStarsBase.py:260
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._makeFgcmVisitSchema
def _makeFgcmVisitSchema(self, nCcd)
Definition: fgcmBuildStarsBase.py:756
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.fgcmMatchStars
def fgcmMatchStars(self, butler, visitCat, obsCat)
Definition: fgcmBuildStarsBase.py:607
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._makeFgcmObsSchema
def _makeFgcmObsSchema(self)
Definition: fgcmBuildStarsBase.py:813
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._makeFgcmRefSchema
def _makeFgcmRefSchema(self, nReferenceBands)
Definition: fgcmBuildStarsBase.py:827
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsRunner.run
def run(self, parsedCmd)
Definition: fgcmBuildStarsBase.py:242
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsRunner.getTargetList
def getTargetList(parsedCmd)
Definition: fgcmBuildStarsBase.py:202
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.fgcmMakeAllStarObservations
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
Definition: fgcmBuildStarsBase.py:388
lsst.fgcmcal.utilities.computeApertureRadiusFromDataRef
def computeApertureRadiusFromDataRef(dataRef, fluxField)
Definition: utilities.py:787
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._getReferenceFilterNames
def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict)
Definition: fgcmBuildStarsBase.py:850
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.__init__
def __init__(self, butler=None, **kwargs)
Definition: fgcmBuildStarsBase.py:268
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.fgcmMakeVisitCatalog
def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, visitCatDataRef=None, inVisitCat=None)
Definition: fgcmBuildStarsBase.py:425
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._fillVisitCatalog
def _fillVisitCatalog(self, visitCat, groupedDataRefs, visitCatDataRef=None)
Definition: fgcmBuildStarsBase.py:473
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.findAndGroupDataRefs
def findAndGroupDataRefs(self, butler, dataRefs)
Definition: fgcmBuildStarsBase.py:370
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask.runDataRef
def runDataRef(self, butler, dataRefs)
Definition: fgcmBuildStarsBase.py:279
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsBaseTask._makeFgcmObjSchema
def _makeFgcmObjSchema(self)
Definition: fgcmBuildStarsBase.py:793
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase.sourceSelector
sourceSelector
Definition: fgcmBuildStarsBase.py:142
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsRunner.__call__
def __call__(self, args)
Definition: fgcmBuildStarsBase.py:210
lsst.fgcmcal.fgcmBuildStarsBase.FgcmBuildStarsConfigBase
Definition: fgcmBuildStarsBase.py:49