lsst.jointcal gd82bb2a289+52e0a19ca3
testUtils.py
Go to the documentation of this file.
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22"""Functions to help create jointcal tests by generating fake data."""
23
24__all__ = ['createFakeCatalog', 'createTwoFakeCcdImages', 'getMeasuredStarsFromCatalog']
25
26import os
27import unittest
28
29import numpy as np
30
31import lsst.afw.geom
32import lsst.afw.table
33import lsst.daf.butler
34import lsst.pipe.base
35
36import lsst.jointcal.star
37
38
40 """Returns True if the necessary packages and files are available.
41
42 We need ``obs_cfht`` to load the test/data/cfht_minimal dataset, which
43 includes the metadata that is used to build the fake catalogs.
44 """
45 try:
46 import lsst.obs.cfht # noqa: F401
47 return True
48 except ImportError:
49 return False
50
51
52def createTwoFakeCcdImages(num1=4, num2=4, seed=100, fakeDetectorId=12,
53 photoCalibMean1=1e-2, photoCalibMean2=1.2e-2,
54 fakeWcses=(None, None),
55 fakeVisitInfos=(None, None)):
56 """Return two fake ccdImages built on CFHT Megacam metadata.
57
58 If ``num1 == num2``, the catalogs will align on-sky so each source will
59 have a match in the other catalog.
60
61 This uses the butler dataset stored in `tests/data/cfht_minimal` to
62 bootstrap the metadata.
63
64 Parameters
65 ----------
66 num1, num2 : `int`, optional
67 Number of sources to put in the first and second catalogs. Should be
68 a square, to have sqrt(num) centroids on a grid.
69 seed : `int`, optional
70 Seed value for np.random.
71 fakeDetectorId : `int`, optional
72 Sensor identifier to use for both CcdImages. The wcs, bbox, photoCalib, etc.
73 will still be drawn from the CFHT ccd=12 files, as that is the only
74 testdata that is included in this simple test dataset.
75 photoCalibMean1, photoCalibMean2: `float`, optional
76 The mean photometric calibration to pass to each ccdImage construction.
77 Note: this value is 1/instFluxMag0, so it should be less than 1.
78 fakeWcses : `list` [`lsst.afw.geom.SkyWcs`], optional
79 The SkyWcses to use instead of the ones read from disk.
80 fakeWcses : `list` [`lsst.afw.image.VisitInfo`], optional
81 The VisitInfos to use instead of the ones read from disk.
82
83 Returns
84 -------
85 struct : `lsst.pipe.base.Struct`
86 Result struct with components:
87
88 - `camera` : Camera representing these catalogs
90 - `catalogs` : Catalogs containing fake sources
91 (`list` of `lsst.afw.table.SourceCatalog`).
92 - `ccdImageList` : CcdImages containing the metadata and fake sources
93 (`list` of `lsst.jointcal.CcdImage`).
94 - `bbox` : Bounding Box of the image (`lsst.geom.Box2I`).
95 - 'fluxFieldName' : name of the instFlux field in the catalogs ('str').
96 """
97 if not canRunTests():
98 msg = "Necessary packages not available to run tests that use the cfht_minimal dataset."
99 raise unittest.SkipTest(msg)
100
101 np.random.seed(seed)
102
103 visit1 = 849375
104 visit2 = 850587
105 fluxFieldName = "SomeFlux"
106
107 # Load or fake the necessary metadata for each CcdImage
108 dataDir = lsst.utils.getPackageDir('jointcal')
109 inputDir = os.path.join(dataDir, 'tests/data/cfht_minimal/repo')
110 # Ensure this butler is not writeable, so that we don't mess up the repo accidentally.
111 butler = lsst.daf.butler.Butler(inputDir, collections=["singleFrame"], writeable=False)
112
113 # so we can access parts of the camera later (e.g. focal plane)
114 camera = butler.get('camera', instrument="MegaPrime")
115
116 struct1 = createFakeCcdImage(butler, visit1, num1, fluxFieldName,
117 photoCalibMean=photoCalibMean1, photoCalibErr=1.0,
118 fakeDetectorId=fakeDetectorId,
119 fakeWcs=fakeWcses[0], fakeVisitInfo=fakeVisitInfos[0])
120 struct2 = createFakeCcdImage(butler, visit2, num2, fluxFieldName,
121 photoCalibMean=photoCalibMean2, photoCalibErr=5.0,
122 fakeDetectorId=fakeDetectorId,
123 fakeWcs=fakeWcses[1], fakeVisitInfo=fakeVisitInfos[1])
124
125 return lsst.pipe.base.Struct(camera=camera,
126 catalogs=[struct1.catalog, struct2.catalog],
127 ccdImageList=[struct1.ccdImage, struct2.ccdImage],
128 bbox=struct1.bbox,
129 skyWcs=[struct1.skyWcs, struct2.skyWcs],
130 fluxFieldName=fluxFieldName)
131
132
133def createFakeCcdImage(butler, visit, num, fluxFieldName,
134 photoCalibMean=1e-2, photoCalibErr=1.0, fakeDetectorId=12,
135 fakeWcs=None, fakeVisitInfo=None):
136 """Create a fake CcdImage by making a fake catalog.
137
138 Parameters
139 ----------
140 butler : `lsst.daf.butler.Butler`
141 Butler to load metadata from.
142 visit : `int`
143 Visit identifier to build a butler dataId.
144 num : `int`
145 Number of sources to put in the catalogs. Should be
146 a square, to have sqrt(num) centroids on a grid.
147 fluxFieldName : `str`
148 Name of the flux field to populate in the catalog, without `_instFlux`
149 (e.g. "slot_CalibFlux").
150 photoCalibMean : `float`, optional
151 Value to set for calibrationMean in the created PhotoCalib.
152 Note: this value is 1/instFluxMag0, so it should be less than 1.
153 photoCalibErr : `float`, optional
154 Value to set for calibrationErr in the created PhotoCalib.
155 fakeDetectorId : `int`, optional
156 Use this as the detectorId in the returned CcdImage.
157 fakeWcs : `lsst.afw.geom.SkyWcs`, optional
158 A SkyWcs to use instead of one read from disk.
159 fakeVisitInfo : `lsst.afw.image.VisitInfo`, optional
160 A VisitInfo to use instead of one read from disk.
161
162 Returns
163 -------
164 struct : `lsst.pipe.base.Struct`
165 Result struct with components:
166
167 - `catalog` : Catalogs containing fake sources
169 - `ccdImage` : CcdImage containing the metadata and fake sources
171 - `bbox` : Bounding Box of the image (`lsst.geom.Box2I`).
172 - `skyWcs` : SkyWcs of the image (`lsst.afw.geom.SkyWcs`).
173 """
174 detectorId = 12 # we only have data for detector=12
175
176 dataId = dict(visit=visit, detector=detectorId, instrument="MegaPrime")
177 skyWcs = fakeWcs if fakeWcs is not None else butler.get('calexp.wcs', dataId=dataId)
178 visitInfo = fakeVisitInfo if fakeVisitInfo is not None else butler.get('calexp.visitInfo', dataId=dataId)
179 bbox = butler.get('calexp.bbox', dataId=dataId)
180 detector = butler.get('calexp.detector', dataId=dataId)
181 filt = butler.get("calexp.filter", dataId=dataId).bandLabel
182 photoCalib = lsst.afw.image.PhotoCalib(photoCalibMean, photoCalibErr)
183
184 catalog = createFakeCatalog(num, bbox, fluxFieldName, skyWcs=skyWcs)
185 ccdImage = lsst.jointcal.ccdImage.CcdImage(catalog, skyWcs, visitInfo, bbox, filt, photoCalib,
186 detector, visit, fakeDetectorId, fluxFieldName)
187
188 return lsst.pipe.base.Struct(catalog=catalog, ccdImage=ccdImage, bbox=bbox, skyWcs=skyWcs)
189
190
191def createFakeCatalog(num, bbox, fluxFieldName, skyWcs=None, refCat=False):
192 """Return a fake minimally-useful catalog for jointcal.
193
194 Parameters
195 ----------
196 num : `int`
197 Number of sources to put in the catalogs. Should be
198 a square, to have sqrt(num) centroids on a grid.
199 bbox : `lsst.geom.Box2I`
200 Bounding Box of the detector to populate.
201 fluxFieldName : `str`
202 Name of the flux field to populate in the catalog, without `_instFlux`
203 (e.g. "slot_CalibFlux").
204 skyWcs : `lsst.afw.geom.SkyWcs` or None, optional
205 If supplied, use this to fill in coordinates from centroids.
206 refCat : `bool`, optional
207 Return a ``SimpleCatalog`` so that it behaves like a reference catalog?
208
209 Returns
210 -------
212 A populated source catalog.
213 """
215 # centroid
216 centroidKey = lsst.afw.table.Point2DKey.addFields(schema, "centroid", "centroid", "pixels")
217 xErrKey = schema.addField("centroid_xErr", type="F")
218 yErrKey = schema.addField("centroid_yErr", type="F")
219 # shape
220 shapeKey = lsst.afw.table.QuadrupoleKey.addFields(schema, "shape", "",
221 lsst.afw.table.CoordinateType.PIXEL)
222 # Put the fake sources in the minimal catalog.
223 schema.addField(fluxFieldName+"_instFlux", type="D", doc="post-ISR instFlux")
224 schema.addField(fluxFieldName+"_instFluxErr", type="D", doc="post-ISR instFlux stddev")
225 schema.addField(fluxFieldName+"_flux", type="D", doc="source flux (nJy)")
226 schema.addField(fluxFieldName+"_fluxErr", type="D", doc="flux stddev (nJy)")
227 schema.addField(fluxFieldName+"_mag", type="D", doc="magnitude")
228 schema.addField(fluxFieldName+"_magErr", type="D", doc="magnitude stddev")
229 return fillCatalog(schema, num, bbox,
230 centroidKey, xErrKey, yErrKey, shapeKey, fluxFieldName,
231 skyWcs=skyWcs, refCat=refCat)
232
233
234def fillCatalog(schema, num, bbox,
235 centroidKey, xErrKey, yErrKey, shapeKey, fluxFieldName,
236 skyWcs=None, fluxErrFraction=0.05, refCat=False):
237 """Return a catalog populated with fake, but reasonable, sources.
238
239 Centroids are placed on a uniform grid, errors are normally distributed.
240
241 Parameters
242 ----------
243 schema : `lsst.afw.table.Schema`
244 Pre-built schema to make the catalog from.
245 num : `int`
246 Number of sources to put in the catalog.
247 bbox : `lsst.geom.Box2I`
248 Bounding box of the ccd to put sources in.
249 centroidKey : `lsst.afw.table.Key`
250 Key for the centroid field to populate.
251 xErrKey : `lsst.afw.table.Key`
252 Key for the xErr field to populate.
253 yErrKey : `lsst.afw.table.Key`
254 Key for the yErr field to populate.
255 shapeKey : `lsst.afw.table.Key`
256 Key for the shape field to populate.
257 fluxFieldName : `str`
258 Name of the flux field to populate in the catalog, without `_instFlux`
259 (e.g. "slot_CalibFlux").
260 skyWcs : `lsst.afw.geom.SkyWcs` or None, optional
261 If supplied, use this to fill in coordinates from centroids.
262 fluxErrFraction : `float`, optional
263 Fraction of instFlux to use for the instFluxErr.
264 refCat : `bool`, optional
265 Return a ``SimpleCatalog`` so that it behaves like a reference catalog?
266
267 Returns
268 -------
270 The filled catalog.
271 """
272 table = lsst.afw.table.SourceTable.make(schema)
273 table.defineCentroid('centroid')
274 table.defineShape('shape')
275 table.defineCalibFlux(fluxFieldName)
276 if refCat:
277 catalog = lsst.afw.table.SimpleCatalog(table)
278 else:
279 catalog = lsst.afw.table.SourceCatalog(table)
280
281 instFlux = np.random.random(num)*10000
282 instFluxErr = np.abs(instFlux * np.random.normal(fluxErrFraction, scale=0.1, size=num))
283 xx = np.linspace(bbox.getMinX(), bbox.getMaxX(), int(np.sqrt(num)))
284 yy = np.linspace(bbox.getMinY(), bbox.getMaxY(), int(np.sqrt(num)))
285 xv, yv = np.meshgrid(xx, yy)
286 vx = np.random.normal(scale=0.1, size=num)
287 vy = np.random.normal(scale=0.1, size=num)
288
289 # make all the sources perfectly spherical, for simplicity.
290 mxx = 1
291 myy = 1
292 mxy = 0
293
294 for i, (x, y) in enumerate(zip(xv.ravel(), yv.ravel())):
295 record = catalog.addNew()
296 record.set('id', i)
297 record.set(centroidKey, lsst.geom.Point2D(x, y))
298 record.set(shapeKey, lsst.afw.geom.ellipses.Quadrupole(mxx, myy, mxy))
299
300 if skyWcs is not None:
301 lsst.afw.table.updateSourceCoords(skyWcs, catalog)
302
303 catalog[xErrKey] = vx
304 catalog[yErrKey] = vy
305 catalog[fluxFieldName + '_instFlux'] = instFlux
306 catalog[fluxFieldName + '_instFluxErr'] = instFluxErr
307
308 return catalog
309
310
311def getMeasuredStarsFromCatalog(catalog, pixToFocal):
312 """Return a list of measuredStars built from a catalog.
313
314 Parameters
315 ----------
317 The table to get sources from.
319 Transform that goes from pixel to focal plane coordinates, to set the
320 MeasuredStar x/y focal points.
321
322 Returns
323 -------
324 stars : `list` of `lsst.jointcal.MeasuredStar`
325 MeasuredStars built from the catalog sources.
326 """
327 stars = []
328 for record in catalog:
329 star = lsst.jointcal.star.MeasuredStar()
330 star.x = record.getX()
331 star.y = record.getY()
332 star.setInstFluxAndErr(record.getCalibInstFlux(), record.getCalibInstFluxErr())
333 # TODO: cleanup after DM-4044
334 point = lsst.geom.Point2D(star.x, star.y)
335 pointFocal = pixToFocal.applyForward(point)
336 star.setXFocal(pointFocal.getX())
337 star.setYFocal(pointFocal.getY())
338 stars.append(star)
339
340 return stars
static QuadrupoleKey addFields(Schema &schema, std::string const &name, std::string const &doc, CoordinateType coordType=CoordinateType::PIXEL)
static std::shared_ptr< SourceTable > make(Schema const &schema, std::shared_ptr< IdFactory > const &idFactory)
static Schema makeMinimalSchema()
Handler of an actual image from a single CCD.
Definition: CcdImage.h:64
Sources measured on images.
Definition: MeasuredStar.h:51
void updateSourceCoords(geom::SkyWcs const &wcs, SourceCollection &sourceList)
def fillCatalog(schema, num, bbox, centroidKey, xErrKey, yErrKey, shapeKey, fluxFieldName, skyWcs=None, fluxErrFraction=0.05, refCat=False)
Definition: testUtils.py:236
def createFakeCcdImage(butler, visit, num, fluxFieldName, photoCalibMean=1e-2, photoCalibErr=1.0, fakeDetectorId=12, fakeWcs=None, fakeVisitInfo=None)
Definition: testUtils.py:135
def createTwoFakeCcdImages(num1=4, num2=4, seed=100, fakeDetectorId=12, photoCalibMean1=1e-2, photoCalibMean2=1.2e-2, fakeWcses=(None, None), fakeVisitInfos=(None, None))
Definition: testUtils.py:55
def getMeasuredStarsFromCatalog(catalog, pixToFocal)
Definition: testUtils.py:311
def createFakeCatalog(num, bbox, fluxFieldName, skyWcs=None, refCat=False)
Definition: testUtils.py:191