Coverage for tests/fgcmcalTestBase.py : 8%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""General fgcmcal testing class.
25This class is used as the basis for individual obs package tests using
26data from testdata_jointcal.
27"""
29import os
30import shutil
31import numpy as np
32import numpy.testing as testing
33import glob
34import esutil
36import lsst.daf.persistence as dafPersist
37import lsst.geom as geom
38import lsst.log
39from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, LoadIndexedReferenceObjectsConfig
40from astropy import units
42import lsst.fgcmcal as fgcmcal
45class FgcmcalTestBase(object):
46 """
47 Base class for fgcmcal tests, to genericize some test running and setup.
49 Derive from this first, then from TestCase.
50 """
52 def setUp_base(self, inputDir=None, testDir=None, logLevel=None, otherArgs=[]):
53 """
54 Call from your child class's setUp() to get variables built.
56 Parameters
57 ----------
58 inputDir: `str`, optional
59 Input directory
60 testDir: `str`, optional
61 Test directory
62 logLevel: `str`, optional
63 Override loglevel for command-line tasks
64 otherArgs: `list`, default=[]
65 List of additional arguments to send to command-line tasks
66 """
68 self.inputDir = inputDir
69 self.testDir = testDir
70 self.logLevel = logLevel
71 self.otherArgs = otherArgs
73 self.config = None
74 self.configfiles = []
76 lsst.log.setLevel("daf.persistence.butler", lsst.log.FATAL)
77 lsst.log.setLevel("CameraMapper", lsst.log.FATAL)
79 if self.logLevel is not None:
80 self.otherArgs.extend(['--loglevel', 'fgcmcal=%s'%self.logLevel])
82 def _testFgcmMakeLut(self, nBand, i0Std, i0Recon, i10Std, i10Recon):
83 """
84 Test running of FgcmMakeLutTask
86 Parameters
87 ----------
88 nBand: `int`
89 Number of bands tested
90 i0Std: `np.array', size nBand
91 Values of i0Std to compare to
92 i10Std: `np.array`, size nBand
93 Values of i10Std to compare to
94 i0Recon: `np.array`, size nBand
95 Values of reconstructed i0 to compare to
96 i10Recon: `np.array`, size nBand
97 Values of reconsntructed i10 to compare to
98 """
100 args = [self.inputDir, '--output', self.testDir,
101 '--doraise']
102 if len(self.configfiles) > 0:
103 args.extend(['--configfile', *self.configfiles])
104 args.extend(self.otherArgs)
106 result = fgcmcal.FgcmMakeLutTask.parseAndRun(args=args, config=self.config)
107 self._checkResult(result)
109 butler = dafPersist.butler.Butler(self.testDir)
110 tempTask = fgcmcal.FgcmFitCycleTask()
111 lutCat = butler.get('fgcmLookUpTable')
112 fgcmLut, lutIndexVals, lutStd = fgcmcal.utilities.translateFgcmLut(lutCat,
113 dict(tempTask.config.filterMap))
115 # Check that we got the requested number of bands...
116 self.assertEqual(nBand, len(lutIndexVals[0]['FILTERNAMES']))
118 self.assertFloatsAlmostEqual(i0Std, lutStd[0]['I0STD'], msg='I0Std', rtol=1e-5)
119 self.assertFloatsAlmostEqual(i10Std, lutStd[0]['I10STD'], msg='I10Std', rtol=1e-5)
121 indices = fgcmLut.getIndices(np.arange(nBand, dtype=np.int32),
122 np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']),
123 np.zeros(nBand) + lutStd[0]['O3STD'],
124 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']),
125 np.zeros(nBand) + lutStd[0]['ALPHASTD'],
126 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])),
127 np.zeros(nBand, dtype=np.int32),
128 np.zeros(nBand) + lutStd[0]['PMBSTD'])
129 i0 = fgcmLut.computeI0(np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']),
130 np.zeros(nBand) + lutStd[0]['O3STD'],
131 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']),
132 np.zeros(nBand) + lutStd[0]['ALPHASTD'],
133 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])),
134 np.zeros(nBand) + lutStd[0]['PMBSTD'],
135 indices)
137 self.assertFloatsAlmostEqual(i0Recon, i0, msg='i0Recon', rtol=1e-5)
139 i1 = fgcmLut.computeI1(np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']),
140 np.zeros(nBand) + lutStd[0]['O3STD'],
141 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']),
142 np.zeros(nBand) + lutStd[0]['ALPHASTD'],
143 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])),
144 np.zeros(nBand) + lutStd[0]['PMBSTD'],
145 indices)
147 self.assertFloatsAlmostEqual(i10Recon, i1/i0, msg='i10Recon', rtol=1e-5)
149 def _testFgcmBuildStarsTable(self, visits, nStar, nObs):
150 """
151 Test running of FgcmBuildStarsTableTask
153 Parameters
154 ----------
155 visits: `list`
156 List of visits to calibrate
157 nStar: `int`
158 Number of stars expected
159 nObs: `int`
160 Number of observations of stars expected
161 """
163 args = [self.inputDir, '--output', self.testDir,
164 '--id', 'visit='+'^'.join([str(visit) for visit in visits]),
165 '--doraise']
166 if len(self.configfiles) > 0:
167 args.extend(['--configfile', *self.configfiles])
168 args.extend(self.otherArgs)
170 result = fgcmcal.FgcmBuildStarsTableTask.parseAndRun(args=args, config=self.config)
171 self._checkResult(result)
173 butler = dafPersist.butler.Butler(self.testDir)
175 visitCat = butler.get('fgcmVisitCatalog')
176 self.assertEqual(len(visits), len(visitCat))
178 starIds = butler.get('fgcmStarIds')
179 self.assertEqual(nStar, len(starIds))
181 starObs = butler.get('fgcmStarObservations')
182 self.assertEqual(nObs, len(starObs))
184 def _testFgcmBuildStarsAndCompare(self, visits):
185 """
186 Test running of FgcmBuildStarsTask and compare to Table run
188 Parameters
189 ----------
190 visits: `list`
191 List of visits to calibrate
192 """
193 args = [self.testDir, '--output', os.path.join(self.testDir, 'rerun', 'src'),
194 '--id', 'visit='+'^'.join([str(visit) for visit in visits]),
195 '--doraise']
196 if len(self.configfiles) > 0:
197 args.extend(['--configfile', *self.configfiles])
198 args.extend(self.otherArgs)
200 result = fgcmcal.FgcmBuildStarsTask.parseAndRun(args=args, config=self.config)
201 self._checkResult(result)
203 butlerSrc = dafPersist.Butler(os.path.join(self.testDir, 'rerun', 'src'))
204 butlerTable = dafPersist.Butler(os.path.join(self.testDir))
206 # We compare the two catalogs to ensure they contain the same data. They will
207 # not be identical in ordering because the input data was ingested in a different
208 # order (hence the stars are rearranged).
209 self._compareBuildStars(butlerSrc, butlerTable)
211 def _testFgcmFitCycle(self, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, skipChecks=False):
212 """
213 Test running of FgcmFitCycleTask
215 Parameters
216 ----------
217 nZp: `int`
218 Number of zeropoints created by the task
219 nGoodZp: `int`
220 Number of good (photometric) zeropoints created
221 nOkZp: `int`
222 Number of constrained zeropoints (photometric or not)
223 nBadZp: `int`
224 Number of unconstrained (bad) zeropoints
225 nStdStars: `int`
226 Number of standard stars produced
227 nPlots: `int`
228 Number of plots produced
229 skipChecks: `bool`, optional
230 Skip number checks, when running less-than-final cycle.
231 Default is False.
232 """
234 args = [self.inputDir, '--output', self.testDir,
235 '--doraise']
236 if len(self.configfiles) > 0:
237 args.extend(['--configfile', *self.configfiles])
238 args.extend(self.otherArgs)
240 # Move into the test directory so the plots will get cleaned in tearDown
241 # In the future, with Gen3, we will probably have a better way of managing
242 # non-data output such as plots.
243 cwd = os.getcwd()
244 os.chdir(self.testDir)
246 result = fgcmcal.FgcmFitCycleTask.parseAndRun(args=args, config=self.config)
247 self._checkResult(result)
249 # Move back to the previous directory
250 os.chdir(cwd)
252 if skipChecks:
253 return
255 # Check that the expected number of plots are there.
256 plots = glob.glob(os.path.join(self.testDir, self.config.outfileBase +
257 '_cycle%02d_plots/' % (self.config.cycleNumber) +
258 '*.png'))
259 self.assertEqual(len(plots), nPlots)
261 butler = dafPersist.butler.Butler(self.testDir)
263 zps = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
265 # Check the numbers of zeropoints in all, good, okay, and bad
266 self.assertEqual(len(zps), nZp)
268 gd, = np.where(zps['fgcmFlag'] == 1)
269 self.assertEqual(len(gd), nGoodZp)
271 ok, = np.where(zps['fgcmFlag'] < 16)
272 self.assertEqual(len(ok), nOkZp)
274 bd, = np.where(zps['fgcmFlag'] >= 16)
275 self.assertEqual(len(bd), nBadZp)
277 # Check that there are no illegal values with the ok zeropoints
278 test, = np.where(zps['fgcmZpt'][gd] < -9000.0)
279 self.assertEqual(len(test), 0)
281 stds = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
283 self.assertEqual(len(stds), nStdStars)
285 def _testFgcmOutputProducts(self, visitDataRefName, ccdDataRefName, filterMapping,
286 zpOffsets, testVisit, testCcd, testFilter, testBandIndex):
287 """
288 Test running of FgcmOutputProductsTask
290 Parameters
291 ----------
292 visitDataRefName: `str`
293 Name of column in dataRef to get the visit
294 ccdDataRefName: `str`
295 Name of column in dataRef to get the ccd
296 filterMapping: `dict`
297 Mapping of filterName to dataRef filter names
298 zpOffsets: `np.array`
299 Zeropoint offsets expected
300 testVisit: `int`
301 Visit id to check for round-trip computations
302 testCcd: `int`
303 Ccd id to check for round-trip computations
304 testFilter: `str`
305 Filtername for testVisit/testCcd
306 testBandIndex: `int`
307 Band index for testVisit/testCcd
308 """
310 args = [self.inputDir, '--output', self.testDir,
311 '--doraise']
312 if len(self.configfiles) > 0:
313 args.extend(['--configfile', *self.configfiles])
314 args.extend(self.otherArgs)
316 result = fgcmcal.FgcmOutputProductsTask.parseAndRun(args=args, config=self.config,
317 doReturnResults=True)
318 self._checkResult(result)
320 # Extract the offsets from the results
321 offsets = result.resultList[0].results.offsets
323 self.assertFloatsAlmostEqual(offsets, zpOffsets, atol=1e-6)
325 butler = dafPersist.butler.Butler(self.testDir)
327 # Test the reference catalog stars
329 # Read in the raw stars...
330 rawStars = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
332 # Read in the new reference catalog...
333 config = LoadIndexedReferenceObjectsConfig()
334 config.ref_dataset_name = 'fgcm_stars'
335 task = LoadIndexedReferenceObjectsTask(butler, config=config)
337 # Read in a giant radius to get them all
338 refStruct = task.loadSkyCircle(rawStars[0].getCoord(), 5.0*geom.degrees,
339 filterName='r')
341 # Make sure all the stars are there
342 self.assertEqual(len(rawStars), len(refStruct.refCat))
344 # And make sure the numbers are consistent
345 test, = np.where(rawStars['id'][0] == refStruct.refCat['id'])
347 # Perform math on numpy arrays to maintain datatypes
348 mags = rawStars['mag_std_noabs'][:, 0].astype(np.float64) + offsets[0]
349 fluxes = (mags*units.ABmag).to_value(units.nJy)
350 fluxErrs = (np.log(10.)/2.5)*fluxes*rawStars['magErr_std'][:, 0].astype(np.float64)
351 # Only check the first one
352 self.assertFloatsAlmostEqual(fluxes[0], refStruct.refCat['r_flux'][test[0]])
353 self.assertFloatsAlmostEqual(fluxErrs[0], refStruct.refCat['r_fluxErr'][test[0]])
355 # Test the psf candidate counting, ratio should be between 0.0 and 1.0
356 candRatio = (refStruct.refCat['r_nPsfCandidate'].astype(np.float64) /
357 refStruct.refCat['r_nTotal'].astype(np.float64))
358 self.assertFloatsAlmostEqual(candRatio.min(), 0.0)
359 self.assertFloatsAlmostEqual(candRatio.max(), 1.0)
361 # Test the fgcm_photoCalib output
363 zptCat = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
364 selected = (zptCat['fgcmFlag'] < 16)
366 # Read in all the calibrations, these should all be there
367 # This test is simply to ensure that all the photoCalib files exist
368 for rec in zptCat[selected]:
369 testCal = butler.get('fgcm_photoCalib',
370 dataId={visitDataRefName: int(rec['visit']),
371 ccdDataRefName: int(rec['ccd']),
372 'filter': filterMapping[rec['filtername']]})
373 self.assertIsNotNone(testCal)
375 # We do round-trip value checking on just the final one (chosen arbitrarily)
376 testCal = butler.get('fgcm_photoCalib',
377 dataId={visitDataRefName: int(testVisit),
378 ccdDataRefName: int(testCcd),
379 'filter': filterMapping[testFilter]})
380 self.assertIsNotNone(testCal)
382 src = butler.get('src', dataId={visitDataRefName: int(testVisit),
383 ccdDataRefName: int(testCcd)})
385 # Only test sources with positive flux
386 gdSrc = (src['slot_CalibFlux_instFlux'] > 0.0)
388 # We need to apply the calibration offset to the fgcmzpt (which is internal
389 # and doesn't know about that yet)
390 testZpInd, = np.where((zptCat['visit'] == testVisit) &
391 (zptCat['ccd'] == testCcd))
392 fgcmZpt = zptCat['fgcmZpt'][testZpInd] + offsets[testBandIndex]
393 fgcmZptGrayErr = np.sqrt(zptCat['fgcmZptVar'][testZpInd])
395 if self.config.doComposeWcsJacobian:
396 # The raw zeropoint needs to be modified to know about the wcs jacobian
397 camera = butler.get('camera')
398 approxPixelAreaFields = fgcmcal.utilities.computeApproxPixelAreaFields(camera)
399 center = approxPixelAreaFields[testCcd].getBBox().getCenter()
400 pixAreaCorr = approxPixelAreaFields[testCcd].evaluate(center)
401 fgcmZpt += -2.5*np.log10(pixAreaCorr)
403 # This is the magnitude through the mean calibration
404 photoCalMeanCalMags = np.zeros(gdSrc.sum())
405 # This is the magnitude through the full focal-plane variable mags
406 photoCalMags = np.zeros_like(photoCalMeanCalMags)
407 # This is the magnitude with the FGCM (central-ccd) zeropoint
408 zptMeanCalMags = np.zeros_like(photoCalMeanCalMags)
410 for i, rec in enumerate(src[gdSrc]):
411 photoCalMeanCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux'])
412 photoCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux'],
413 rec.getCentroid())
414 zptMeanCalMags[i] = fgcmZpt - 2.5*np.log10(rec['slot_CalibFlux_instFlux'])
416 # These should be very close but some tiny differences because the fgcm value
417 # is defined at the center of the bbox, and the photoCal is the mean over the box
418 self.assertFloatsAlmostEqual(photoCalMeanCalMags,
419 zptMeanCalMags, rtol=1e-6)
420 # These should be roughly equal, but not precisely because of the focal-plane
421 # variation. However, this is a useful sanity check for something going totally
422 # wrong.
423 self.assertFloatsAlmostEqual(photoCalMeanCalMags,
424 photoCalMags, rtol=1e-2)
426 # And the photoCal error is just the zeropoint gray error
427 self.assertFloatsAlmostEqual(testCal.getCalibrationErr(),
428 (np.log(10.0)/2.5)*testCal.getCalibrationMean()*fgcmZptGrayErr)
430 # Test the transmission output
432 visitCatalog = butler.get('fgcmVisitCatalog')
433 lutCat = butler.get('fgcmLookUpTable')
435 testTrans = butler.get('transmission_atmosphere_fgcm',
436 dataId={visitDataRefName: visitCatalog[0]['visit']})
437 testResp = testTrans.sampleAt(position=geom.Point2D(0, 0),
438 wavelengths=lutCat[0]['atmLambda'])
440 # The test fit is performed with the atmosphere parameters frozen
441 # (freezeStdAtmosphere = True). Thus the only difference between
442 # these output atmospheres and the standard is the different
443 # airmass. Furthermore, this is a very rough comparison because
444 # the look-up table is computed with very coarse sampling for faster
445 # testing.
447 # To account for overall throughput changes, we scale by the median ratio,
448 # we only care about the shape
449 ratio = np.median(testResp/lutCat[0]['atmStdTrans'])
450 self.assertFloatsAlmostEqual(testResp/ratio, lutCat[0]['atmStdTrans'], atol=0.04)
452 # The second should be close to the first, but there is the airmass
453 # difference so they aren't identical.
454 testTrans2 = butler.get('transmission_atmosphere_fgcm',
455 dataId={visitDataRefName: visitCatalog[1]['visit']})
456 testResp2 = testTrans2.sampleAt(position=geom.Point2D(0, 0),
457 wavelengths=lutCat[0]['atmLambda'])
459 # As above, we scale by the ratio to compare the shape of the curve.
460 ratio = np.median(testResp/testResp2)
461 self.assertFloatsAlmostEqual(testResp/ratio, testResp2, atol=0.04)
463 def _testFgcmCalibrateTract(self, visits, tract,
464 rawRepeatability, filterNCalibMap):
465 """
466 Test running of FgcmCalibrateTractTask
468 Parameters
469 ----------
470 visits: `list`
471 List of visits to calibrate
472 tract: `int`
473 Tract number
474 rawRepeatability: `np.array`
475 Expected raw repeatability after convergence.
476 Length should be number of bands.
477 filterNCalibMap: `dict`
478 Mapping from filter name to number of photoCalibs created.
479 """
481 args = [self.inputDir, '--output', self.testDir,
482 '--id', 'visit='+'^'.join([str(visit) for visit in visits]),
483 'tract=%d' % (tract),
484 '--doraise']
485 if len(self.configfiles) > 0:
486 args.extend(['--configfile', *self.configfiles])
487 args.extend(self.otherArgs)
489 # Move into the test directory so the plots will get cleaned in tearDown
490 # In the future, with Gen3, we will probably have a better way of managing
491 # non-data output such as plots.
492 cwd = os.getcwd()
493 os.chdir(self.testDir)
495 result = fgcmcal.FgcmCalibrateTractTableTask.parseAndRun(args=args, config=self.config,
496 doReturnResults=True)
497 self._checkResult(result)
499 # Move back to the previous directory
500 os.chdir(cwd)
502 # Check that the converged repeatability is what we expect
503 repeatability = result.resultList[0].results.repeatability
504 self.assertFloatsAlmostEqual(repeatability, rawRepeatability, atol=4e-6)
506 butler = dafPersist.butler.Butler(self.testDir)
508 # Check that the number of photoCalib objects in each filter are what we expect
509 for filterName in filterNCalibMap.keys():
510 subset = butler.subset('fgcm_tract_photoCalib', tract=tract, filter=filterName)
511 tot = 0
512 for dataRef in subset:
513 if butler.datasetExists('fgcm_tract_photoCalib', dataId=dataRef.dataId):
514 tot += 1
515 self.assertEqual(tot, filterNCalibMap[filterName])
517 # Check that every visit got a transmission
518 visits = butler.queryMetadata('fgcm_tract_photoCalib', ('visit'), tract=tract)
519 for visit in visits:
520 self.assertTrue(butler.datasetExists('transmission_atmosphere_fgcm_tract',
521 tract=tract, visit=visit))
523 # Check that we got the reference catalog output.
524 # This will raise an exception if the catalog is not there.
525 config = LoadIndexedReferenceObjectsConfig()
526 config.ref_dataset_name = 'fgcm_stars_%d' % (tract)
527 task = LoadIndexedReferenceObjectsTask(butler, config=config)
529 coord = geom.SpherePoint(337.656174*geom.degrees, 0.823595*geom.degrees)
531 refStruct = task.loadSkyCircle(coord, 5.0*geom.degrees, filterName='r')
533 # Test the psf candidate counting, ratio should be between 0.0 and 1.0
534 candRatio = (refStruct.refCat['r_nPsfCandidate'].astype(np.float64) /
535 refStruct.refCat['r_nTotal'].astype(np.float64))
536 self.assertFloatsAlmostEqual(candRatio.min(), 0.0)
537 self.assertFloatsAlmostEqual(candRatio.max(), 1.0)
539 # Test that temporary files aren't stored
540 self.assertFalse(butler.datasetExists('fgcmVisitCatalog'))
541 self.assertFalse(butler.datasetExists('fgcmStarObservations'))
542 self.assertFalse(butler.datasetExists('fgcmStarIndices'))
543 self.assertFalse(butler.datasetExists('fgcmReferenceStars'))
545 def _compareBuildStars(self, butler1, butler2):
546 """
547 Compare the full set of BuildStars outputs with files from two
548 repos.
550 Parameters
551 ----------
552 butler1, butler2 : `lsst.daf.persistence.Butler`
553 """
554 # Check the visit catalogs are identical
555 visitCat1 = butler1.get('fgcmVisitCatalog').asAstropy()
556 visitCat2 = butler2.get('fgcmVisitCatalog').asAstropy()
558 for col in visitCat1.columns:
559 if isinstance(visitCat1[col][0], str):
560 testing.assert_array_equal(visitCat1[col], visitCat2[col])
561 else:
562 testing.assert_array_almost_equal(visitCat1[col], visitCat2[col])
564 # Check that the observation catalogs have the same length
565 # Detailed comparisons of the contents are below.
566 starObs1 = butler1.get('fgcmStarObservations')
567 starObs2 = butler2.get('fgcmStarObservations')
568 self.assertEqual(len(starObs1), len(starObs2))
570 # Check that the number of stars is the same and all match.
571 starIds1 = butler1.get('fgcmStarIds')
572 starIds2 = butler2.get('fgcmStarIds')
573 self.assertEqual(len(starIds1), len(starIds2))
574 matcher = esutil.htm.Matcher(11, starIds1['ra'], starIds1['dec'])
575 matches = matcher.match(starIds2['ra'], starIds2['dec'], 1./3600., maxmatch=1)
576 self.assertEqual(len(matches[0]), len(starIds1))
578 # Check that the number of observations of each star is the same.
579 testing.assert_array_equal(starIds1['nObs'][matches[1]],
580 starIds2['nObs'][matches[0]])
582 # And to test the contents, we need to unravel the observations and make
583 # sure that they are matched individually, because the two catalogs
584 # are constructed in a different order.
585 starIndices1 = butler1.get('fgcmStarIndices')
586 starIndices2 = butler2.get('fgcmStarIndices')
588 test1 = np.zeros(len(starIndices1), dtype=[('ra', 'f8'),
589 ('dec', 'f8'),
590 ('x', 'f8'),
591 ('y', 'f8'),
592 ('psf_candidate', 'b1'),
593 ('visit', 'i4'),
594 ('ccd', 'i4'),
595 ('instMag', 'f4'),
596 ('instMagErr', 'f4'),
597 ('jacobian', 'f4')])
598 test2 = np.zeros_like(test1)
600 # Fill the test1 numpy recarray with sorted and unpacked data from starObs1.
601 # Note that each star has a different number of observations, leading to
602 # a "ragged" array that is packed in here.
603 counter = 0
604 obsIndex = starIndices1['obsIndex']
605 for i in range(len(starIds1)):
606 ind = starIds1['obsArrIndex'][matches[1][i]]
607 nObs = starIds1['nObs'][matches[1][i]]
608 for name in test1.dtype.names:
609 test1[name][counter: counter + nObs] = starObs1[name][obsIndex][ind: ind + nObs]
610 counter += nObs
612 # Fill the test2 numpy recarray with sorted and unpacked data from starObs2.
613 # Note that we have to match these observations per star by matching "visit"
614 # (implicitly assuming each star is observed only once per visit) to ensure
615 # that the observations in test2 are in the same order as test1.
616 counter = 0
617 obsIndex = starIndices2['obsIndex']
618 for i in range(len(starIds2)):
619 ind = starIds2['obsArrIndex'][matches[0][i]]
620 nObs = starIds2['nObs'][matches[0][i]]
621 a, b = esutil.numpy_util.match(test1['visit'][counter: counter + nObs],
622 starObs2['visit'][obsIndex][ind: ind + nObs])
623 for name in test2.dtype.names:
624 test2[name][counter: counter + nObs][a] = starObs2[name][obsIndex][ind: ind + nObs][b]
625 counter += nObs
627 for name in test1.dtype.names:
628 testing.assert_array_almost_equal(test1[name], test2[name])
630 def _checkResult(self, result):
631 """
632 Check the result output from the task
634 Parameters
635 ----------
636 result: `pipeBase.struct`
637 Result structure output from a task
638 """
640 self.assertNotEqual(result.resultList, [], 'resultList should not be empty')
641 self.assertEqual(result.resultList[0].exitStatus, 0)
643 def tearDown(self):
644 """
645 Tear down and clear directories
646 """
648 if getattr(self, 'config', None) is not None:
649 del self.config
650 if os.path.exists(self.testDir):
651 shutil.rmtree(self.testDir, True)