Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

# 

# This file is part of ap_pipe. 

# 

# Developed for the LSST Data Management System. 

# This product includes software developed by the LSST Project 

# (http://www.lsst.org). 

# See the COPYRIGHT file at the top-level directory of this distribution 

# for details of code ownership. 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program. If not, see <http://www.gnu.org/licenses/>. 

# 

 

__all__ = ["ApPipeConfig", "ApPipeTask"] 

 

import os 

import warnings 

 

from sqlalchemy.exc import OperationalError, ProgrammingError 

 

import lsst.dax.ppdb as daxPpdb 

import lsst.pex.config as pexConfig 

import lsst.pipe.base as pipeBase 

 

from lsst.pipe.tasks.processCcd import ProcessCcdTask 

from lsst.pipe.tasks.imageDifference import ImageDifferenceTask 

from lsst.ap.association import ( 

AssociationTask, 

DiaForcedSourceTask, 

MapDiaSourceTask, 

make_dia_object_schema, 

make_dia_source_schema) 

from lsst.ap.pipe.apPipeParser import ApPipeParser 

from lsst.ap.pipe.apPipeTaskRunner import ApPipeTaskRunner 

from lsst.utils import getPackageDir 

 

 

class ApPipeConfig(pexConfig.Config): 

"""Settings and defaults for ApPipeTask. 

""" 

 

ccdProcessor = pexConfig.ConfigurableField( 

target=ProcessCcdTask, 

doc="Task used to perform basic image reduction and characterization.", 

) 

differencer = pexConfig.ConfigurableField( 

target=ImageDifferenceTask, 

doc="Task used to do image subtraction and DiaSource detection.", 

) 

ppdb = pexConfig.ConfigurableField( 

target=daxPpdb.Ppdb, 

ConfigClass=daxPpdb.PpdbConfig, 

doc="Database connection for storing associated DiaSources and " 

"DiaObjects. Must already be initialized.", 

) 

diaSourceDpddifier = pexConfig.ConfigurableField( 

target=MapDiaSourceTask, 

doc="Task for assigning columns from the raw output of ip_diffim into " 

"a schema that more closely resembles the DPDD.", 

) 

associator = pexConfig.ConfigurableField( 

target=AssociationTask, 

doc="Task used to associate DiaSources with DiaObjects.", 

) 

diaForcedSource = pexConfig.ConfigurableField( 

target=DiaForcedSourceTask, 

doc="Task used for force photometer DiaObject locations in direct and " 

"difference images.", 

) 

 

def setDefaults(self): 

"""Settings appropriate for most or all ap_pipe runs. 

""" 

# Always prefer decorrelation; may eventually become ImageDifferenceTask default 

self.differencer.doDecorrelation = True 

self.differencer.detection.thresholdValue = 5.0 # needed with doDecorrelation 

 

# Don't have source catalogs for templates 

self.differencer.doSelectSources = False 

 

# make sure the db schema and config is set up for ap_association. 

self.ppdb.dia_object_index = "baseline" 

self.ppdb.dia_object_columns = [] 

self.ppdb.extra_schema_file = os.path.join( 

getPackageDir("ap_association"), 

"data", 

"ppdb-ap-pipe-schema-extra.yaml") 

 

def validate(self): 

pexConfig.Config.validate(self) 

if not self.differencer.doMeasurement: 

raise ValueError("Source association needs diaSource fluxes [differencer.doMeasurement].") 

if not self.differencer.doWriteSources: 

raise ValueError("Source association needs diaSource catalogs [differencer.doWriteSources].") 

if not self.differencer.doWriteSubtractedExp: 

raise ValueError("Source association needs difference exposures " 

"[differencer.doWriteSubtractedExp].") 

 

 

class ApPipeTask(pipeBase.CmdLineTask): 

"""Command-line task representing the entire AP pipeline. 

 

``ApPipeTask`` processes raw DECam images from basic processing through 

source association. Other observatories will be supported in the future. 

 

``ApPipeTask`` can be run from the command line, but it can also be called 

from other pipeline code. It provides public methods for executing each 

major step of the pipeline by itself. 

 

Parameters 

---------- 

butler : `lsst.daf.persistence.Butler` 

A Butler providing access to the science, calibration, and (unless 

``config.differencer.getTemplate`` is overridden) template data to 

be processed. Its output repository must be both readable 

and writable. 

""" 

 

ConfigClass = ApPipeConfig 

RunnerClass = ApPipeTaskRunner 

_DefaultName = "apPipe" 

 

def __init__(self, butler, *args, **kwargs): 

pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

 

self.makeSubtask("ccdProcessor", butler=butler) 

self.makeSubtask("differencer", butler=butler) 

self.ppdb = self.config.ppdb.apply( 

afw_schemas=dict(DiaObject=make_dia_object_schema(), 

DiaSource=make_dia_source_schema())) 

self.makeSubtask("diaSourceDpddifier", 

inputSchema=self.differencer.schema) 

self.makeSubtask("associator") 

self.makeSubtask("diaForcedSource") 

 

@pipeBase.timeMethod 

def runDataRef(self, rawRef, templateIds=None, reuse=None): 

"""Execute the ap_pipe pipeline on a single image. 

 

Parameters 

---------- 

rawRef : `lsst.daf.persistence.ButlerDataRef` 

A reference to the raw data to process. 

templateIds : `list` of `dict`, optional 

A list of parsed data IDs for templates to use. Only used if 

``config.differencer`` is configured to do so. ``differencer`` or 

its subtasks may restrict the allowed IDs. 

reuse : `list` of `str`, optional 

The names of all subtasks that may be skipped if their output is 

present. Defaults to skipping nothing. 

 

Returns 

------- 

result : `lsst.pipe.base.Struct` 

Result struct with components: 

 

- l1Database : handle for accessing the final association database, conforming to 

`ap_association`'s DB access API 

- ccdProcessor : output of `config.ccdProcessor.runDataRef` (`lsst.pipe.base.Struct` or `None`). 

- differencer : output of `config.differencer.runDataRef` (`lsst.pipe.base.Struct` or `None`). 

- associator : output of `config.associator.run` (`lsst.pipe.base.Struct` or `None`). 

""" 

if reuse is None: 

reuse = [] 

# Work around mismatched HDU lists for raw and processed data 

calexpId = rawRef.dataId.copy() 

if 'hdu' in calexpId: 

del calexpId['hdu'] 

calexpRef = rawRef.getButler().dataRef("calexp", dataId=calexpId) 

 

# Ensure that templateIds make it through basic data reduction 

# TODO: treat as independent jobs (may need SuperTask framework?) 

if templateIds is not None: 

for templateId in templateIds: 

# templateId is typically visit-only; consider only the same raft/CCD/etc. as rawRef 

rawTemplateRef = _siblingRef(rawRef, "raw", templateId) 

calexpTemplateRef = _siblingRef(calexpRef, "calexp", templateId) 

if "ccdProcessor" not in reuse or not calexpTemplateRef.datasetExists("calexp", write=True): 

self.runProcessCcd(rawTemplateRef) 

 

if "ccdProcessor" in reuse and calexpRef.datasetExists("calexp", write=True): 

self.log.info("ProcessCcd has already been run for {0}, skipping...".format(rawRef.dataId)) 

processResults = None 

else: 

processResults = self.runProcessCcd(rawRef) 

 

diffType = self.config.differencer.coaddName 

if "differencer" in reuse and calexpRef.datasetExists(diffType + "Diff_diaSrc", write=True): 

self.log.info("DiffIm has already been run for {0}, skipping...".format(calexpRef.dataId)) 

diffImResults = None 

else: 

diffImResults = self.runDiffIm(calexpRef, templateIds) 

 

try: 

if "associator" in reuse: 

warnings.warn( 

"Reusing association results for some images while rerunning " 

"others may change the associations. If exact reproducibility " 

"matters, please clear the association database and run " 

"ap_pipe.py with --reuse-output-from=differencer to redo all " 

"association results consistently.") 

if "associator" in reuse and \ 

daxPpdb.isVisitProcessed(self.ppdb, calexpRef.get("calexp_visitInfo")): 

message = "Association has already been run for {0}, skipping...".format(calexpRef.dataId) 

self.log.info(message) 

associationResults = None 

else: 

associationResults = self.runAssociation(calexpRef) 

except (OperationalError, ProgrammingError) as e: 

# Don't use lsst.pipe.base.TaskError because it mixes poorly with exception chaining 

raise RuntimeError("Database query failed; did you call make_ppdb.py first?") from e 

 

return pipeBase.Struct( 

l1Database=self.ppdb, 

ccdProcessor=processResults if processResults else None, 

differencer=diffImResults if diffImResults else None, 

associator=associationResults.taskResults if associationResults else None 

) 

 

@pipeBase.timeMethod 

def runProcessCcd(self, sensorRef): 

"""Perform ISR with ingested images and calibrations via processCcd. 

 

The output repository associated with ``sensorRef`` will be populated with the 

usual post-ISR data (bkgd, calexp, icExp, icSrc, postISR). 

 

Parameters 

---------- 

sensorRef : `lsst.daf.persistence.ButlerDataRef` 

Data reference for raw data. 

 

Returns 

------- 

result : `lsst.pipe.base.Struct` 

Output of `config.ccdProcessor.runDataRef`. 

 

Notes 

----- 

The input repository corresponding to ``sensorRef`` must already contain the refcats. 

""" 

self.log.info("Running ProcessCcd...") 

return self.ccdProcessor.runDataRef(sensorRef) 

 

@pipeBase.timeMethod 

def runDiffIm(self, sensorRef, templateIds=None): 

"""Do difference imaging with a template and a science image 

 

The output repository associated with ``sensorRef`` will be populated with difference images 

and catalogs of detected sources (diaSrc, diffexp, and metadata files) 

 

Parameters 

---------- 

sensorRef : `lsst.daf.persistence.ButlerDataRef` 

Data reference for multiple dataset types, both input and output. 

templateIds : `list` of `dict`, optional 

A list of parsed data IDs for templates to use. Only used if 

``config.differencer`` is configured to do so. ``differencer`` or 

its subtasks may restrict the allowed IDs. 

 

Returns 

------- 

result : `lsst.pipe.base.Struct` 

Output of `config.differencer.runDataRef`. 

""" 

self.log.info("Running ImageDifference...") 

return self.differencer.runDataRef(sensorRef, templateIdList=templateIds) 

 

@pipeBase.timeMethod 

def runAssociation(self, sensorRef): 

"""Do source association. 

 

Parameters 

---------- 

sensorRef : `lsst.daf.persistence.ButlerDataRef` 

Data reference for multiple input dataset types. 

 

Returns 

------- 

result : `lsst.pipe.base.Struct` 

Result struct with components: 

 

- ppdb : `lsst.dax.ppdb.Ppdb` Initialized association database containing final association 

results. 

- taskResults : output of `config.associator.run` (`lsst.pipe.base.Struct`). 

""" 

self.log.info("Running Association...") 

 

diffType = self.config.differencer.coaddName 

 

catalog = sensorRef.get(diffType + "Diff_diaSrc") 

diffim = sensorRef.get(diffType + "Diff_differenceExp") 

 

dia_sources = self.diaSourceDpddifier.run(catalog, 

diffim, 

return_pandas=True) 

results = self.associator.run(dia_sources, diffim, self.ppdb) 

dia_forced_sources = self.diaForcedSource.run( 

results.dia_objects, 

sensorRef.get("ccdExposureId_bits"), 

sensorRef.get("calexp"), 

diffim) 

self.ppdb.storeDiaForcedSources(dia_forced_sources) 

 

return pipeBase.Struct( 

l1Database=self.ppdb, 

taskResults=results 

) 

 

@classmethod 

def _makeArgumentParser(cls): 

"""A parser that can handle extra arguments for ap_pipe. 

""" 

return ApPipeParser(name=cls._DefaultName) 

 

 

def _siblingRef(original, datasetType, dataId): 

"""Construct a new dataRef using an existing dataRef as a template. 

 

The typical application is to construct a data ID that differs from an 

existing ID in one or two keys, but is more specific than expanding a 

partial data ID would be. 

 

Parameters 

---------- 

original : `lsst.daf.persistence.ButlerDataRef` 

A dataRef related to the desired one. Assumed to represent a unique dataset. 

datasetType : `str` 

The desired type of the new dataRef. Must be compatible 

with ``original``. 

dataId : `dict` from `str` to any 

A possibly partial data ID for the new dataRef. Any properties left 

unspecified shall be copied from ``original``. 

 

Returns 

------- 

dataRef : `lsst.daf.persistence.ButlerDataRef` 

A dataRef to the same butler as ``original``, but of type 

``datasetType`` and with data ID equivalent to 

``original.dataId.update(dataId)``. 

""" 

butler = original.getButler() 

return butler.dataRef(datasetType, dataId=original.dataId, **dataId)