4 from dateutil
import parser
12 def _convertToDate(dateString):
13 """Convert a string into a date object"""
14 return parser.parse(dateString).date()
18 """Task that will parse the filename and/or its contents to get the
19 required information to populate the calibration registry."""
22 """Return a a known calibration dataset type using
23 the observation type in the header keyword OBSTYPE
25 @param filename: Input filename
27 md = readMetadata(filename, self.config.hdu)
28 if not md.exists(
"OBSTYPE"):
29 raise RuntimeError(
"Unable to find the required header keyword OBSTYPE in %s, hdu %d" %
30 (filename, self.config.hdu))
31 obstype = md.getScalar(
"OBSTYPE").strip().lower()
34 elif "zero" in obstype
or "bias" in obstype:
36 elif "dark" in obstype:
38 elif "fringe" in obstype:
40 elif "sky" in obstype:
42 elif "illumcor" in obstype:
44 elif "defects" in obstype:
46 elif "qe_curve" in obstype:
48 elif "linearizer" in obstype:
49 obstype =
"linearizer"
50 elif "crosstalk" in obstype:
52 elif "BFK" in obstype:
57 """Get destination for the file
59 @param butler Data butler
60 @param info File properties, used as dataId for the butler
61 @param filename Input filename
62 @return Destination filename
67 tempinfo = {k: v
for (k, v)
in info.items()
if v
is not None}
69 raw = butler.get(calibType +
"_filename", tempinfo)[0]
78 """Configuration for the CalibsRegisterTask"""
79 tables = ListField(dtype=str, default=[
"bias",
"dark",
"flat",
"fringe",
"sky",
"defects",
"qe_curve",
80 "linearizer",
"crosstalk",
"bfk"], doc=
"Names of tables")
81 calibDate = Field(dtype=str, default=
"calibDate", doc=
"Name of column for calibration date")
82 validStart = Field(dtype=str, default=
"validStart", doc=
"Name of column for validity start")
83 validEnd = Field(dtype=str, default=
"validEnd", doc=
"Name of column for validity stop")
84 detector = ListField(dtype=str, default=[
"filter",
"ccd"],
85 doc=
"Columns that identify individual detectors")
86 validityUntilSuperseded = ListField(dtype=str, default=[
"defects",
"qe_curve",
"linearizer",
"crosstalk",
88 doc=
"Tables for which to set validity for a calib from when it is "
89 "taken until it is superseded by the next; validity in other tables "
90 "is calculated by applying the validity range.")
91 incrementValidEnd = Field(
94 doc=
"Fix the off-by-one error by incrementing validEnd. See "
95 "fixSubsetValidity for more details.",
100 """Task that will generate the calibration registry for the Mapper"""
101 ConfigClass = CalibsRegisterConfig
103 def openRegistry(self, directory, create=False, dryrun=False, name="calibRegistry.sqlite3"):
104 """Open the registry and return the connection handle"""
105 return RegisterTask.openRegistry(self, directory, create, dryrun, name)
108 """Create the registry tables"""
109 for table
in self.config.tables:
110 RegisterTask.createTable(self, conn, table=table, forceCreateTables=forceCreateTables)
112 def addRow(self, conn, info, *args, **kwargs):
113 """Add a row to the file table"""
114 info[self.config.validStart] =
None
115 info[self.config.validEnd] =
None
116 RegisterTask.addRow(self, conn, info, *args, **kwargs)
119 """Loop over all tables, filters, and ccdnums,
120 and update the validity ranges in the registry.
122 @param conn: Database connection
123 @param validity: Validity range (days)
125 conn.row_factory = sqlite3.Row
126 cursor = conn.cursor()
128 tables = self.config.tables
130 sql =
"SELECT DISTINCT %s FROM %s" % (
", ".join(self.config.detector), table)
132 rows = cursor.fetchall()
137 """Update the validity ranges among selected rows in the registry.
139 For defects and qe_curve, the products are valid from their start date until
140 they are superseded by subsequent defect data.
141 For other calibration products, the validity ranges are checked and
142 if there are overlaps, a midpoint is used to fix the overlaps,
143 so that the calibration data with whose date is nearest the date
144 of the observation is used.
146 DM generated calibrations contain a CALIB_ID header
147 keyword. These calibrations likely require the
148 incrementValidEnd configuration option set to True. Other
149 calibrations generate the calibDate via the DATE-OBS header
150 keyword, and likely require incrementValidEnd=False.
152 @param conn: Database connection
153 @param table: Name of table to be selected
154 @param detectorData: Values identifying a detector (from columns in self.config.detector)
155 @param validity: Validity range (days)
157 columns =
", ".join([self.config.calibDate, self.config.validStart, self.config.validEnd])
158 sql =
"SELECT id, %s FROM %s" % (columns, table)
159 sql +=
" WHERE " +
" AND ".join(col +
"=?" for col
in self.config.detector)
160 sql +=
" ORDER BY " + self.config.calibDate
161 cursor = conn.cursor()
162 cursor.execute(sql, detectorData)
163 rows = cursor.fetchall()
166 valids = collections.OrderedDict([(_convertToDate(row[self.config.calibDate]), [
None,
None])
for
169 det =
" ".join(
"%s=%s" % (k, v)
for k, v
in zip(self.config.detector, detectorData))
171 self.log.warn(str(
"Skipped setting the validity overlaps for %s %s: missing calibration dates" %
174 dates = list(valids.keys())
175 if table
in self.config.validityUntilSuperseded:
177 for thisDate, nextDate
in zip(dates[:-1], dates[1:]):
178 valids[thisDate][0] = thisDate
179 valids[thisDate][1] = nextDate
180 valids[dates[-1]][0] = dates[-1]
181 valids[dates[-1]][1] = _convertToDate(
"2037-12-31")
185 valids[dd] = [dd - datetime.timedelta(validity), dd + datetime.timedelta(validity)]
188 midpoints = [t1 + (t2 - t1)//2
for t1, t2
in zip(dates[:-1], dates[1:])]
189 for i, (date, midpoint)
in enumerate(zip(dates[:-1], midpoints)):
190 if valids[date][1] > midpoint:
191 nextDate = dates[i + 1]
192 valids[nextDate][0] = midpoint + datetime.timedelta(1)
193 if self.config.incrementValidEnd:
194 valids[date][1] = midpoint + datetime.timedelta(1)
196 valids[date][1] = midpoint
201 calibDate = _convertToDate(row[self.config.calibDate])
202 validStart = valids[calibDate][0].isoformat()
203 validEnd = valids[calibDate][1].isoformat()
204 sql =
"UPDATE %s" % table
205 sql +=
" SET %s=?, %s=?" % (self.config.validStart, self.config.validEnd)
207 conn.execute(sql, (validStart, validEnd, row[
"id"]))
211 """Argument parser to support ingesting calibration images into the repository"""
214 InputOnlyArgumentParser.__init__(self, *args, **kwargs)
215 self.add_argument(
"-n",
"--dry-run", dest=
"dryrun", action=
"store_true",
216 default=
False, help=
"Don't perform any action?")
217 self.add_argument(
"--mode", choices=[
"move",
"copy",
"link",
"skip"], default=
"move",
218 help=
"Mode of delivering the files to their destination")
219 self.add_argument(
"--create", action=
"store_true", help=
"Create new registry?")
220 self.add_argument(
"--validity", type=int, required=
True, help=
"Calibration validity period (days)")
221 self.add_argument(
"--ignore-ingested", dest=
"ignoreIngested", action=
"store_true",
222 help=
"Don't register files that have already been registered")
223 self.add_argument(
"files", nargs=
"+", help=
"Names of file")
227 """Configuration for IngestCalibsTask"""
228 parse = ConfigurableField(target=CalibsParseTask, doc=
"File parsing")
229 register = ConfigurableField(target=CalibsRegisterTask, doc=
"Registry entry")
230 allowError = Field(dtype=bool, default=
False, doc=
"Allow error in ingestion?")
231 clobber = Field(dtype=bool, default=
False, doc=
"Clobber existing file?")
235 """Task that generates registry for calibration images"""
236 ConfigClass = IngestCalibsConfig
237 ArgumentParser = IngestCalibsArgumentParser
238 _DefaultName =
"ingestCalibs"
241 """Ingest all specified files and add them to the registry"""
242 calibRoot = args.calib
if args.calib
is not None else args.output
243 filenameList = self.
expandFilesexpandFiles(args.files)
244 with self.register.openRegistry(calibRoot, create=args.create, dryrun=args.dryrun)
as registry:
246 for infile
in filenameList:
247 fileInfo, hduInfoList = self.parse.getInfo(infile)
248 calibType = self.parse.getCalibType(infile)
249 if calibType
not in self.register.config.tables:
250 self.log.warn(str(
"Skipped adding %s of observation type '%s' to registry "
251 "(must be one of %s)" %
252 (infile, calibType,
", ".join(self.register.config.tables))))
254 calibTypes.add(calibType)
255 if args.mode !=
'skip':
256 outfile = self.parse.getDestination(args.butler, fileInfo, infile)
257 ingested = self.
ingestingest(infile, outfile, mode=args.mode, dryrun=args.dryrun)
259 self.log.warn(str(
"Failed to ingest %s of observation type '%s'" %
260 (infile, calibType)))
262 if self.register.check(registry, fileInfo, table=calibType):
263 if args.ignoreIngested:
266 self.log.warn(
"%s: already ingested: %s" % (infile, fileInfo))
267 for info
in hduInfoList:
268 self.register.addRow(registry, info, dryrun=args.dryrun,
269 create=args.create, table=calibType)
271 self.register.updateValidityRanges(registry, args.validity, tables=calibTypes)
273 self.log.info(
"Would update validity ranges here, but dryrun")
def expandFiles(self, fileNameList)
Expand a set of filenames and globs, returning a list of filenames.
def ingest(self, infile, outfile, mode="move", dryrun=False)
def getCalibType(self, filename)
def getDestination(self, butler, info, filename)
def fixSubsetValidity(self, conn, table, detectorData, validity)
def addRow(self, conn, info, *args, **kwargs)
def openRegistry(self, directory, create=False, dryrun=False, name="calibRegistry.sqlite3")
def createTable(self, conn, forceCreateTables=False)
def updateValidityRanges(self, conn, validity, tables=None)
def __init__(self, *args, **kwargs)