11#include "boost/multi_index_container.hpp"
12#include "boost/multi_index/sequenced_index.hpp"
13#include "boost/multi_index/ordered_index.hpp"
14#include "boost/multi_index/hashed_index.hpp"
15#include "boost/multi_index/member.hpp"
34template <std::
string FitsSchemaItem::*Member>
35struct SetFitsSchemaString {
36 void operator()(FitsSchemaItem &item) { item.*Member = _v; }
37 explicit SetFitsSchemaString(
std::string const &v) : _v(v) {}
51 using InputContainer = boost::multi_index_container<FitsSchemaItem, boost::multi_index::indexed_by<boost::multi_index::ordered_non_unique<boost::multi_index::member<FitsSchemaItem, int, &FitsSchemaItem::column>>, boost::multi_index::ordered_non_unique<boost::multi_index::member<FitsSchemaItem, int, &FitsSchemaItem::bit>>, boost::multi_index::hashed_unique<boost::multi_index::member<FitsSchemaItem, std::string, &FitsSchemaItem::ttype>>, boost::multi_index::sequenced<>>>;
54 using SetTTYPE = SetFitsSchemaString<&FitsSchemaItem::ttype>;
55 using SetTFORM = SetFitsSchemaString<&FitsSchemaItem::tform>;
56 using SetTCCLS = SetFitsSchemaString<&FitsSchemaItem::tccls>;
57 using SetTUNIT = SetFitsSchemaString<&FitsSchemaItem::tunit>;
58 using SetDoc = SetFitsSchemaString<&FitsSchemaItem::doc>;
90 : _impl(
std::make_shared<
Impl>()) {
93 if (!metadata.
exists(
"AFW_TYPE")) {
96 _impl->version = metadata.
get(
"AFW_TABLE_VERSION", _impl->version);
97 _impl->type = metadata.
get(
"AFW_TYPE", _impl->type);
99 metadata.
remove(
"AFW_TABLE_VERSION");
102 metadata.
remove(
"AFW_TYPE");
106 _impl->archiveHdu = metadata.
get(
"AR_HDU", -1);
107 if (_impl->archiveHdu > 0) {
110 metadata.
remove(
"AR_HDU");
117 for (
auto const &rawAliase : rawAliases) {
119 if (pos == std::string::npos) {
121 (boost::format(
"Malformed alias definition: '%s'") % rawAliase).
str());
123 _impl->schema.getAliasMap()->set(rawAliase.substr(0, pos), rawAliase.substr(pos + 1, std::string::npos));
132 if (_impl->version == 0) {
141 for (
auto const &oldSlotKey : oldSlotKeys) {
144 _impl->schema.getAliasMap()->set(oldSlotKey.second,
target);
146 metadata.
remove(oldSlotKey.first);
147 metadata.
remove(oldSlotKey.first +
"_ERR_SLOT");
148 metadata.
remove(oldSlotKey.first +
"_FLAG_SLOT");
156 for (
auto const &key : keyList) {
157 if (key.compare(0, 5,
"TTYPE") == 0) {
158 int column =
std::stoi(key.substr(5)) - 1;
159 auto iter = _impl->byColumn().lower_bound(column);
160 if (
iter == _impl->byColumn().end() ||
iter->column != column) {
165 if (
iter->doc.empty()) {
171 }
else if (key.compare(0, 5,
"TFLAG") == 0) {
173 auto iter = _impl->byBit().lower_bound(bit);
174 if (
iter == _impl->byBit().end() ||
iter->bit != bit) {
179 if (
iter->doc.empty()) {
185 }
else if (key.compare(0, 4,
"TDOC") == 0) {
186 int column =
std::stoi(key.substr(4)) - 1;
187 auto iter = _impl->byColumn().lower_bound(column);
188 if (
iter == _impl->byColumn().end() ||
iter->column != column) {
195 }
else if (key.compare(0, 5,
"TFDOC") == 0) {
197 auto iter = _impl->byBit().lower_bound(bit);
198 if (
iter == _impl->byBit().end() ||
iter->bit != bit) {
205 }
else if (key.compare(0, 5,
"TUNIT") == 0) {
206 int column =
std::stoi(key.substr(5)) - 1;
207 auto iter = _impl->byColumn().lower_bound(column);
208 if (
iter == _impl->byColumn().end() ||
iter->column != column) {
215 }
else if (key.compare(0, 5,
"TCCLS") == 0) {
216 int column =
std::stoi(key.substr(5)) - 1;
217 auto iter = _impl->byColumn().lower_bound(column);
218 if (
iter == _impl->byColumn().end() ||
iter->column != column) {
225 }
else if (key.compare(0, 5,
"TFORM") == 0) {
226 int column =
std::stoi(key.substr(5)) - 1;
227 auto iter = _impl->byColumn().lower_bound(column);
228 if (
iter == _impl->byColumn().end() ||
iter->column != column) {
235 }
else if (key.compare(0, 5,
"TZERO") == 0) {
239 }
else if (key.compare(0, 5,
"TSCAL") == 0) {
243 }
else if (key.compare(0, 5,
"TNULL") == 0) {
247 }
else if (key.compare(0, 5,
"TDISP") == 0) {
255 _impl->flagColumn = metadata.
get(
"FLAGCOL", 0);
256 if (_impl->flagColumn > 0) {
258 metadata.
remove(
"FLAGCOL");
261 auto iter = _impl->byColumn().find(_impl->flagColumn);
262 if (
iter == _impl->byColumn().end()) {
265 (boost::format(
"Column for flag data not found; FLAGCOL=%d") % _impl->flagColumn).str());
270 static std::regex const regex(
"(\\d+)?X\\(?(\\d)*\\)?");
275 (boost::format(
"Invalid TFORM key for flags column: '%s'") %
iter->tform).str());
281 _impl->flagKeys.resize(
nFlags);
282 _impl->flagWorkspace.reset(
new bool[
nFlags]);
285 _impl->byColumn().erase(
iter);
298 int oldHdu =
fits.getHdu();
299 if (_impl->archiveHdu < 0) _impl->archiveHdu = oldHdu + 1;
301 fits.setHdu(_impl->archiveHdu);
308 _impl->archiveHdu = -1;
316 auto iter = _impl->byName().find(ttype);
317 if (
iter == _impl->byName().end()) {
324 auto iter = _impl->byColumn().lower_bound(column);
325 if (
iter == _impl->byColumn().end() ||
iter->column != column) {
332 auto iter = _impl->byColumn().lower_bound(item->
column);
333 assert(
iter != _impl->byColumn().end() &&
iter->column == item->
column);
334 _impl->byColumn().erase(
iter);
338 auto iter = _impl->byName().find(ttype);
339 if (
iter != _impl->byName().end() &&
iter->ttype == ttype) {
340 _impl->byName().erase(
iter);
345 auto iter = _impl->byColumn().lower_bound(column);
346 if (
iter != _impl->byColumn().end() &&
iter->column == column) {
347 _impl->byColumn().erase(
iter);
354 _impl->readers.push_back(
std::move(reader));
368 : _column(item.column), _key(
schema.addField<T>(item.ttype, item.doc, item.tunit, base)),
369 _cache(), _cacheFirstRow(0)
376 if (_key.getElementCount() == 1u) {
377 std::size_t nElements = nRows*_key.getElementCount();
378 _cache.resize(nElements);
379 _cacheFirstRow = firstRow;
380 fits.readTableArray(firstRow, _column, nElements, &_cache.front());
384 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
386 if (_cache.empty()) {
387 fits.readTableArray(row, _column, _key.getElementCount(), record.getElement(_key));
389 assert(row >= _cacheFirstRow);
391 assert(offset < _cache.size());
392 std::copy_n(_cache.begin() + offset, _key.getElementCount(), record.getElement(_key));
404class AngleReader :
public FitsColumnReader {
407 Schema &
schema, FitsSchemaItem
const &item,
408 FieldBase<lsst::geom::Angle>
const &base = FieldBase<lsst::geom::Angle>()) {
412 AngleReader(Schema &
schema, FitsSchemaItem
const &item, FieldBase<lsst::geom::Angle>
const &base)
413 : _column(item.column), _key(
schema.addField<
lsst::geom::
Angle>(item.ttype, item.doc,
"", base)) {
418 if (!item.tunit.empty() && item.tunit !=
"rad") {
420 "Angle fields must be persisted in radians (TUNIT='rad').");
425 assert(_key.getElementCount() == 1u);
426 _cache.resize(nRows);
427 _cacheFirstRow = firstRow;
428 fits.readTableArray(firstRow, _column, nRows, &_cache.front());
431 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
433 if (_cache.empty()) {
435 fits.readTableScalar(row, _column, tmp);
438 assert(row >= _cacheFirstRow);
440 assert(offset < _cache.size());
447 Key<lsst::geom::Angle> _key;
452class StringReader :
public FitsColumnReader {
458 StringReader(Schema &
schema, FitsSchemaItem
const &item,
int size)
459 : _column(item.column),
460 _key(
schema.addField<
std::string>(item.ttype, item.doc, item.tunit, size)),
461 _isVariableLength(size == 0) {}
463 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
466 fits.readTableScalar(row, _column, s, _isVariableLength);
472 Key<std::string> _key;
473 bool _isVariableLength;
477class VariableLengthArrayReader :
public FitsColumnReader {
483 VariableLengthArrayReader(Schema &
schema, FitsSchemaItem
const &item)
484 : _column(item.column), _key(
schema.addField<Array<T>>(item.ttype, item.doc, item.tunit, 0)) {}
486 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
488 int size =
fits.getTableArraySize(row, _column);
489 ndarray::Array<T, 1, 1> array = ndarray::allocate(size);
490 fits.readTableArray(row, _column, size, array.getData());
491 record.set(_key, array);
502class PointConversionReader :
public FitsColumnReader {
508 PointConversionReader(Schema &
schema, FitsSchemaItem
const &item)
509 : _column(item.column), _key(PointKey<T>::addFields(
schema, item.ttype, item.doc, item.tunit)) {}
511 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
514 fits.readTableArray(row, _column, 2, buffer.
data());
525class CoordConversionReader :
public FitsColumnReader {
531 CoordConversionReader(Schema &
schema, FitsSchemaItem
const &item)
532 : _column(item.column), _key(CoordKey::addFields(
schema, item.ttype, item.doc)) {}
534 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
537 fits.readTableArray(row, _column, 2, buffer.
data());
548class MomentsConversionReader :
public FitsColumnReader {
554 MomentsConversionReader(Schema &
schema, FitsSchemaItem
const &item)
555 : _column(item.column),
558 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
561 fits.readTableArray(row, _column, 3, buffer.
data());
562 record.set(_key, geom::ellipses::Quadrupole(buffer[0], buffer[1], buffer[2],
false));
573template <
typename T,
int N>
574class CovarianceConversionReader :
public FitsColumnReader {
577 static std::regex const regex(
"(.*)(\\^(\\d+))?");
592 CovarianceConversionReader(Schema &
schema, FitsSchemaItem
const &item,
594 : _column(item.column),
596 _key(CovarianceMatrixKey<T, N>::addFields(
schema, item.ttype, names, guessUnits(item.tunit))),
599 void readCell(BaseRecord &record,
std::size_t row, afw::fits::Fits &
fits,
602 for (
int i = 0; i < _size; ++i) {
603 for (
int j = i; j < _size; ++j) {
612 CovarianceMatrixKey<T, N> _key;
620 static std::regex const regex(
"(\\d+)?([PQ])?([A-Z])\\(?(\\d)*\\)?");
630 char code =
m[3].str()[0];
640 if (item.tccls ==
"Array") {
641 return StandardReader<Array<std::uint8_t>>::make(
schema, item, size);
643 return StandardReader<std::uint8_t>::make(
schema, item);
646 return VariableLengthArrayReader<std::uint8_t>::make(
schema, item);
648 return StandardReader<Array<std::uint8_t>>::make(
schema, item, size);
653 if (item.tccls ==
"Array") {
654 return StandardReader<Array<std::uint16_t>>::make(
schema, item, size);
656 return StandardReader<std::uint16_t>::make(
schema, item);
659 return VariableLengthArrayReader<std::uint16_t>::make(
schema, item);
661 return StandardReader<Array<std::uint16_t>>::make(
schema, item, size);
664 return VariableLengthArrayReader<std::int32_t>::make(
schema, item);
666 if (item.tccls ==
"Point") {
667 return PointConversionReader<std::int32_t>::make(
schema, item);
669 if (size > 1 || item.tccls ==
"Array") {
670 return StandardReader<Array<std::int32_t>>::make(
schema, item, size);
672 return StandardReader<std::int32_t>::make(
schema, item);
675 return StandardReader<std::int64_t>::make(
schema, item);
679 return VariableLengthArrayReader<float>::make(
schema, item);
682 if (item.tccls ==
"Array") {
683 return StandardReader<Array<float>>::make(
schema, item, 1);
687 return StandardReader<float>::make(
schema, item);
689 if (size == 3 && item.tccls ==
"Covariance(Point)") {
691 return CovarianceConversionReader<float, 2>::make(
schema, item, names);
693 if (size == 6 && item.tccls ==
"Covariance(Moments)") {
695 return CovarianceConversionReader<float, 3>::make(
schema, item, names);
697 if (item.tccls ==
"Covariance") {
698 double v = 0.5 * (
std::sqrt(1 + 8 * size) - 1);
700 if (n * (n + 1) != size * 2) {
701 throw LSST_EXCEPT(afw::fits::FitsError,
"Covariance field has invalid size.");
707 return CovarianceConversionReader<float, Eigen::Dynamic>::make(
schema, item, names);
709 return StandardReader<Array<float>>::make(
schema, item, size);
712 return VariableLengthArrayReader<double>::make(
schema, item);
715 if (item.tccls ==
"Angle") {
716 return AngleReader::make(
schema, item);
718 if (item.tccls ==
"Array") {
719 return StandardReader<Array<double>>::make(
schema, item, 1);
721 return StandardReader<double>::make(
schema, item);
724 if (item.tccls ==
"Point") {
725 return PointConversionReader<double>::make(
schema, item);
727 if (item.tccls ==
"Coord") {
728 return CoordConversionReader::make(
schema, item);
731 if (size == 3 && item.tccls ==
"Moments") {
732 return MomentsConversionReader::make(
schema, item);
734 return StandardReader<Array<double>>::make(
schema, item, size);
737 return StringReader::make(
schema, item, size);
747bool isInstFlux(FitsSchemaItem
const & item) {
752 if (!
includes(item.ttype,
"flux"))
return false;
753 if (
includes(item.ttype,
"modelfit_CModel") && item.tunit.empty()) {
760 std::transform(units.begin(), units.end(), units.begin(), [](
char c) { return std::tolower(c); } );
772 if (_impl->version == 0) {
773 AliasMap &aliases = *_impl->schema.getAliasMap();
774 for (
auto iter = _impl->asList().begin();
iter != _impl->asList().end(); ++
iter) {
776 if (flagPos != std::string::npos) {
785 ttype.
replace(flagPos, 5,
"flag");
792 }
else if (isInstFlux(*
iter)) {
794 if (endswith(
iter->ttype,
"_err")) {
795 aliases.
set(replace(
iter->ttype,
"_err",
"_instFluxErr"),
iter->ttype);
797 aliases.
set(
iter->ttype +
"_instFlux",
iter->ttype);
799 }
else if (endswith(
iter->ttype,
"_err")) {
805 if (
iter->tccls ==
"Covariance(Point)") {
806 aliases.
set(replace(
iter->ttype,
"_err",
"_yErr"),
iter->ttype +
"_yErr");
807 aliases.
set(replace(
iter->ttype,
"_err",
"_xErr"),
iter->ttype +
"_xErr");
808 aliases.
set(replace(
iter->ttype,
"_err",
"_x_y_Cov"),
iter->ttype +
"_x_y_Cov");
809 }
else if (
iter->tccls ==
"Covariance(Moments)") {
810 aliases.
set(replace(
iter->ttype,
"_err",
"_xxErr"),
iter->ttype +
"_xxErr");
811 aliases.
set(replace(
iter->ttype,
"_err",
"_yyErr"),
iter->ttype +
"_yyErr");
812 aliases.
set(replace(
iter->ttype,
"_err",
"_xyErr"),
iter->ttype +
"_xyErr");
813 aliases.
set(replace(
iter->ttype,
"_err",
"_xx_yy_Cov"),
iter->ttype +
"_xx_yy_Cov");
814 aliases.
set(replace(
iter->ttype,
"_err",
"_xx_xy_Cov"),
iter->ttype +
"_xx_xy_Cov");
815 aliases.
set(replace(
iter->ttype,
"_err",
"_yy_xy_Cov"),
iter->ttype +
"_yy_xy_Cov");
819 }
else if (_impl->version < 3) {
823 AliasMap &aliases = *_impl->schema.getAliasMap();
824 for (
auto iter = _impl->asList().begin();
iter != _impl->asList().end(); ++
iter) {
826 if (_impl->version < 2 && endswith(
name,
"Sigma")) {
829 if (_impl->version < 3 && isInstFlux(*
iter)) {
837 for (
auto iter = _impl->asList().begin();
iter != _impl->asList().end(); ++
iter) {
841 _impl->readers.push_back(
std::move(reader));
843 LOGLS_WARN(
"lsst.afw.FitsSchemaInputMapper",
"Format " <<
iter->tform <<
" for column "
845 <<
" not supported; skipping.");
850 (boost::format(
"Flag field '%s' is is in bit %d (0-indexed) of only %d") %
851 iter->ttype %
iter->bit % _impl->flagKeys.size())
854 _impl->flagKeys[
iter->bit] = _impl->schema.addField<Flag>(
iter->ttype,
iter->doc);
857 _impl->asList().clear();
858 if (_impl->schema.getRecordSize() <= 0) {
861 (boost::format(
"Non-positive record size: %d; file is corrupt or invalid.") %
862 _impl->schema.getRecordSize()).str()
866 return _impl->schema;
870 if (!_impl->flagKeys.empty()) {
871 fits.readTableArray<
bool>(row, _impl->flagColumn, _impl->flagKeys.size(), _impl->flagWorkspace.get());
872 for (
std::size_t bit = 0; bit < _impl->flagKeys.size(); ++bit) {
873 record.
set(_impl->flagKeys[bit], _impl->flagWorkspace[bit]);
876 if (_impl->nRowsToPrep != 1 && row % _impl->nRowsToPrep == 0) {
880 for (
auto const &reader : _impl->readers) {
881 reader->prepRead(row, size,
fits);
884 for (
auto const & reader : _impl->readers) {
885 reader->readCell(record, row,
fits, _impl->archive);
table::Key< std::string > name
Key< Flag > const & target
#define LSST_EXCEPT(type,...)
#define LOGLS_WARN(logger, message)
An exception thrown when problems are found when reading or writing FITS files.
A simple struct that combines the two arguments that must be passed to most cfitsio routines and cont...
Mapping class that holds aliases for a Schema.
void set(std::string const &alias, std::string const &target)
Add an alias to the schema or replace an existing one.
Base class for all records.
void set(Key< T > const &key, U const &value)
Set value of a field for the given key.
Defines the fields and offsets for a table.
Polymorphic reader interface used to read different kinds of objects from one or more FITS binary tab...
T get(std::string const &name) const
std::string const & getComment(std::string const &name) const
std::vector< T > getArray(std::string const &name) const
std::vector< std::string > getOrderedNames() const
virtual void remove(std::string const &name)
bool exists(std::string const &name) const
std::size_t computeCovariancePackedSize(std::size_t size)
Defines the packed size of a covariance matrices.
std::size_t indexCovariance(std::size_t i, std::size_t j)
Defines the ordering of packed covariance matrices.
CoordinateType
Enum used to set units for geometric FunctorKeys.
AngleUnit constexpr radians
Field base class default implementation (used for numeric scalars and lsst::geom::Angle).
A structure that describes a field as a collection of related strings read from the FITS header.