33 #include "muse_quality.h" 34 #include "muse_instrument.h" 36 #include "muse_cplwrappers.h" 37 #include "muse_pfits.h" 38 #include "muse_quadrants.h" 39 #include "muse_tracing.h" 40 #include "muse_utils.h" 41 #include "muse_data_format_z.h" 82 cpl_ensure(aDark, CPL_ERROR_NULL_INPUT, -1);
83 float *data = cpl_image_get_data_float(aDark->
data);
84 int *dq = cpl_image_get_data_int(aDark->
dq);
85 cpl_ensure(data && dq, CPL_ERROR_ILLEGAL_INPUT, -2);
86 #ifdef MUSE_DARK_PREPARE_TEST_DATA 88 x->
data = cpl_image_extract(aDark->
data, 1780, 1922, 3000, 4112);
89 x->
dq = cpl_image_extract(aDark->
dq, 1780, 1922, 3000, 4112);
90 x->
stat = cpl_image_extract(aDark->
stat, 1780, 1922, 3000, 4112);
92 cpl_propertylist_update_int(x->
header,
"ESO DET OUT1 NX", 269);
93 cpl_propertylist_update_int(x->
header,
"ESO DET OUT1 NY", 135);
94 cpl_propertylist_update_int(x->
header,
"ESO DET OUT2 NX", 952);
95 cpl_propertylist_update_int(x->
header,
"ESO DET OUT2 NY", 135);
96 cpl_propertylist_update_int(x->
header,
"ESO DET OUT3 NX", 269);
98 cpl_propertylist_update_int(x->
header,
"ESO DET OUT4 NX", 952);
100 const char *fn =
"muse_test_quality_dark.fits";
102 cpl_msg_debug(__func__,
"saved as \"%s\"", fn);
110 cpl_msg_debug(__func__,
"%d incoming bad pixels", nbad);
111 cpl_binary *databpm = cpl_mask_get_data(cpl_image_get_bpm(aDark->
data)),
114 statbpm = cpl_mask_get_data(cpl_image_get_bpm(aDark->
stat));
117 int nlo = 0, nhi = 0;
119 for (n = 1; n <= 4; n++) {
121 cpl_stats_mode smode = CPL_STATS_MEDIAN | CPL_STATS_MEDIAN_DEV
122 | CPL_STATS_MIN | CPL_STATS_MAX;
123 cpl_stats *s = cpl_stats_new_from_image_window(aDark->
data, smode,
124 w[0], w[2], w[1], w[3]);
125 double median = cpl_stats_get_median(s),
126 mdev = cpl_stats_get_median_dev(s),
127 min = cpl_stats_get_min(s),
128 max = cpl_stats_get_max(s),
129 locut = aSigmaLo > 0 ? median - aSigmaLo * mdev : min,
130 hicut = aSigmaHi > 0 ? median + aSigmaHi * mdev : max;
131 cpl_msg_debug(__func__,
"quadrant %d bad pixel limits: %g ... %g +/- %g ... %g",
132 n, locut, median, mdev, hicut);
134 cpl_stats_dump(s, smode, stdout);
139 int i, nx = cpl_image_get_size_x(aDark->
data);
140 for (i = w[0] - 1; i < w[1]; i++) {
142 for (j = w[2] - 1; j < w[3]; j++) {
143 cpl_size pos = i + j*nx;
144 double value = data[pos];
146 dq[pos] |= EURO3D_DEADPIXEL;
147 databpm[pos] = CPL_BINARY_1;
149 statbpm[pos] = CPL_BINARY_1;
154 dq[pos] |= EURO3D_HOTPIXEL;
155 databpm[pos] = CPL_BINARY_1;
157 statbpm[pos] = CPL_BINARY_1;
165 if (nlo || aSigmaLo > 0) {
166 cpl_msg_info(__func__,
"%d pixel%s lower than %.3f sigma marked as dark",
167 nlo, nlo != 1 ?
"s" :
"", aSigmaLo);
169 if (nhi || aSigmaHi > 0) {
170 cpl_msg_info(__func__,
"%d pixel%s higher than %.3f sigma marked as hot",
171 nhi, nhi != 1 ?
"s" :
"", aSigmaHi);
195 cpl_ensure(aBias && aBias->
data && aBias->
dq && aBias->
stat && aBias->
header,
196 CPL_ERROR_NULL_INPUT, -1);
198 int nx = cpl_image_get_size_x(aBias->
data),
201 for (n = 1; n <= 4; n++) {
204 cpl_vector *vmean = cpl_vector_new(w[1] - w[0] + 1),
205 *vstdev = cpl_vector_new(w[1] - w[0] + 1);
207 for (i = w[0]; i <= w[1]; i++) {
208 double mean = cpl_image_get_mean_window(aBias->
data, i, w[2], i, w[3]),
209 stdev = cpl_image_get_stdev_window(aBias->
data, i, w[2], i, w[3]);
210 cpl_vector_set(vmean, i - w[0], mean);
211 cpl_vector_set(vstdev, i - w[0], stdev);
215 double cmedian = cpl_vector_get_median_const(vmean),
217 hicut = cmedian + aHigh*cmdev,
218 locut = cmedian - aLow*cmdev;
219 char *keyword = cpl_sprintf(QC_BIAS_MASTER_RON, n);
220 double ron = cpl_propertylist_get_double(aBias->
header, keyword);
222 cpl_msg_debug(__func__,
"quadrant %1d: mean %f+/-%f(%f); valid range " 223 "%f...(%f+/-%f)...%f RON=%f", n, cpl_vector_get_mean(vmean),
224 cpl_vector_get_stdev(vmean), cpl_vector_get_mean(vstdev),
225 locut, cmedian, cmdev, hicut, ron);
228 float *data = cpl_image_get_data_float(aBias->
data);
229 int *dq = cpl_image_get_data_int(aBias->
dq);
230 for (i = w[0]; i <= w[1]; i++) {
231 double cmean = cpl_vector_get(vmean, i - w[0]),
232 cstdev = cpl_vector_get(vstdev, i - w[0]);
233 if (cmean > hicut && cstdev > ron) {
234 cpl_msg_debug(__func__,
"hot column %d (%f+/-%f)", i, cmean, cstdev);
236 int jfirst = w[2], jlast = w[3];
237 for (j = w[2]; j <= w[3]; j++) {
238 if (data[(i-1) + (j-1)*nx] > hicut) {
244 for (j = w[3]; j >= w[2]; j--) {
245 if (data[(i-1) + (j-1)*nx] > hicut) {
251 for (j = jfirst; j <= jlast; j++) {
252 dq[(i-1) + (j-1)*nx] |= EURO3D_DEADPIXEL;
255 }
else if (cmean < locut) {
256 cpl_msg_debug(__func__,
"dark column %d (%f+/-%f)", i, cmean, cstdev);
258 int jfirst = w[2], jlast = w[3];
259 for (j = w[2]; j <= w[3]; j++) {
260 if (data[(i-1) + (j-1)*nx] < locut) {
266 for (j = w[3]; j >= w[2]; j--) {
267 if (data[(i-1) + (j-1)*nx] < locut) {
273 for (j = jfirst; j <= jlast; j++) {
274 dq[(i-1) + (j-1)*nx] |= EURO3D_DEADPIXEL;
280 cpl_vector_delete(vmean);
281 cpl_vector_delete(vstdev);
285 cpl_msg_info(__func__,
"%d low and %d high pixels found", nlo, nhi);
316 double aSigmaLo,
double aSigmaHi)
318 cpl_ensure(aFlat && aFlat->
data && aFlat->
dq && aFlat->
stat && aTrace,
319 CPL_ERROR_NULL_INPUT, -1);
321 cpl_msg_info(__func__,
"Marking dark/bright pixels using sigmas %.2f/%.2f",
323 int ndark = 0, nhot = 0, nlowqe = 0,
324 nx = cpl_image_get_size_x(aFlat->
data),
325 ny = cpl_image_get_size_y(aFlat->
data);
326 float *data = cpl_image_get_data_float(aFlat->
data);
327 int *dq = cpl_image_get_data_int(aFlat->
dq);
331 double mean = cpl_image_get_mean(aFlat->
data);
333 cpl_msg_debug(__func__,
"mean flat value: %f", mean);
336 for (nslice = 1; nslice <= kMuseSlicesPerCCD; nslice++) {
341 cpl_msg_warning(__func__,
"slice %2d: tracing polynomials missing!",
348 for (j = 0; j < ny; j++) {
351 cpl_errorstate prestate = cpl_errorstate_get();
352 double x1 = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT], j+1, NULL),
353 x2 = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT], j+1, NULL);
354 if (!cpl_errorstate_is_equal(prestate) || !isnormal(x1) || !isnormal(x2)
355 || x1 < 1 || x2 > nx || x1 > x2) {
356 cpl_msg_warning(__func__,
"slice %2d: faulty polynomial detected at " 357 "y=%d (borders: %f ... %f): %s", nslice, j+1, x1, x2,
358 cpl_error_get_message());
364 int mleft = ceil(x1),
368 double xc = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER], j+1, NULL);
369 cpl_msg_debug(__func__,
"%02d %04d: %04d/%.3f...%.3f...%04d/%.3f " 370 "(%02d/%.3f wide)", nslice, j+1, mleft,x1, xc, mright,x2,
371 mright - mleft, x2 - x1);
373 unsigned statmask = CPL_STATS_MIN | CPL_STATS_MAX
374 | CPL_STATS_MEAN | CPL_STATS_STDEV
375 | CPL_STATS_MEDIAN | CPL_STATS_MEDIAN_DEV;
376 cpl_stats *stats = cpl_stats_new_from_image_window(aFlat->
data, statmask,
380 cpl_stats_dump(stats, statmask, stdout);
383 double median = cpl_stats_get_median(stats),
384 llim = median - aSigmaLo * cpl_stats_get_median_dev(stats),
385 hlim = median + aSigmaHi * cpl_stats_get_median_dev(stats);
386 cpl_stats_delete(stats);
389 llim = llim <= 0 ? 1.e-4 : llim;
391 cpl_msg_debug(__func__,
"limits: %f...%f (sigmas %.2f/%.2f)", llim, hlim,
397 int i, i1 = mleft - 1, i2 = mright - 1;
398 for (i = mleft - 1; i < mleft + kMuseSliceMaxEdgeWidth; i++) {
399 if (data[i + j*nx] > llim) {
404 for (i = mright - 1; i > mright - kMuseSliceMaxEdgeWidth; i--) {
405 if (data[i + j*nx] > llim) {
413 for (i = i1; i <= i2; i++) {
414 if (data[i + j*nx] < llim) {
415 dq[i + j*nx] |= EURO3D_DARKPIXEL;
416 if (data[i + j*nx] < 0.2 * mean) {
419 dq[i + j*nx] |= EURO3D_LOWQEPIXEL;
423 cpl_msg_debug(__func__,
"%04d,%04d is DARK%s: %f", i+1, j+1,
424 data[i + j*nx] < 0.2 * mean ?
" and low QE" :
"",
428 }
else if (data[i + j*nx] > hlim) {
429 dq[i + j*nx] |= EURO3D_HOTPIXEL;
431 cpl_msg_debug(__func__,
"%04d,%04d is HOT: %f", i+1, j+1, data[i + j*nx]);
443 for (i = 0; i < nx; i++) {
445 for (j = 0; j < ny; j++) {
446 if (data[i + j*nx] <= 0) {
447 dq[i + j*nx] |= EURO3D_BADOTHER;
453 cpl_msg_info(__func__,
"Found %d dark (%d of them are also low QE), %d hot, " 454 "and %d non-positive pixels", ndark, nlowqe, nhot, nnonpos);
475 cpl_ensure(aImage && aImage->
data && aImage->
dq, CPL_ERROR_NULL_INPUT, -1);
477 float *data = cpl_image_get_data_float(aImage->
data);
478 int *dq = cpl_image_get_data_int(aImage->
dq);
479 int i, j, nsaturated = 0,
480 nx = cpl_image_get_size_x(aImage->
data),
481 ny = cpl_image_get_size_y(aImage->
data);
482 for (i = 0; i < nx; i++) {
483 for (j = 0; j < ny; j++) {
484 if (data[i + j*nx] > kMuseSaturationLimit ||
485 data[i + j*nx] < FLT_EPSILON) {
486 dq[i + j*nx] |= EURO3D_SATURATED;
513 cpl_ensure(aDQ, CPL_ERROR_NULL_INPUT, NULL);
514 int nx = cpl_image_get_size_x(aDQ),
515 ny = cpl_image_get_size_y(aDQ);
516 const int *dq = cpl_image_get_data_int_const(aDQ);
518 for (i = 0; i < nx; i++) {
519 for (j = 0; j < ny; j++) {
520 if (dq[i + j*nx] != EURO3D_GOODPIXEL) {
532 for (i = 0; i < nx; i++) {
533 for (j = 0; j < ny; j++) {
534 if (dq[i + j*nx] == EURO3D_GOODPIXEL) {
541 cpl_table_set_int(table, MUSE_BADPIX_X, nbad, xbad);
542 cpl_table_set_int(table, MUSE_BADPIX_Y, nbad, ybad);
543 cpl_table_set_int(table, MUSE_BADPIX_DQ, nbad, dq[i + j*nx]);
569 cpl_ensure_code(aTable && aToMerge, CPL_ERROR_NULL_INPUT);
570 cpl_error_code rc = cpl_table_insert(aTable, aToMerge,
571 cpl_table_get_nrow(aTable));
572 cpl_ensure_code(rc == CPL_ERROR_NONE, rc);
575 cpl_propertylist *order = cpl_propertylist_new();
576 cpl_propertylist_append_bool(order, MUSE_BADPIX_X, FALSE);
577 cpl_propertylist_append_bool(order, MUSE_BADPIX_Y, FALSE);
578 cpl_table_sort(aTable, order);
579 cpl_propertylist_delete(order);
582 cpl_table_unselect_all(aTable);
583 const int *x = cpl_table_get_data_int_const(aTable, MUSE_BADPIX_X),
584 *y = cpl_table_get_data_int_const(aTable, MUSE_BADPIX_Y);
585 int *dq = cpl_table_get_data_int(aTable, MUSE_BADPIX_DQ);
586 float *value = cpl_table_get_data_float(aTable, MUSE_BADPIX_VALUE);
587 int i, nrow = cpl_table_get_nrow(aTable);
588 for (i = 1; i < nrow; i++) {
589 if (x[i] == x[i-1] && y[i] == y[i-1]) {
593 value[i-1] = fmax(value[i-1], value[i]);
595 cpl_table_select_row(aTable, i);
598 rc = cpl_table_erase_selected(aTable);
631 cpl_ensure(aData && aDQ, CPL_ERROR_NULL_INPUT, -1);
632 int nx = cpl_image_get_size_x(aData),
633 ny = cpl_image_get_size_y(aData);
634 cpl_ensure(nx == cpl_image_get_size_x(aDQ) &&
635 ny == cpl_image_get_size_y(aDQ), CPL_ERROR_INCOMPATIBLE_INPUT, -2);
637 cpl_ensure(nx == cpl_image_get_size_x(aStat) &&
638 ny == cpl_image_get_size_y(aStat), CPL_ERROR_INCOMPATIBLE_INPUT,
641 const int *dq = cpl_image_get_data_int_const(aDQ);
646 double cputime = cpl_test_get_cputime(),
647 walltime = cpl_test_get_walltime();
649 cpl_binary *bpm = cpl_mask_get_data(cpl_image_get_bpm(aData)),
652 statbpm = cpl_mask_get_data(cpl_image_get_bpm(aStat));
655 for (i = 0; i < nx; i++) {
656 for (j = 0; j < ny; j++) {
657 if (dq[i + j*nx] == EURO3D_GOODPIXEL) {
661 bpm[i + j*nx] = CPL_BINARY_1;
663 statbpm[i + j*nx] = CPL_BINARY_1;
668 double cputime2 = cpl_test_get_cputime(),
669 walltime2 = cpl_test_get_walltime();
670 cpl_msg_debug(__func__,
"reject_using_dq times new: cpu %f wall %f", cputime2 - cputime,
671 walltime2 - walltime);
699 const char *aExtname,
int *aExt)
701 cpl_ensure(aTable && aInFile, CPL_ERROR_NULL_INPUT, NULL);
703 int extension = cpl_fits_find_extension(aInFile, aExtname);
704 cpl_table *table = NULL;
706 if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
707 printf(
"Input table \"%s\" does not contain a table for EXTNAME=\"%s\"" 708 " yet\n", aInFile, aExtname);
711 table = cpl_table_load(aInFile, extension, 1);
713 printf(
"WARNING: could not load BADPIX_TABLE from EXTNAME=\"%s\" from " 714 "table \"%s\" (the headers say that it should be extension %d)!\n",
715 aExtname, aInFile, extension);
718 cpl_ensure(table, CPL_ERROR_DATA_NOT_FOUND, NULL);
724 cpl_size nold = cpl_table_get_nrow(table);
726 if (rc != CPL_ERROR_NONE) {
727 printf(
"WARNING: Merging input and new table failed: %s\n",
728 cpl_error_get_message());
729 printf(
"Table still has %"CPL_SIZE_FORMAT
" bad pixel%s\n", nold,
730 nold != 1 ?
"s" :
"");
732 cpl_size nbad = cpl_table_get_nrow(table);
733 printf(
"Merged %"CPL_SIZE_FORMAT
" of %"CPL_SIZE_FORMAT
" bad pixel%s into " 734 "the input table (now %"CPL_SIZE_FORMAT
" entries)\n", nbad - nold,
735 cpl_table_get_nrow(table), (nbad - nold) != 1 ?
"s" :
"", nbad);
765 int aExtension,
const cpl_table *aTable)
767 cpl_ensure_code(aInFile && aOutFile && aTable, CPL_ERROR_NULL_INPUT);
769 cpl_errorstate state = cpl_errorstate_get();
770 cpl_error_code rc = CPL_ERROR_NONE;
771 cpl_size nextensions = cpl_fits_count_extensions(aInFile);
772 if (!cpl_errorstate_is_equal(state)) {
773 rc = cpl_error_get_code();
775 if (nextensions > 0) {
776 printf(
"Saving primary header and %"CPL_SIZE_FORMAT
" extensions to \"%s\"\n",
777 nextensions, aOutFile);
780 for (i = 0; i <= nextensions; i++) {
781 cpl_propertylist *extheader = cpl_propertylist_load(aInFile, i);
785 cpl_propertylist_update_string(extheader,
"PIPEFILE", aOutFile);
786 cpl_propertylist_set_comment(extheader,
"PIPEFILE",
787 "pretend to be a pipeline output file");
788 cpl_propertylist_save(extheader, aOutFile, CPL_IO_CREATE);
789 if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
790 printf(
"Saved primary header to \"%s\"\n", aOutFile);
792 cpl_propertylist_delete(extheader);
795 if (aTable && i == aExtension) {
797 printf(
"Saving merged table of IFU %2hhu to extension %"CPL_SIZE_FORMAT
799 cpl_table_save(aTable, NULL, extheader, aOutFile, CPL_IO_EXTEND);
800 cpl_propertylist_delete(extheader);
804 cpl_table *tab = NULL;
805 const char *xtension = cpl_propertylist_get_string(extheader,
"XTENSION");
806 if (xtension && strncmp(xtension,
"BINTABLE", 8)) {
807 if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
808 printf(
"WARNING: Not a binary table extension, skipping data section " 809 "(extension %"CPL_SIZE_FORMAT
")", i);
811 cpl_propertylist_save(extheader, aOutFile, CPL_IO_EXTEND);
813 tab = cpl_table_load(aInFile, i, 1);
814 cpl_table_save(tab, NULL, extheader, aOutFile, CPL_IO_EXTEND);
815 if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
816 printf(
"Saved table extension %"CPL_SIZE_FORMAT
" to \"%s\"\n", i,
820 cpl_table_delete(tab);
821 cpl_propertylist_delete(extheader);
cpl_polynomial ** muse_trace_table_get_polys_for_slice(const cpl_table *aTable, const unsigned short aSlice)
construct polynomial from the trace table entry for the given slice
void muse_image_delete(muse_image *aImage)
Deallocate memory associated to a muse_image object.
int muse_quality_flat_badpix(muse_image *aFlat, cpl_table *aTrace, double aSigmaLo, double aSigmaHi)
Find bad (especially dark) pixels (in a master flat).
cpl_size * muse_quadrants_get_window(const muse_image *aImage, unsigned char aQuadrant)
Determine the data window of a given quadrant on the CCD.
cpl_table * muse_quality_convert_dq(cpl_image *aDQ)
Convert a data quality (DQ) image extension to a bad pixel table.
unsigned char muse_utils_get_ifu(const cpl_propertylist *aHeaders)
Find out the IFU/channel from which this header originated.
cpl_image * data
the data extension
int muse_quality_bad_columns(muse_image *aBias, double aLow, double aHigh)
Find bad columns (in a master bias).
cpl_error_code muse_quality_copy_badpix_table(const char *aInFile, const char *aOutFile, int aExtension, const cpl_table *aTable)
Copy bad pixel table on disk, replacing the table in one extension.
cpl_image * stat
the statistics extension
cpl_table * muse_quality_merge_badpix_from_file(const cpl_table *aTable, const char *aInFile, const char *aExtname, int *aExt)
Merge bad pixel table in memory with table from file on disk.
Structure definition of MUSE three extension FITS file.
cpl_propertylist * header
the FITS header
void muse_trace_polys_delete(cpl_polynomial *aPolys[])
Delete the multi-polynomial array created in relation to tracing.
cpl_image * dq
the data quality extension
cpl_table * muse_cpltable_new(const muse_cpltable_def *aDef, cpl_size aLength)
Create an empty table according to the specified definition.
int muse_quality_set_saturated(muse_image *aImage)
Set all pixels above the saturation limit in the bad pixel image.
cpl_error_code muse_quality_merge_badpix(cpl_table *aTable, const cpl_table *aToMerge)
Merge two bad pixel tables.
cpl_error_code muse_image_save(muse_image *aImage, const char *aFilename)
Save the three image extensions and the FITS headers of a MUSE image to a file.
int muse_quality_dark_badpix(muse_image *aDark, double aSigmaLo, double aSigmaHi)
Find bad (especially hot) pixels (in a master dark).
cpl_error_code muse_quadrants_coords_to_raw(cpl_propertylist *aHeader, int *aX, int *aY)
Convert coordinates of a trimmed image to raw-image coordinates.
double muse_cplvector_get_adev_const(const cpl_vector *aVector, double aCenter)
Compute the average absolute deviation of a (constant) vector.
muse_image * muse_image_new(void)
Allocate memory for a new muse_image object.
int muse_quality_image_reject_using_dq(cpl_image *aData, cpl_image *aDQ, cpl_image *aStat)
Reject pixels of one or two images on a DQ image.