MUSE Pipeline Reference Manual  2.1.1
muse_quality.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  * This file is part of the MUSE Instrument Pipeline
5  * Copyright (C) 2005-2014 European Southern Observatory
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  */
21 
22 #ifdef HAVE_CONFIG_H
23 #include <config.h>
24 #endif
25 
26 /*---------------------------------------------------------------------------*
27  * Includes *
28  *---------------------------------------------------------------------------*/
29 #include <cpl.h>
30 #include <math.h>
31 #include <string.h>
32 
33 #include "muse_quality.h"
34 #include "muse_instrument.h"
35 
36 #include "muse_cplwrappers.h"
37 #include "muse_pfits.h"
38 #include "muse_quadrants.h"
39 #include "muse_tracing.h"
40 #include "muse_utils.h"
41 #include "muse_data_format_z.h"
42 
43 /*---------------------------------------------------------------------------*/
50 /*---------------------------------------------------------------------------*/
51 
54 /*---------------------------------------------------------------------------*/
78 /*---------------------------------------------------------------------------*/
79 int
80 muse_quality_dark_badpix(muse_image *aDark, double aSigmaLo, double aSigmaHi)
81 {
82  cpl_ensure(aDark, CPL_ERROR_NULL_INPUT, -1);
83  float *data = cpl_image_get_data_float(aDark->data);
84  int *dq = cpl_image_get_data_int(aDark->dq);
85  cpl_ensure(data && dq, CPL_ERROR_ILLEGAL_INPUT, -2);
86 #ifdef MUSE_DARK_PREPARE_TEST_DATA /* extract a region for (automated) testing, adapt header accordingly */
88  x->data = cpl_image_extract(aDark->data, 1780, 1922, 3000, 4112);
89  x->dq = cpl_image_extract(aDark->dq, 1780, 1922, 3000, 4112);
90  x->stat = cpl_image_extract(aDark->stat, 1780, 1922, 3000, 4112);
91  x->header = cpl_propertylist_duplicate(aDark->header);
92  cpl_propertylist_update_int(x->header, "ESO DET OUT1 NX", 269); /* [1780:2048,1922:2056] */
93  cpl_propertylist_update_int(x->header, "ESO DET OUT1 NY", 135);
94  cpl_propertylist_update_int(x->header, "ESO DET OUT2 NX", 952); /* [2049:3000,1922:2056] */
95  cpl_propertylist_update_int(x->header, "ESO DET OUT2 NY", 135);
96  cpl_propertylist_update_int(x->header, "ESO DET OUT3 NX", 269); /* [1780:2048,2057:4112] */
97  /* stays the same: cpl_propertylist_update_int(x->header, "ESO DET OUT3 NY", 2056); */
98  cpl_propertylist_update_int(x->header, "ESO DET OUT4 NX", 952); /* [2049:3000,2057:4112] */
99  /* stays the same: cpl_propertylist_update_int(x->header, "ESO DET OUT4 NY", 2056); */
100  const char *fn = "muse_test_quality_dark.fits";
101  muse_image_save(x, fn);
102  cpl_msg_debug(__func__, "saved as \"%s\"", fn);
104 #endif
105 
106  /* transfer already known bad pixels into the cpl_mask of the image, *
107  * so that they don't influence the mean/stdev values computed below */
108  int nbad = muse_quality_image_reject_using_dq(aDark->data, aDark->dq,
109  aDark->stat);
110  cpl_msg_debug(__func__, "%d incoming bad pixels", nbad);
111  cpl_binary *databpm = cpl_mask_get_data(cpl_image_get_bpm(aDark->data)),
112  *statbpm = NULL;
113  if (aDark->stat) {
114  statbpm = cpl_mask_get_data(cpl_image_get_bpm(aDark->stat));
115  }
116 
117  int nlo = 0, nhi = 0;
118  unsigned char n;
119  for (n = 1; n <= 4; n++) {
120  cpl_size *w = muse_quadrants_get_window(aDark, n);
121  cpl_stats_mode smode = CPL_STATS_MEDIAN | CPL_STATS_MEDIAN_DEV
122  | CPL_STATS_MIN | CPL_STATS_MAX;
123  cpl_stats *s = cpl_stats_new_from_image_window(aDark->data, smode,
124  w[0], w[2], w[1], w[3]);
125  double median = cpl_stats_get_median(s),
126  mdev = cpl_stats_get_median_dev(s),
127  min = cpl_stats_get_min(s),
128  max = cpl_stats_get_max(s),
129  locut = aSigmaLo > 0 ? median - aSigmaLo * mdev : min,
130  hicut = aSigmaHi > 0 ? median + aSigmaHi * mdev : max;
131  cpl_msg_debug(__func__, "quadrant %d bad pixel limits: %g ... %g +/- %g ... %g",
132  n, locut, median, mdev, hicut);
133 #if 0
134  cpl_stats_dump(s, smode, stdout);
135  fflush(stdout);
136 #endif
137  cpl_stats_delete(s);
138 
139  int i, nx = cpl_image_get_size_x(aDark->data);
140  for (i = w[0] - 1; i < w[1]; i++) {
141  int j;
142  for (j = w[2] - 1; j < w[3]; j++) {
143  cpl_size pos = i + j*nx;
144  double value = data[pos];
145  if (value < locut) {
146  dq[pos] |= EURO3D_DEADPIXEL;
147  databpm[pos] = CPL_BINARY_1;
148  if (statbpm) {
149  statbpm[pos] = CPL_BINARY_1;
150  }
151  nlo++;
152  }
153  if (value > hicut) {
154  dq[pos] |= EURO3D_HOTPIXEL;
155  databpm[pos] = CPL_BINARY_1;
156  if (statbpm) {
157  statbpm[pos] = CPL_BINARY_1;
158  }
159  nhi++;
160  }
161  } /* for j (y direction) */
162  } /* for i (x direction) */
163  cpl_free(w);
164  } /* for n (quadrants) */
165  if (nlo || aSigmaLo > 0) {
166  cpl_msg_info(__func__, "%d pixel%s lower than %.3f sigma marked as dark",
167  nlo, nlo != 1 ? "s" : "", aSigmaLo);
168  }
169  if (nhi || aSigmaHi > 0) {
170  cpl_msg_info(__func__, "%d pixel%s higher than %.3f sigma marked as hot",
171  nhi, nhi != 1 ? "s" : "", aSigmaHi);
172  }
173  return nlo + nhi;
174 } /* muse_quality_dark_badpix() */
175 
176 /*---------------------------------------------------------------------------*/
191 /*---------------------------------------------------------------------------*/
192 int
193 muse_quality_bad_columns(muse_image *aBias, double aLow, double aHigh)
194 {
195  cpl_ensure(aBias && aBias->data && aBias->dq && aBias->stat && aBias->header,
196  CPL_ERROR_NULL_INPUT, -1);
197 
198  int nx = cpl_image_get_size_x(aBias->data),
199  nlo = 0, nhi = 0; /* numbers of low and high bad pixels */
200  unsigned char n;
201  for (n = 1; n <= 4; n++) {
202  /* get the image window for this quadrant */
203  cpl_size *w = muse_quadrants_get_window(aBias, n);
204  cpl_vector *vmean = cpl_vector_new(w[1] - w[0] + 1),
205  *vstdev = cpl_vector_new(w[1] - w[0] + 1);
206  int i;
207  for (i = w[0]; i <= w[1]; i++) {
208  double mean = cpl_image_get_mean_window(aBias->data, i, w[2], i, w[3]),
209  stdev = cpl_image_get_stdev_window(aBias->data, i, w[2], i, w[3]);
210  cpl_vector_set(vmean, i - w[0], mean);
211  cpl_vector_set(vstdev, i - w[0], stdev);
212  } /* for i (all columns) */
213 
214  /* median and median deviation of the averages of all columns */
215  double cmedian = cpl_vector_get_median_const(vmean),
216  cmdev = muse_cplvector_get_adev_const(vmean, cmedian),
217  hicut = cmedian + aHigh*cmdev,
218  locut = cmedian - aLow*cmdev;
219  char *keyword = cpl_sprintf(QC_BIAS_MASTER_RON, n);
220  double ron = cpl_propertylist_get_double(aBias->header, keyword);
221  cpl_free(keyword);
222  cpl_msg_debug(__func__, "quadrant %1d: mean %f+/-%f(%f); valid range "
223  "%f...(%f+/-%f)...%f RON=%f", n, cpl_vector_get_mean(vmean),
224  cpl_vector_get_stdev(vmean), cpl_vector_get_mean(vstdev),
225  locut, cmedian, cmdev, hicut, ron);
226 
227  int j;
228  float *data = cpl_image_get_data_float(aBias->data);
229  int *dq = cpl_image_get_data_int(aBias->dq);
230  for (i = w[0]; i <= w[1]; i++) {
231  double cmean = cpl_vector_get(vmean, i - w[0]),
232  cstdev = cpl_vector_get(vstdev, i - w[0]);
233  if (cmean > hicut && cstdev > ron) {
234  cpl_msg_debug(__func__, "hot column %d (%f+/-%f)", i, cmean, cstdev);
235  /* now find first high pixel in this column */
236  int jfirst = w[2], jlast = w[3];
237  for (j = w[2]; j <= w[3]; j++) {
238  if (data[(i-1) + (j-1)*nx] > hicut) {
239  jfirst = j;
240  break;
241  }
242  } /* for j */
243  /* find last high pixel in this column */
244  for (j = w[3]; j >= w[2]; j--) {
245  if (data[(i-1) + (j-1)*nx] > hicut) {
246  jlast = j;
247  break;
248  }
249  } /* for j */
250  /* everything in between is untrustworthy, mark as bad */
251  for (j = jfirst; j <= jlast; j++) {
252  dq[(i-1) + (j-1)*nx] |= EURO3D_DEADPIXEL;
253  nhi++;
254  } /* for j */
255  } else if (cmean < locut) {
256  cpl_msg_debug(__func__, "dark column %d (%f+/-%f)", i, cmean, cstdev);
257  /* now find first high pixel in this column */
258  int jfirst = w[2], jlast = w[3];
259  for (j = w[2]; j <= w[3]; j++) {
260  if (data[(i-1) + (j-1)*nx] < locut) {
261  jfirst = j;
262  break;
263  }
264  } /* for j */
265  /* find last high pixel in this column */
266  for (j = w[3]; j >= w[2]; j--) {
267  if (data[(i-1) + (j-1)*nx] < locut) {
268  jlast = j;
269  break;
270  }
271  } /* for j */
272  /* everything in between is untrustworthy, mark as bad */
273  for (j = jfirst; j <= jlast; j++) {
274  dq[(i-1) + (j-1)*nx] |= EURO3D_DEADPIXEL;
275  nhi++;
276  } /* for j */
277  } /* if < locut */
278  } /* for i (all columns) */
279 
280  cpl_vector_delete(vmean);
281  cpl_vector_delete(vstdev);
282  cpl_free(w);
283  } /* for n (quadrants) */
284 
285  cpl_msg_info(__func__, "%d low and %d high pixels found", nlo, nhi);
286 
287  return nlo + nhi;
288 } /* muse_quality_bad_columns() */
289 
290 /*---------------------------------------------------------------------------*/
313 /*---------------------------------------------------------------------------*/
314 int
315 muse_quality_flat_badpix(muse_image *aFlat, cpl_table *aTrace,
316  double aSigmaLo, double aSigmaHi)
317 {
318  cpl_ensure(aFlat && aFlat->data && aFlat->dq && aFlat->stat && aTrace,
319  CPL_ERROR_NULL_INPUT, -1);
320 
321  cpl_msg_info(__func__, "Marking dark/bright pixels using sigmas %.2f/%.2f",
322  aSigmaLo, aSigmaHi);
323  int ndark = 0, nhot = 0, nlowqe = 0,
324  nx = cpl_image_get_size_x(aFlat->data),
325  ny = cpl_image_get_size_y(aFlat->data);
326  float *data = cpl_image_get_data_float(aFlat->data);
327  int *dq = cpl_image_get_data_int(aFlat->dq);
328 
329  /* get mean count level of the flat-field, *
330  * to decide for or against low QE pixels */
331  double mean = cpl_image_get_mean(aFlat->data);
332 #if 0
333  cpl_msg_debug(__func__, "mean flat value: %f", mean);
334 #endif
335  int nslice;
336  for (nslice = 1; nslice <= kMuseSlicesPerCCD; nslice++) {
337  /* get the tracing polynomials for this slice */
338  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
339  nslice);
340  if (!ptrace) {
341  cpl_msg_warning(__func__, "slice %2d: tracing polynomials missing!",
342  nslice);
343  continue;
344  }
345 
346  /* within each slice, loop from bottom to top */
347  int j;
348  for (j = 0; j < ny; j++) {
349  /* determine the slice edges for this vertical position so that we *
350  * can loop over those pixels, excluding slice edges for the moment */
351  cpl_errorstate prestate = cpl_errorstate_get();
352  double x1 = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT], j+1, NULL),
353  x2 = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT], j+1, NULL);
354  if (!cpl_errorstate_is_equal(prestate) || !isnormal(x1) || !isnormal(x2)
355  || x1 < 1 || x2 > nx || x1 > x2) {
356  cpl_msg_warning(__func__, "slice %2d: faulty polynomial detected at "
357  "y=%d (borders: %f ... %f): %s", nslice, j+1, x1, x2,
358  cpl_error_get_message());
359  j = ny; /* skip the rest of this slice */
360  continue;
361  }
362  /* integer limits where we can be sure to get only well- *
363  * illuminated pixels within the width of the slice */
364  int mleft = ceil(x1), /* FITS coords, starting at 1! */
365  mright = floor(x2);
366 
367 #if 0
368  double xc = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER], j+1, NULL);
369  cpl_msg_debug(__func__, "%02d %04d: %04d/%.3f...%.3f...%04d/%.3f "
370  "(%02d/%.3f wide)", nslice, j+1, mleft,x1, xc, mright,x2,
371  mright - mleft, x2 - x1);
372 #endif
373  unsigned statmask = CPL_STATS_MIN | CPL_STATS_MAX
374  | CPL_STATS_MEAN | CPL_STATS_STDEV
375  | CPL_STATS_MEDIAN | CPL_STATS_MEDIAN_DEV;
376  cpl_stats *stats = cpl_stats_new_from_image_window(aFlat->data, statmask,
377  mleft, j+1,
378  mright, j+1);
379 #if 0
380  cpl_stats_dump(stats, statmask, stdout);
381  fflush(stdout);
382 #endif
383  double median = cpl_stats_get_median(stats),
384  llim = median - aSigmaLo * cpl_stats_get_median_dev(stats),
385  hlim = median + aSigmaHi * cpl_stats_get_median_dev(stats);
386  cpl_stats_delete(stats);
387  /* limit <= 0 doesn't make sense for master flats, set *
388  * to value that would also work for normalized image */
389  llim = llim <= 0 ? 1.e-4 : llim;
390 #if 0
391  cpl_msg_debug(__func__, "limits: %f...%f (sigmas %.2f/%.2f)", llim, hlim,
392  aSigmaLo, aSigmaHi);
393 #endif
394 
395  /* except contiguous regions of low flux at the edges, but *
396  * at most kMuseSliceMaxEdgeWidth pixels on each side */
397  int i, i1 = mleft - 1, i2 = mright - 1; /* indices, starting at 0! */
398  for (i = mleft - 1; i < mleft + kMuseSliceMaxEdgeWidth; i++) {
399  if (data[i + j*nx] > llim) {
400  i1 = i;
401  break;
402  }
403  } /* for i (horizontal pixels) */
404  for (i = mright - 1; i > mright - kMuseSliceMaxEdgeWidth; i--) {
405  if (data[i + j*nx] > llim) {
406  i2 = i;
407  break;
408  }
409  } /* for i (horizontal pixels) */
410 
411  /* now loop over all "normal" pixels of this slice *
412  * horizontally, excluding the pixels at the very edge */
413  for (i = i1; i <= i2; i++) {
414  if (data[i + j*nx] < llim) {
415  dq[i + j*nx] |= EURO3D_DARKPIXEL;
416  if (data[i + j*nx] < 0.2 * mean) {
417  /* is a low QE pixel according to the Euro3D specifications, *
418  * as it has less than 20% of the average flat-field level */
419  dq[i + j*nx] |= EURO3D_LOWQEPIXEL;
420  nlowqe++;
421  }
422 #if 0
423  cpl_msg_debug(__func__, "%04d,%04d is DARK%s: %f", i+1, j+1,
424  data[i + j*nx] < 0.2 * mean ? " and low QE" : "",
425  data[i + j*nx]);
426 #endif
427  ndark++;
428  } else if (data[i + j*nx] > hlim) {
429  dq[i + j*nx] |= EURO3D_HOTPIXEL;
430 #if 0
431  cpl_msg_debug(__func__, "%04d,%04d is HOT: %f", i+1, j+1, data[i + j*nx]);
432 #endif
433  nhot++;
434  }
435  } /* for i (horizontal pixels) */
436  } /* for j (vertical direction) */
437 
438  muse_trace_polys_delete(ptrace);
439  } /* for nslice */
440 
441  /* search and mark non-positive pixels */
442  int i, nnonpos = 0;
443  for (i = 0; i < nx; i++) {
444  int j;
445  for (j = 0; j < ny; j++) {
446  if (data[i + j*nx] <= 0) {
447  dq[i + j*nx] |= EURO3D_BADOTHER;
448  nnonpos++;
449  }
450  } /* for j (vertical direction) */
451  } /* for i (horizontal pixels) */
452 
453  cpl_msg_info(__func__, "Found %d dark (%d of them are also low QE), %d hot, "
454  "and %d non-positive pixels", ndark, nlowqe, nhot, nnonpos);
455  return ndark + nhot;
456 } /* muse_quality_flat_badpix() */
457 
458 /*---------------------------------------------------------------------------*/
471 /*---------------------------------------------------------------------------*/
472 int
474 {
475  cpl_ensure(aImage && aImage->data && aImage->dq, CPL_ERROR_NULL_INPUT, -1);
476 
477  float *data = cpl_image_get_data_float(aImage->data);
478  int *dq = cpl_image_get_data_int(aImage->dq);
479  int i, j, nsaturated = 0,
480  nx = cpl_image_get_size_x(aImage->data),
481  ny = cpl_image_get_size_y(aImage->data);
482  for (i = 0; i < nx; i++) {
483  for (j = 0; j < ny; j++) {
484  if (data[i + j*nx] > kMuseSaturationLimit ||
485  data[i + j*nx] < FLT_EPSILON) {
486  dq[i + j*nx] |= EURO3D_SATURATED;
487  nsaturated++;
488  }
489  } /* for j */
490  } /* for i */
491 
492  return nsaturated;
493 } /* muse_quality_set_saturated() */
494 
495 /*---------------------------------------------------------------------------*/
509 /*---------------------------------------------------------------------------*/
510 cpl_table *
511 muse_quality_convert_dq(cpl_image *aDQ)
512 {
513  cpl_ensure(aDQ, CPL_ERROR_NULL_INPUT, NULL);
514  int nx = cpl_image_get_size_x(aDQ),
515  ny = cpl_image_get_size_y(aDQ);
516  const int *dq = cpl_image_get_data_int_const(aDQ);
517  int i, j, nbad = 0;
518  for (i = 0; i < nx; i++) {
519  for (j = 0; j < ny; j++) {
520  if (dq[i + j*nx] != EURO3D_GOODPIXEL) {
521  nbad++;
522  }
523  } /* for j (columns) */
524  } /* for i (rows) */
525  cpl_table *table = muse_cpltable_new(muse_badpix_table_def, nbad);
526  /* if no bad pixels were found, return the empty table */
527  if (!nbad) {
528  return table;
529  }
530 
531  nbad = 0;
532  for (i = 0; i < nx; i++) {
533  for (j = 0; j < ny; j++) {
534  if (dq[i + j*nx] == EURO3D_GOODPIXEL) {
535  continue;
536  }
537  int xbad = i + 1,
538  ybad = j + 1;
539  /* transform to raw positions */
540  muse_quadrants_coords_to_raw(NULL, &xbad, &ybad);
541  cpl_table_set_int(table, MUSE_BADPIX_X, nbad, xbad);
542  cpl_table_set_int(table, MUSE_BADPIX_Y, nbad, ybad);
543  cpl_table_set_int(table, MUSE_BADPIX_DQ, nbad, dq[i + j*nx]);
544  /* no way to set MUSE_BADPIX_VALUE from the data in the DQ extension */
545  nbad++;
546  } /* for j (columns) */
547  } /* for i (rows) */
548  return table;
549 } /* muse_quality_convert_dq() */
550 
551 /*---------------------------------------------------------------------------*/
565 /*---------------------------------------------------------------------------*/
566 cpl_error_code
567 muse_quality_merge_badpix(cpl_table *aTable, const cpl_table *aToMerge)
568 {
569  cpl_ensure_code(aTable && aToMerge, CPL_ERROR_NULL_INPUT);
570  cpl_error_code rc = cpl_table_insert(aTable, aToMerge,
571  cpl_table_get_nrow(aTable));
572  cpl_ensure_code(rc == CPL_ERROR_NONE, rc);
573 
574  /* sort the merged table, to ease search of duplicate pixel entries */
575  cpl_propertylist *order = cpl_propertylist_new();
576  cpl_propertylist_append_bool(order, MUSE_BADPIX_X, FALSE);
577  cpl_propertylist_append_bool(order, MUSE_BADPIX_Y, FALSE);
578  cpl_table_sort(aTable, order);
579  cpl_propertylist_delete(order);
580 
581  /* now select rows for duplicate pixel coordinate entries, merge their DQ */
582  cpl_table_unselect_all(aTable);
583  const int *x = cpl_table_get_data_int_const(aTable, MUSE_BADPIX_X),
584  *y = cpl_table_get_data_int_const(aTable, MUSE_BADPIX_Y);
585  int *dq = cpl_table_get_data_int(aTable, MUSE_BADPIX_DQ);
586  float *value = cpl_table_get_data_float(aTable, MUSE_BADPIX_VALUE);
587  int i, nrow = cpl_table_get_nrow(aTable);
588  for (i = 1; i < nrow; i++) {
589  if (x[i] == x[i-1] && y[i] == y[i-1]) {
590  dq[i-1] |= dq[i]; /* merge Euro3D status flags */
591  if (value) {
592  /* as we know nothing else, use the maximum for the value */
593  value[i-1] = fmax(value[i-1], value[i]);
594  }
595  cpl_table_select_row(aTable, i);
596  }
597  } /* for i (all table rows) */
598  rc = cpl_table_erase_selected(aTable);
599 
600  return rc;
601 } /* muse_quality_merge_badpix() */
602 
603 /*---------------------------------------------------------------------------*/
626 /*---------------------------------------------------------------------------*/
627 int
628 muse_quality_image_reject_using_dq(cpl_image *aData, cpl_image *aDQ,
629  cpl_image *aStat)
630 {
631  cpl_ensure(aData && aDQ, CPL_ERROR_NULL_INPUT, -1);
632  int nx = cpl_image_get_size_x(aData),
633  ny = cpl_image_get_size_y(aData);
634  cpl_ensure(nx == cpl_image_get_size_x(aDQ) &&
635  ny == cpl_image_get_size_y(aDQ), CPL_ERROR_INCOMPATIBLE_INPUT, -2);
636  if (aStat) {
637  cpl_ensure(nx == cpl_image_get_size_x(aStat) &&
638  ny == cpl_image_get_size_y(aStat), CPL_ERROR_INCOMPATIBLE_INPUT,
639  -2);
640  }
641  const int *dq = cpl_image_get_data_int_const(aDQ);
642  if (!dq) {
643  return -3;
644  }
645 #if 0
646  double cputime = cpl_test_get_cputime(),
647  walltime = cpl_test_get_walltime();
648 #endif
649  cpl_binary *bpm = cpl_mask_get_data(cpl_image_get_bpm(aData)),
650  *statbpm = NULL;
651  if (aStat) {
652  statbpm = cpl_mask_get_data(cpl_image_get_bpm(aStat));
653  }
654  int i, j, nbad = 0;
655  for (i = 0; i < nx; i++) {
656  for (j = 0; j < ny; j++) {
657  if (dq[i + j*nx] == EURO3D_GOODPIXEL) {
658  continue;
659  }
660  nbad++;
661  bpm[i + j*nx] = CPL_BINARY_1;
662  if (aStat) {
663  statbpm[i + j*nx] = CPL_BINARY_1;
664  }
665  } /* for j (columns) */
666  } /* for i (rows) */
667 #if 0
668  double cputime2 = cpl_test_get_cputime(),
669  walltime2 = cpl_test_get_walltime();
670  cpl_msg_debug(__func__, "reject_using_dq times new: cpu %f wall %f", cputime2 - cputime,
671  walltime2 - walltime);
672 #endif
673  return nbad;
674 } /* muse_quality_image_reject_using_dq() */
675 
676 /*---------------------------------------------------------------------------*/
696 /*---------------------------------------------------------------------------*/
697 cpl_table *
698 muse_quality_merge_badpix_from_file(const cpl_table *aTable, const char *aInFile,
699  const char *aExtname, int *aExt)
700 {
701  cpl_ensure(aTable && aInFile, CPL_ERROR_NULL_INPUT, NULL);
702 
703  int extension = cpl_fits_find_extension(aInFile, aExtname);
704  cpl_table *table = NULL;
705  if (extension < 1) {
706  if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
707  printf("Input table \"%s\" does not contain a table for EXTNAME=\"%s\""
708  " yet\n", aInFile, aExtname);
709  }
710  } else {
711  table = cpl_table_load(aInFile, extension, 1);
712  if (!table) {
713  printf("WARNING: could not load BADPIX_TABLE from EXTNAME=\"%s\" from "
714  "table \"%s\" (the headers say that it should be extension %d)!\n",
715  aExtname, aInFile, extension);
716  }
717  } /* else */
718  cpl_ensure(table, CPL_ERROR_DATA_NOT_FOUND, NULL);
719 
720  if (aExt) {
721  *aExt = extension;
722  }
723 
724  cpl_size nold = cpl_table_get_nrow(table);
725  cpl_error_code rc = muse_quality_merge_badpix(table, aTable);
726  if (rc != CPL_ERROR_NONE) {
727  printf("WARNING: Merging input and new table failed: %s\n",
728  cpl_error_get_message());
729  printf("Table still has %"CPL_SIZE_FORMAT" bad pixel%s\n", nold,
730  nold != 1 ? "s" : "");
731  } else {
732  cpl_size nbad = cpl_table_get_nrow(table);
733  printf("Merged %"CPL_SIZE_FORMAT" of %"CPL_SIZE_FORMAT" bad pixel%s into "
734  "the input table (now %"CPL_SIZE_FORMAT" entries)\n", nbad - nold,
735  cpl_table_get_nrow(table), (nbad - nold) != 1 ? "s" : "", nbad);
736  }
737 
738  return table;
739 } /* muse_quality_merge_badpix_from_file() */
740 
741 /*---------------------------------------------------------------------------*/
762 /*---------------------------------------------------------------------------*/
763 cpl_error_code
764 muse_quality_copy_badpix_table(const char *aInFile, const char *aOutFile,
765  int aExtension, const cpl_table *aTable)
766 {
767  cpl_ensure_code(aInFile && aOutFile && aTable, CPL_ERROR_NULL_INPUT);
768 
769  cpl_errorstate state = cpl_errorstate_get();
770  cpl_error_code rc = CPL_ERROR_NONE;
771  cpl_size nextensions = cpl_fits_count_extensions(aInFile);
772  if (!cpl_errorstate_is_equal(state)) {
773  rc = cpl_error_get_code();
774  }
775  if (nextensions > 0) {
776  printf("Saving primary header and %"CPL_SIZE_FORMAT" extensions to \"%s\"\n",
777  nextensions, aOutFile);
778  }
779  cpl_size i;
780  for (i = 0; i <= nextensions; i++) {
781  cpl_propertylist *extheader = cpl_propertylist_load(aInFile, i);
782  if (i == 0) {
783  /* just save the header for the primary data unit but update *
784  * the pipeline filename in the header to the one we write here */
785  cpl_propertylist_update_string(extheader, "PIPEFILE", aOutFile);
786  cpl_propertylist_set_comment(extheader, "PIPEFILE",
787  "pretend to be a pipeline output file");
788  cpl_propertylist_save(extheader, aOutFile, CPL_IO_CREATE);
789  if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
790  printf("Saved primary header to \"%s\"\n", aOutFile);
791  }
792  cpl_propertylist_delete(extheader);
793  continue;
794  }
795  if (aTable && i == aExtension) {
796  unsigned char nifu = muse_utils_get_ifu(extheader);
797  printf("Saving merged table of IFU %2hhu to extension %"CPL_SIZE_FORMAT
798  "\n", nifu, i);
799  cpl_table_save(aTable, NULL, extheader, aOutFile, CPL_IO_EXTEND);
800  cpl_propertylist_delete(extheader);
801  continue;
802  }
803 
804  cpl_table *tab = NULL;
805  const char *xtension = cpl_propertylist_get_string(extheader, "XTENSION");
806  if (xtension && strncmp(xtension, "BINTABLE", 8)) {
807  if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
808  printf("WARNING: Not a binary table extension, skipping data section "
809  "(extension %"CPL_SIZE_FORMAT")", i);
810  }
811  cpl_propertylist_save(extheader, aOutFile, CPL_IO_EXTEND);
812  } else {
813  tab = cpl_table_load(aInFile, i, 1);
814  cpl_table_save(tab, NULL, extheader, aOutFile, CPL_IO_EXTEND);
815  if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
816  printf("Saved table extension %"CPL_SIZE_FORMAT" to \"%s\"\n", i,
817  aOutFile);
818  }
819  }
820  cpl_table_delete(tab);
821  cpl_propertylist_delete(extheader);
822  } /* for i (all FITS extensions) */
823 
824  return rc;
825 } /* muse_quality_copy_badpix_table() */
826 
cpl_polynomial ** muse_trace_table_get_polys_for_slice(const cpl_table *aTable, const unsigned short aSlice)
construct polynomial from the trace table entry for the given slice
void muse_image_delete(muse_image *aImage)
Deallocate memory associated to a muse_image object.
Definition: muse_image.c:85
int muse_quality_flat_badpix(muse_image *aFlat, cpl_table *aTrace, double aSigmaLo, double aSigmaHi)
Find bad (especially dark) pixels (in a master flat).
Definition: muse_quality.c:315
cpl_size * muse_quadrants_get_window(const muse_image *aImage, unsigned char aQuadrant)
Determine the data window of a given quadrant on the CCD.
cpl_table * muse_quality_convert_dq(cpl_image *aDQ)
Convert a data quality (DQ) image extension to a bad pixel table.
Definition: muse_quality.c:511
unsigned char muse_utils_get_ifu(const cpl_propertylist *aHeaders)
Find out the IFU/channel from which this header originated.
Definition: muse_utils.c:98
cpl_image * data
the data extension
Definition: muse_image.h:46
int muse_quality_bad_columns(muse_image *aBias, double aLow, double aHigh)
Find bad columns (in a master bias).
Definition: muse_quality.c:193
cpl_error_code muse_quality_copy_badpix_table(const char *aInFile, const char *aOutFile, int aExtension, const cpl_table *aTable)
Copy bad pixel table on disk, replacing the table in one extension.
Definition: muse_quality.c:764
cpl_image * stat
the statistics extension
Definition: muse_image.h:64
cpl_table * muse_quality_merge_badpix_from_file(const cpl_table *aTable, const char *aInFile, const char *aExtname, int *aExt)
Merge bad pixel table in memory with table from file on disk.
Definition: muse_quality.c:698
Structure definition of MUSE three extension FITS file.
Definition: muse_image.h:40
cpl_propertylist * header
the FITS header
Definition: muse_image.h:72
void muse_trace_polys_delete(cpl_polynomial *aPolys[])
Delete the multi-polynomial array created in relation to tracing.
cpl_image * dq
the data quality extension
Definition: muse_image.h:56
cpl_table * muse_cpltable_new(const muse_cpltable_def *aDef, cpl_size aLength)
Create an empty table according to the specified definition.
int muse_quality_set_saturated(muse_image *aImage)
Set all pixels above the saturation limit in the bad pixel image.
Definition: muse_quality.c:473
const muse_cpltable_def muse_badpix_table_def[]
cpl_error_code muse_quality_merge_badpix(cpl_table *aTable, const cpl_table *aToMerge)
Merge two bad pixel tables.
Definition: muse_quality.c:567
cpl_error_code muse_image_save(muse_image *aImage, const char *aFilename)
Save the three image extensions and the FITS headers of a MUSE image to a file.
Definition: muse_image.c:405
int muse_quality_dark_badpix(muse_image *aDark, double aSigmaLo, double aSigmaHi)
Find bad (especially hot) pixels (in a master dark).
Definition: muse_quality.c:80
cpl_error_code muse_quadrants_coords_to_raw(cpl_propertylist *aHeader, int *aX, int *aY)
Convert coordinates of a trimmed image to raw-image coordinates.
double muse_cplvector_get_adev_const(const cpl_vector *aVector, double aCenter)
Compute the average absolute deviation of a (constant) vector.
muse_image * muse_image_new(void)
Allocate memory for a new muse_image object.
Definition: muse_image.c:66
int muse_quality_image_reject_using_dq(cpl_image *aData, cpl_image *aDQ, cpl_image *aStat)
Reject pixels of one or two images on a DQ image.
Definition: muse_quality.c:628