MUSE Pipeline Reference Manual  2.1.1
muse_lingain.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  * This file is part of the MUSE Instrument Pipeline
5  * Copyright (C) 2015-2016 European Southern Observatory
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  */
21 
22 #ifdef HAVE_CONFIG_H
23 #include <config.h>
24 #endif
25 
26 /*---------------------------------------------------------------------------*
27  * Includes *
28  *---------------------------------------------------------------------------*/
29 #include <string.h>
30 #include <math.h>
31 #include <cpl.h>
32 #include <muse.h>
33 #include "muse_lingain_z.h"
34 
35 
36 typedef void (*muse_free_func)(void *);
37 
38 struct muse_gain_fit_t
39 {
40  double gain;
41  double rms;
42 };
43 
44 typedef struct muse_gain_fit_t muse_gain_fit_t;
45 
46 struct muse_linearity_fit_t
47 {
48  cpl_array *coefficients;
49  double range[2];
50  double rms;
51 };
52 
53 typedef struct muse_linearity_fit_t muse_linearity_fit_t;
54 
55 /*---------------------------------------------------------------------------*
56  * Functions code *
57  *---------------------------------------------------------------------------*/
58 
59 /* XXX: Make this a public function. Could go to muse_utils? */
60 static void
61 muse_vfree(void **array, cpl_size size, muse_free_func deallocator)
62 {
63  if (array) {
64  cpl_size idx;
65  for (idx = 0; idx < size; ++idx) {
66  if (deallocator) {
67  deallocator(array[idx]);
68  }
69  }
70  cpl_free(array);
71  }
72  return;
73 }
74 
75 static void
76 muse_linearity_fit_clear(muse_linearity_fit_t *aFit)
77 {
78  if (aFit) {
79  if (aFit->coefficients) {
80  cpl_array_delete(aFit->coefficients);
81  }
82  }
83  return;
84 }
85 
86 /*---------------------------------------------------------------------------*/
97 /*---------------------------------------------------------------------------*/
98 static cpl_boolean
99 muse_lingain_validate_parameters(muse_lingain_params_t *aParams)
100 {
101  if (aParams->ybox <= 0) {
102  cpl_msg_error(__func__, "Invalid measurement window size: window must "
103  "be larger than 0!");
104  return CPL_FALSE;
105  }
106 
107  if (aParams->xgap < 0) {
108  cpl_msg_error(__func__, "Invalid tracing edge offset: offset is "
109  "less than 0!");
110  return CPL_FALSE;
111  }
112 
113  if (aParams->xborder < 0) {
114  cpl_msg_error(__func__, "Invalid offset from detector edge: offset is "
115  "less than 0!");
116  return CPL_FALSE;
117  }
118 
119  if (aParams->order < 0) {
120  cpl_msg_error(__func__, "Invalid polynomial fit order: polynomial "
121  "degree is less than 0!");
122  return CPL_FALSE;
123  }
124 
125  if (aParams->toffset < 0.) {
126  cpl_msg_error(__func__, "Invalid exposure time offset: offset is less "
127  "than 0.!");
128  return CPL_FALSE;
129  }
130 
131  if (aParams->sigma < 0.) {
132  cpl_msg_error(__func__, "Invalid sigma clipping threshold used for "
133  "signal value cleaning: threshold is less than 0.!");
134  return CPL_FALSE;
135  }
136 
137  if (aParams->signalmin < 0.) {
138  cpl_msg_error(__func__, "Invalid signal level grid: minimum signal is "
139  "less than 0.!");
140  return CPL_FALSE;
141  }
142 
143  if (aParams->signalbin <= 0.) {
144  cpl_msg_error(__func__, "Invalid signal level grid: signal bin size "
145  "must larger than 0.!");
146  return CPL_FALSE;
147  }
148 
149  if (aParams->signalmax < aParams->signalmin + aParams->signalbin) {
150  cpl_msg_error(__func__, "Invalid signal level grid: maximum signal is "
151  "too small for chosen minimum value and bin size!");
152  return CPL_FALSE;
153  }
154 
155  if (aParams->gainlimit < 0.) {
156  cpl_msg_error(__func__, "Invalid minimum signal used for gain fit: "
157  "minimum signal is less than 0.!");
158  return CPL_FALSE;
159  }
160 
161  if (aParams->gainsigma < 0.) {
162  cpl_msg_error(__func__, "Invalid sigma clipping threshold used for "
163  "gain value cleaning: threshold is less than 0.!");
164  return CPL_FALSE;
165  }
166 
167  if (aParams->ctsmin < 0.) {
168  cpl_msg_error(__func__, "Invalid signal level grid used for "
169  "non-linearity analysis: minimum signal is "
170  "less than 0.!");
171  return CPL_FALSE;
172  }
173 
174  if (aParams->ctsbin <= 0.) {
175  cpl_msg_error(__func__, "Invalid signal level grid used for "
176  "non-linearity analysis: signal bin size must larger "
177  "than 0.!");
178  return CPL_FALSE;
179  }
180 
181  if (aParams->ctsmax < aParams->ctsmin + aParams->ctsbin) {
182  cpl_msg_error(__func__, "Invalid signal level grid used for "
183  "non-linearity analysis: maximum signal is "
184  "too small for chosen minimum value and bin size!");
185  return CPL_FALSE;
186  }
187 
188  if (aParams->linearmin < 0.) {
189  cpl_msg_error(__func__, "Invalid signal range used for residual "
190  "non-linearity fit: minimum signal is less than 0.!");
191  return CPL_FALSE;
192  }
193 
194  if (aParams->linearmax <= aParams->linearmin) {
195  cpl_msg_error(__func__, "Invalid signal range used for residual "
196  "non-linearity fit: maximum signal is too small for "
197  "the cosen minimum value!");
198  return CPL_FALSE;
199  }
200 
201  return CPL_TRUE;
202 }
203 
204 /*---------------------------------------------------------------------------*/
211 /*---------------------------------------------------------------------------*/
212 static muse_imagelist *
213 muse_lingain_load_images(muse_processing *aProcessing, unsigned short aIFU)
214 {
215  muse_imagelist *images;
216 
217  /* Look for the master bias of the current IFU and determine the *
218  * basic processing parameters so that a consistent set of parameters *
219  * is used here. */
220  cpl_frame *bias = muse_frameset_find_master(aProcessing->inframes,
221  MUSE_TAG_MASTER_BIAS, aIFU);
222 
223  /* Get primary header containing the processing options used during *
224  * the master bias creation. */
225  cpl_propertylist *properties = cpl_propertylist_load(cpl_frame_get_filename(bias), 0);
226  cpl_frame_delete(bias);
228  cpl_propertylist_delete(properties);
229 
230  images = muse_basicproc_load(aProcessing, aIFU, bpars);
232 
233  return images;
234 }
235 
236 /*---------------------------------------------------------------------------*/
246 /*---------------------------------------------------------------------------*/
247 static cpl_table *
248 muse_lingain_sort_images(const muse_imagelist *aList)
249 {
250  cpl_ensure(aList, CPL_ERROR_NULL_INPUT, NULL);
251 
252  cpl_errorstate status = cpl_errorstate_get();
253 
254  cpl_table *exposures = cpl_table_new(aList->size);
255 
256  cpl_table_new_column(exposures, "INDEX", CPL_TYPE_INT);
257  cpl_table_new_column(exposures, "TYPE", CPL_TYPE_STRING);
258  cpl_table_new_column(exposures, "MJDOBS", CPL_TYPE_DOUBLE);
259  cpl_table_new_column(exposures, "EXPTIME", CPL_TYPE_DOUBLE);
260 
261  unsigned int iexposure;
262  for (iexposure = 0; iexposure < aList->size; ++iexposure) {
263  cpl_propertylist *header = muse_imagelist_get((muse_imagelist *)aList, iexposure)->header;
264  cpl_table_set_int(exposures, "INDEX", iexposure, iexposure);
265  if (strncmp(muse_pfits_get_dpr_type(header), "BIAS,DETCHECK", 14) == 0) {
266  cpl_table_set_string(exposures, "TYPE", iexposure,
267  MUSE_TAG_LINEARITY_BIAS);
268  } else {
269  cpl_table_set_string(exposures, "TYPE", iexposure,
270  MUSE_TAG_LINEARITY_FLAT);
271  }
272  cpl_table_set_double(exposures, "MJDOBS", iexposure,
273  muse_pfits_get_mjdobs(header));
274  cpl_table_set_double(exposures, "EXPTIME", iexposure,
275  muse_pfits_get_exptime(header));
276  }
277 
278  /* This assumes that the bias frames always have an exposure time *
279  * which is smaller than that of the shortest flat field exposure *
280  * and thus are sorted into the table as the first entries. */
281  cpl_propertylist *order = cpl_propertylist_new();
282  cpl_propertylist_append_bool(order, "EXPTIME", CPL_FALSE);
283  cpl_propertylist_append_bool(order, "MJDOBS", CPL_FALSE);
284 
285  cpl_table_sort(exposures, order);
286  cpl_propertylist_delete(order);
287 
288  if (!cpl_errorstate_is_equal(status)) {
289  cpl_table_delete(exposures);
290  return NULL;
291  }
292 
293  /* Validate exposure table: there should be pairs of the same type and *
294  * with the same exposure time */
295  cpl_boolean invalid = CPL_FALSE;
296  cpl_size npairs = (cpl_size)(aList->size / 2);
297  cpl_size ipair;
298  for (ipair = 0; ipair < npairs; ++ipair) {
299  register cpl_size jpair = 2 * ipair;
300  register cpl_size kpair = jpair + 1;
301  if (strcmp(cpl_table_get_string(exposures, "TYPE", jpair),
302  cpl_table_get_string(exposures, "TYPE", kpair)) != 0) {
303  cpl_msg_error(__func__, "Invalid pair of exposures: Type of exposure "
304  "does not match: Got '%s', expected '%s'!",
305  cpl_table_get_string(exposures, "TYPE", jpair),
306  cpl_table_get_string(exposures, "TYPE", kpair));
307  invalid = CPL_TRUE;
308  break;
309  }
310  if ((cpl_table_get_double(exposures, "EXPTIME", jpair, NULL) -
311  cpl_table_get_double(exposures, "EXPTIME", kpair, NULL)) > 0.1) {
312  cpl_msg_error(__func__, "Invalid pair of exposures: exposure times "
313  "do not match: Got '%.4f', expected '%.4f'!",
314  cpl_table_get_double(exposures, "EXPTIME", jpair, NULL),
315  cpl_table_get_double(exposures, "EXPTIME", kpair, NULL));
316  invalid = CPL_TRUE;
317  break;
318  }
319  }
320 
321  if (invalid) {
322  cpl_table_delete(exposures);
323  cpl_error_set(__func__, CPL_ERROR_ILLEGAL_OUTPUT);
324  return NULL;
325  }
326 
327  /* If there is an orphan exposure left, remove it from the table */
328  cpl_size nexposures = 2 * npairs;
329  if (aList->size > nexposures) {
330  cpl_table_erase_window(exposures, nexposures, aList->size - nexposures);
331  }
332 
333  return exposures;
334 }
335 
336 /*---------------------------------------------------------------------------*/
346 /*---------------------------------------------------------------------------*/
347 static cpl_table *
348 muse_lingain_create_windowlist(const cpl_table *aTrace,
349  const cpl_table *aExposureList,
350  const muse_imagelist *aList,
351  muse_lingain_params_t *aParams)
352 {
353  cpl_ensure(aTrace && aExposureList && aList && aParams,
354  CPL_ERROR_NULL_INPUT, NULL);
355 
356  cpl_table *windowlist = cpl_table_new(0);
357  cpl_table_new_column(windowlist, "Quadrant", CPL_TYPE_INT);
358  cpl_table_new_column(windowlist, "SliceNo", CPL_TYPE_INT);
359  cpl_table_new_column(windowlist, "Xmin", CPL_TYPE_INT);
360  cpl_table_new_column(windowlist, "Ymin", CPL_TYPE_INT);
361  cpl_table_new_column(windowlist, "Xmax", CPL_TYPE_INT);
362  cpl_table_new_column(windowlist, "Ymax", CPL_TYPE_INT);
363 
364  /* Get quadrant definitions from the first image in the exposure list */
365  /* XXX: Verify that all exposures have the same quadrant definitions? */
366 
367  int idx = cpl_table_get_int(aExposureList, "INDEX", 0, NULL);
368 
369  cpl_errorstate status = cpl_errorstate_get();
370 
371  muse_image *image = muse_imagelist_get((muse_imagelist *)aList, idx);
372  cpl_size firstrow = 0;
373 
374  unsigned char iquadrant;
375  for (iquadrant = 1; iquadrant <= 4; ++iquadrant) {
376  cpl_size *qwindow = muse_quadrants_get_window(image, iquadrant);
377 
378  unsigned short islice;
379  for (islice = 1; islice <= kMuseSlicesPerCCD; ++islice) {
380  cpl_errorstate _status = cpl_errorstate_get();
381  cpl_polynomial **traces = muse_trace_table_get_polys_for_slice(aTrace,
382  islice);
383  /* If no tracing polynomial is found for the current slice index *
384  * issue a warning and continue with the next slice. */
385  if (!traces) {
386  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: tracing polynomials "
387  "missing!", islice, aParams->nifu);
388  muse_trace_polys_delete(traces);
389  cpl_errorstate_set(_status);
390  continue;
391  }
392  cpl_size torder = 0;
393  double ledge = cpl_polynomial_get_coeff(traces[MUSE_TRACE_LEFT], &torder);
394  double redge = cpl_polynomial_get_coeff(traces[MUSE_TRACE_RIGHT], &torder);
395  if ((ledge < qwindow[0] + aParams->xborder) ||
396  (redge > qwindow[1] - aParams->xborder)) {
397  muse_trace_polys_delete(traces);
398  continue;
399  }
400  else {
401  unsigned int nwindows = (qwindow[3] - qwindow[2] + 1) / aParams->ybox;
402 
403  /* Add nwindows empty rows for the next batch of windows */
404  cpl_table_set_size(windowlist, firstrow + nwindows);
405 
406  unsigned int iwindow;
407  for (iwindow = 0; iwindow < nwindows; ++iwindow) {
408  cpl_size irow = firstrow + iwindow;
409 
410  int ymin = iwindow * aParams->ybox + qwindow[2];
411  int ymax = ymin + aParams->ybox - 1;
412 
413  double ymid = ymin + 0.5 * aParams->ybox;
414  double xleft = cpl_polynomial_eval_1d(traces[MUSE_TRACE_LEFT], ymid, NULL);
415  double xright = cpl_polynomial_eval_1d(traces[MUSE_TRACE_RIGHT], ymid, NULL);
416 
417  int xmin = (int)rint(xleft) + aParams->xgap;
418  int xmax = (int)rint(xright) - aParams->xgap;
419 
420  cpl_table_set_int(windowlist, "Quadrant", irow, iquadrant);
421  cpl_table_set_int(windowlist, "SliceNo", irow, islice);
422  cpl_table_set_int(windowlist, "Xmin", irow, xmin);
423  cpl_table_set_int(windowlist, "Xmax", irow, xmax);
424  cpl_table_set_int(windowlist, "Ymin", irow, ymin);
425  cpl_table_set_int(windowlist, "Ymax", irow, ymax);
426  }
427  firstrow += nwindows;
428  }
429 
430  muse_trace_polys_delete(traces);
431  }
432 
433  cpl_free(qwindow);
434  }
435 
436  if (!cpl_errorstate_is_equal(status)) {
437  cpl_table_delete(windowlist);
438  return NULL;
439  }
440 
441  return windowlist;
442 }
443 
444 /*---------------------------------------------------------------------------*/
455 /*---------------------------------------------------------------------------*/
456 static cpl_array *
457 muse_lingain_quadrant_get_windows(cpl_table *aWindowList,
458  unsigned char aQuadrant)
459 {
460  cpl_table_select_all(aWindowList);
461 
462  /* Determine the window indices for the current detector quadrant */
463  cpl_size nwindows = cpl_table_and_selected_int(aWindowList, "Quadrant",
464  CPL_EQUAL_TO, aQuadrant);
465  if (nwindows == 0) {
466  return NULL;
467  }
468 
469  cpl_array *windex = cpl_array_new(nwindows, CPL_TYPE_SIZE);
470 
471  cpl_size iwindow = 0;
472  cpl_size irow;
473  for (irow = 0; irow < cpl_table_get_nrow(aWindowList); ++irow) {
474  if (cpl_table_is_selected(aWindowList, irow)) {
475  cpl_array_set_cplsize(windex, iwindow, irow);
476  ++iwindow;
477  }
478  }
479 
480  /* Reset window list selection */
481  cpl_table_select_all(aWindowList);
482 
483  return windex;
484 }
485 
486 /*---------------------------------------------------------------------------*/
498 /*---------------------------------------------------------------------------*/
499 static cpl_array *
500 muse_lingain_quadrant_extract_data(const cpl_table *aGainTable,
501  const cpl_array *aWindows,
502  const char *aName,
503  cpl_size aRow)
504 {
505 
506  const cpl_array *src = cpl_table_get_array(aGainTable, aName, aRow);
507 
508  if (!src) {
509  return NULL;
510  }
511 
512  cpl_size nwindows = cpl_array_get_size(aWindows);
513  cpl_array *data = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);
514 
515  cpl_size iwindow;
516  for (iwindow = 0; iwindow < nwindows; ++iwindow) {
517  cpl_size jwindow = cpl_array_get_cplsize(aWindows, iwindow, NULL);
518  int invalid = 0;
519  double value = cpl_array_get_double(src, jwindow, &invalid);
520 
521  if (!invalid) {
522  cpl_array_set_double(data, iwindow, value);
523  } else {
524  cpl_array_set_invalid(data, iwindow);
525  }
526  }
527 
528  return data;
529 
530 }
531 
532 /*---------------------------------------------------------------------------*/
549 /*---------------------------------------------------------------------------*/
550 static cpl_array *
551 muse_lingain_compute_ron(const cpl_table *aWindowList,
552  cpl_table *aExposureList,
553  muse_imagelist *aImageList,
554  muse_lingain_params_t *aParams)
555 {
556  cpl_table_select_all(aExposureList);
557  cpl_size nbias = cpl_table_and_selected_string(aExposureList, "TYPE", CPL_EQUAL_TO,
558  MUSE_TAG_LINEARITY_BIAS);
559  if (nbias > 2) {
560  cpl_msg_warning(__func__, "Found more than 2 (%" CPL_SIZE_FORMAT ") "
561  "input images of type '%s' for IFU %u! Only the first 2 "
562  "images will be used!", nbias, MUSE_TAG_LINEARITY_BIAS,
563  aParams->nifu);
564  }
565 
566  cpl_size ibias = 0;
567  cpl_size iexposure;
568  for (iexposure = 0; iexposure < cpl_table_get_nrow(aExposureList); ++iexposure) {
569  if (cpl_table_is_selected(aExposureList, iexposure)) {
570  ibias = cpl_table_get_int(aExposureList, "INDEX", iexposure, NULL);
571  break;
572  }
573  }
574  cpl_table_select_all(aExposureList);
575 
576  /* Get bias images to compute the RON per window on the difference image *
577  * of the two bias images. Reject bad image pixels according to the data *
578  * quality information so that they don't affect the results. */
579  muse_image *bias1 = muse_imagelist_get(aImageList, ibias);
580  muse_image *bias2 = muse_imagelist_get(aImageList, ibias + 1);
581 
584 
585  cpl_image *dbias = cpl_image_subtract_create(bias1->data, bias2->data);
586 
587  cpl_array *ron = cpl_array_new(cpl_table_get_nrow(aWindowList), CPL_TYPE_DOUBLE);
588 
589  cpl_size iwindow;
590  for (iwindow = 0; iwindow < cpl_table_get_nrow(aWindowList); ++ iwindow) {
591  int xmin = cpl_table_get_int(aWindowList, "Xmin", iwindow, NULL);
592  int ymin = cpl_table_get_int(aWindowList, "Ymin", iwindow, NULL);
593  int xmax = cpl_table_get_int(aWindowList, "Xmax", iwindow, NULL);
594  int ymax = cpl_table_get_int(aWindowList, "Ymax", iwindow, NULL);
595 
596  double _ron = cpl_image_get_stdev_window(dbias, xmin, ymin, xmax, ymax);
597 
598  _ron = sqrt(0.5 * _ron * _ron);
599  cpl_array_set_double(ron, iwindow, _ron);
600  }
601  cpl_image_delete(dbias);
602 
603  return ron;
604 }
605 
606 /*---------------------------------------------------------------------------*/
628 /*---------------------------------------------------------------------------*/
629 static cpl_table *
630 muse_lingain_window_get_gain(const cpl_table *aWindowList,
631  const cpl_array *aRon,
632  cpl_table *aExposureList,
633  muse_imagelist *aImageList,
634  muse_lingain_params_t *aParams)
635 {
636 
637  /* Create a local table containing only the flat field images of the *
638  * linearity sequence. */
639  cpl_table_select_all(aExposureList);
640  cpl_size nflats = cpl_table_and_selected_string(aExposureList, "TYPE", CPL_EQUAL_TO,
641  MUSE_TAG_LINEARITY_FLAT);
642  if (nflats % 2) {
643  cpl_msg_warning(__func__, "Found odd number of input images (%" CPL_SIZE_FORMAT
644  ") input images of type '%s' for IFU %u! Exposure series "
645  "may be incorrect!", nflats, MUSE_TAG_LINEARITY_FLAT,
646  aParams->nifu);
647  }
648 
649  cpl_table *flats = cpl_table_extract_selected(aExposureList);
650  cpl_table_select_all(aExposureList);
651 
652  /* Create output table containing the gain measurements */
653  cpl_errorstate state = cpl_errorstate_get();
654 
655  cpl_size npairs = nflats / 2;
656  cpl_size nwindows = cpl_table_get_nrow(aWindowList);
657 
658  cpl_table *gain = cpl_table_new(npairs);
659  cpl_table_new_column(gain, "ExpTime", CPL_TYPE_DOUBLE);
660  cpl_table_new_column_array(gain, "Signal", CPL_TYPE_DOUBLE, nwindows);
661  cpl_table_new_column_array(gain, "Variance", CPL_TYPE_DOUBLE, nwindows);
662  cpl_table_new_column_array(gain, "Gain", CPL_TYPE_DOUBLE, nwindows);
663 
664  if (!cpl_errorstate_is_equal(state)) {
665  cpl_table_delete(gain);
666  cpl_table_delete(flats);
667  return NULL;
668  }
669 
670  /* Get median RON and its standard deviation for the computation of the *
671  * gain for each measurement window. */
672  double mron_value = cpl_array_get_median(aRon);
673  double mron_sdev = cpl_array_get_stdev(aRon);
674 
675  /* Compute a gain estimate for each pair of input flat fields and each *
676  * measurement window */
677  cpl_size kpair = 0;
678  cpl_size ipair;
679  for (ipair = 0; ipair < npairs; ++ipair) {
680  cpl_size iexposure = 2 * ipair;
681  cpl_size jexposure = iexposure + 1;
682  int iflat = cpl_table_get_int(flats, "INDEX", iexposure, NULL);
683  int jflat = cpl_table_get_int(flats, "INDEX", jexposure, NULL);
684  muse_image *flat1 = muse_imagelist_get(aImageList, iflat);
685  muse_image *flat2 = muse_imagelist_get(aImageList, jflat);
686 
687  double exptime1 = cpl_table_get_double(flats, "EXPTIME", iexposure, NULL);
688  double exptime2 = cpl_table_get_double(flats, "EXPTIME", jexposure, NULL);
689  double flux1 = cpl_image_get_mean(flat1->data);
690  double flux2 = cpl_image_get_mean(flat2->data);
691 
692  if (fabs((flux1 - flux2) / flux1) > aParams->fluxtol) {
693  cpl_msg_warning(__func__, "Inconsistent overall flux level (%.4f, "
694  "%.4f) detected for flat field pair (%d, %d) of IFU %u "
695  "with exposure time %.4f! Skipping this pair!",
696  flux1, flux2, iflat, jflat, aParams->nifu, exptime1);
697  continue;
698  }
699 
700  cpl_table_set_double(gain, "ExpTime", kpair, 0.5 * (exptime1 + exptime2));
701 
702  /* Exclude known bad pixels from the following analysis */
705 
706  /* Compute gain estimate for each measurement window. Use the union *
707  * of the two data quality masks so that the measurement is done for *
708  * the same pixels in both images. If sigma clipping is enabled the *
709  * computations are done on the cleaned data. */
710  cpl_array *_signal = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);
711  cpl_array *_variance = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);
712  cpl_array *_gain = cpl_array_new(nwindows, CPL_TYPE_DOUBLE);
713 
714  if (aParams->sigma > 0.) {
715 
716  cpl_size iwindow;
717  for (iwindow = 0; iwindow < nwindows; ++iwindow) {
718  int xmin = cpl_table_get_int(aWindowList, "Xmin", iwindow, NULL);
719  int ymin = cpl_table_get_int(aWindowList, "Ymin", iwindow, NULL);
720  int xmax = cpl_table_get_int(aWindowList, "Xmax", iwindow, NULL);
721  int ymax = cpl_table_get_int(aWindowList, "Ymax", iwindow, NULL);
722 
723  /* Extract the image region of the window from both flat fields. *
724  * Use the union of the bad pixel masks of the subimages for any *
725  * further analysis. */
726  cpl_image *_flat1 = cpl_image_extract(flat1->data,
727  xmin, ymin, xmax, ymax);
728  cpl_image *_flat2 = cpl_image_extract(flat2->data,
729  xmin, ymin, xmax, ymax);
730  cpl_mask *mask = cpl_image_get_bpm(_flat1);
731  cpl_mask_or(mask, cpl_image_get_bpm(_flat2));
732  cpl_image_reject_from_mask(_flat2, mask);
733 
734  double median1 = cpl_image_get_median(_flat1);
735  double median2 = cpl_image_get_median(_flat2);
736  double clip1 = aParams->sigma * cpl_image_get_stdev(_flat1);
737  double clip2 = aParams->sigma * cpl_image_get_stdev(_flat2);
738 
739  /* Remove outliers from the flat field pixel values by creating *
740  * a bad pixel mask for these pixels. The resulting mask used *
741  * for the following analysis is then the union of all involved *
742  * bad pixels masks. */
743  cpl_mask *mask1 =
744  cpl_mask_threshold_image_create(_flat1,
745  median1 - clip1,
746  median1 + clip1);
747  cpl_mask *mask2 =
748  cpl_mask_threshold_image_create(_flat2,
749  median2 - clip2,
750  median2 + clip2);
751 
752  /* The created masks need to be inverted to flag pixel values *
753  * outside of the defined range as bad, so that they can be *
754  * combined and used as mask for the flat fields. */
755  cpl_mask_not(mask1);
756  cpl_mask_not(mask2);
757  cpl_mask_or(mask1, mask2);
758  cpl_mask_delete(mask2);
759 
760  cpl_mask_or(mask, mask1);
761  cpl_mask_delete(mask1);
762 
763  /* Apply the mask, also to the second member of the flat field *
764  * pair. The subtraction operation propagates the bad pixel map *
765  * to the difference image. */
766  cpl_image_reject_from_mask(_flat2, mask);
767  cpl_image *dflat = cpl_image_subtract_create(_flat1, _flat2);
768 
769  if ((median1 < kMuseSaturationLimit) && (median2 < kMuseSaturationLimit)) {
770  double s1 = cpl_image_get_mean(_flat1);
771  double s2 = cpl_image_get_mean(_flat2);
772  double s = 0.5 * (s1 + s2);
773  double v = 1.;
774  double g = 0.;
775 
776  if ((cpl_array_get_double(aRon, iwindow, NULL) - mron_value) <= 5. * mron_sdev) {
777  double sdev = cpl_image_get_stdev(dflat);
778  v = 0.5 * sdev * sdev - mron_value * mron_value;
779  g = s / v;
780  }
781  cpl_array_set_double(_signal, iwindow, s);
782  cpl_array_set_double(_variance, iwindow, v);
783  cpl_array_set_double(_gain, iwindow, g);
784  } else {
785  cpl_array_set_invalid(_signal, iwindow);
786  cpl_array_set_invalid(_variance, iwindow);
787  cpl_array_set_invalid(_gain, iwindow);
788  }
789 
790  cpl_image_delete(dflat);
791  cpl_image_delete(_flat2);
792  cpl_image_delete(_flat1);
793  }
794 
795  } else {
796 
797  /* Create difference image using the whole image only once, instead *
798  * of doing the subtraction for each window. Pixels having a non-zero *
799  * data quality flag are ignored. As a side effect the subtraction *
800  * creates a bad pixel mask for the result image which is the union *
801  * of the two input mask, and thus can be used as bad pixel mask for *
802  * the per window computations on both flat fields of the current pair */
803 
804  cpl_image *dflat = cpl_image_subtract_create(flat1->data, flat2->data);
805 
806  /* Use the union of the bad pixel masks for both input flat fields */
807  cpl_image_reject_from_mask(flat1->data, cpl_image_get_bpm(dflat));
808  cpl_image_reject_from_mask(flat2->data, cpl_image_get_bpm(dflat));
809 
810  cpl_size iwindow;
811  for (iwindow = 0; iwindow < nwindows; ++iwindow) {
812  int xmin = cpl_table_get_int(aWindowList, "Xmin", iwindow, NULL);
813  int ymin = cpl_table_get_int(aWindowList, "Ymin", iwindow, NULL);
814  int xmax = cpl_table_get_int(aWindowList, "Xmax", iwindow, NULL);
815  int ymax = cpl_table_get_int(aWindowList, "Ymax", iwindow, NULL);
816 
817  double median1 = cpl_image_get_median_window(flat1->data,
818  xmin, ymin, xmax, ymax);
819  double median2 = cpl_image_get_median_window(flat2->data,
820  xmin, ymin, xmax, ymax);
821 
822  if ((median1 < kMuseSaturationLimit) && (median2 < kMuseSaturationLimit)) {
823  double s1 = cpl_image_get_mean_window(flat1->data,
824  xmin, ymin, xmax, ymax);
825  double s2 = cpl_image_get_mean_window(flat2->data,
826  xmin, ymin, xmax, ymax);
827  double s = 0.5 * (s1 + s2);
828  double v = 1.;
829  double g = 0.;
830 
831  if ((cpl_array_get_double(aRon, iwindow, NULL) - mron_value) <= 5. * mron_sdev) {
832  double sdev = cpl_image_get_stdev_window(dflat,
833  xmin, ymin, xmax, ymax);
834  v = 0.5 * sdev * sdev - mron_value * mron_value;
835  g = s / v;
836  }
837  cpl_array_set_double(_signal, iwindow, s);
838  cpl_array_set_double(_variance, iwindow, v);
839  cpl_array_set_double(_gain, iwindow, g);
840  } else {
841  cpl_array_set_invalid(_signal, iwindow);
842  cpl_array_set_invalid(_variance, iwindow);
843  cpl_array_set_invalid(_gain, iwindow);
844  }
845  }
846 
847  cpl_image_delete(dflat);
848 
849  }
850 
851  cpl_table_set_array(gain, "Signal", kpair, _signal);
852  cpl_table_set_array(gain, "Variance", kpair, _variance);
853  cpl_table_set_array(gain, "Gain", kpair, _gain);
854  ++kpair;
855 
856  cpl_array_delete(_signal);
857  cpl_array_delete(_variance);
858  cpl_array_delete(_gain);
859  }
860 
861  cpl_table_delete(flats);
862 
863  /* Remove pairs which have been rejected */
864  cpl_table_erase_invalid_rows(gain);
865  if (cpl_table_get_nrow(gain) == 0) {
866  cpl_table_delete(gain);
867  gain = NULL;
868  }
869 
870  return gain;
871 }
872 
873 /*---------------------------------------------------------------------------*/
887 /*---------------------------------------------------------------------------*/
888 static cpl_error_code
889 muse_lingain_quadrant_get_gain(muse_gain_fit_t *aGain,
890  const cpl_table *aGainTable,
891  const cpl_array *aWindows,
892  muse_lingain_params_t *aParams)
893 {
894  /* Create flat arrays for the signal and the gain from the gain table *
895  * and the windows for a single quadrant. The signal is converted to *
896  * logarithmic scale, and all data elements for which the signal or *
897  * the gain are invalid are marked as such in the destination array. */
898  cpl_size nrows = cpl_table_get_nrow(aGainTable);
899  cpl_size nwindows = cpl_array_get_size(aWindows);
900  cpl_size ndata = nrows * nwindows;
901 
902  cpl_array *qsignal = cpl_array_new(ndata, CPL_TYPE_DOUBLE);
903  cpl_array *qgain = cpl_array_new(ndata, CPL_TYPE_DOUBLE);
904 
905  cpl_size irow;
906  for (irow = 0; irow < nrows; ++irow) {
907  cpl_size iwindow;
908  cpl_size stride = irow * nwindows;
909 
910  const cpl_array *_qsignal = cpl_table_get_array(aGainTable,
911  "Signal", irow);
912  const cpl_array *_qgain = cpl_table_get_array(aGainTable,
913  "Gain", irow);
914 
915  for (iwindow = 0; iwindow < nwindows; ++iwindow) {
916  cpl_size jwindow = cpl_array_get_cplsize(aWindows, iwindow, NULL);
917 
918  int snull = 0;
919  int gnull = 0;
920  double s = cpl_array_get_double(_qsignal, jwindow, &snull);
921  double g = cpl_array_get_double(_qgain, jwindow, &gnull);
922 
923  cpl_size idata = stride + iwindow;
924  if (((snull == 0) && (gnull == 0)) && (s > 0.)) {
925  cpl_array_set_double(qsignal, idata, log10(s));
926  cpl_array_set_double(qgain, idata, g);
927  } else {
928  cpl_array_set_invalid(qsignal, idata);
929  cpl_array_set_invalid(qgain, idata);
930  }
931  }
932  }
933 
934  /* Create the bins for the gain and the abscissa for the gain fits. */
935  cpl_size nbins = (cpl_size)((aParams->signalmax - aParams->signalmin) /
936  aParams->signalbin + 0.5);
937  cpl_array *gx = cpl_array_new(nbins, CPL_TYPE_DOUBLE);
938  cpl_array *gy = cpl_array_new(nbins, CPL_TYPE_DOUBLE);
939 
940  cpl_size ibin;
941  for (ibin = 0; ibin < nbins; ++ibin) {
942  double bmin = aParams->signalmin + ibin * aParams->signalbin;
943  double bmax = bmin + aParams->signalbin;
944  double bmid = 0.5 * (bmin + bmax);
945 
946  if (bmid < log10(aParams->gainlimit)) {
947  cpl_array_set_invalid(gx, ibin);
948  cpl_array_set_invalid(gy, ibin);
949  continue;
950  }
951 
952  /* Change signal scale back to linear for the fit later on. */
953  cpl_array_set(gx, ibin, pow(10., bmid));
954 
955  cpl_array *gdata = cpl_array_duplicate(qgain);
956 
957  cpl_size idata;
958  for (idata = 0; idata < ndata; ++idata) {
959  /* Mark elements outside of the current bin boundaries as *
960  * invalid, so that they are ignored in the computations. */
961  int invalid = 0;
962  double s = cpl_array_get_double(qsignal, idata, &invalid);
963 
964  if (invalid || (s < bmin) || (s >= bmax)) {
965  cpl_array_set_invalid(gdata, idata);
966  }
967  }
968 
969  /* Skip empty bins */
970  if (cpl_array_count_invalid(gdata) == ndata) {
971  cpl_array_set_invalid(gx, ibin);
972  cpl_array_set_invalid(gy, ibin);
973  cpl_array_delete(gdata);
974  continue;
975  }
976 
977  /* Get median and standard deviation of the remaining elements. *
978  * and remove outliers from the gain data before the final median *
979  * gain value for this bin is calculated. */
980  double median = cpl_array_get_median(gdata);
981  double stdev = cpl_array_get_stdev(gdata);
982 
983  stdev *= aParams->gainsigma;
984  for (idata = 0; idata < ndata; ++idata) {
985  /* Mark elements outside of the current bin boundaries as *
986  * invalid, so that they are ignored in the computations. */
987  int invalid = 0;
988  double g = cpl_array_get_double(gdata, idata, &invalid);
989 
990  if (invalid || (g <= median - stdev) || (g >= median + stdev)) {
991  cpl_array_set_invalid(gdata, idata);
992  }
993  }
994 
995  if (cpl_array_count_invalid(gdata) == ndata) {
996  cpl_array_set_invalid(gx, ibin);
997  cpl_array_set_invalid(gy, ibin);
998  } else {
999  /* XXX: For debugging, may be removed later */
1000  double gmean = cpl_array_get_mean(gdata);
1001 
1002  cpl_array_set_double(gy, ibin, gmean);
1003  }
1004 
1005  cpl_array_delete(gdata);
1006  }
1007 
1008  cpl_array_delete(qgain);
1009  cpl_array_delete(qsignal);
1010 
1011 
1012  /* Remove invalid bins prior to fitting a straight line to the gain data */
1015 
1016  cpl_size nx = cpl_array_get_size(gx);
1017  cpl_size ny = cpl_array_get_size(gy);
1018 
1019  if ((nx == 0) || (ny == 0)) {
1020 
1021  cpl_msg_debug(__func__, "Fitting the gain relation failed. No data "
1022  "available!");
1023  cpl_array_delete(gy);
1024  cpl_array_delete(gx);
1025 
1026  return CPL_ERROR_DATA_NOT_FOUND;
1027  }
1028 
1029  if (nx != ny) {
1030  cpl_msg_debug(__func__, "Fitting the gain relation failed. Gain "
1031  "and signal data sets do not match!");
1032  cpl_array_delete(gy);
1033  cpl_array_delete(gx);
1034 
1035  return CPL_ERROR_INCOMPATIBLE_INPUT;
1036  }
1037 
1038  /* One extra data point is required so that residuals could be computed! */
1039  if (nx < 3) {
1040  cpl_msg_debug(__func__, "Fitting the gain relation failed. Insufficient "
1041  "data points; cannot fit a first order polynomial!");
1042  cpl_array_delete(gy);
1043  cpl_array_delete(gx);
1044 
1045  return CPL_ERROR_DATA_NOT_FOUND;
1046  }
1047 
1048  /* Fit a straight line to the cleaned, mean gain values. */
1049  double *_x = cpl_array_get_data_double(gx);
1050  double *_y = cpl_array_get_data_double(gy);
1051 
1052  cpl_matrix *x = cpl_matrix_wrap(1, nx, _x);
1053  cpl_vector *y = cpl_vector_wrap(ny, _y);
1054  cpl_boolean symetric = CPL_FALSE;
1055  const cpl_size mindeg = 0;
1056  const cpl_size maxdeg = 1;
1057 
1058  cpl_polynomial *gfit = cpl_polynomial_new(1);
1059  cpl_error_code ecode = cpl_polynomial_fit(gfit, x, &symetric, y, NULL,
1060  CPL_FALSE, &mindeg, &maxdeg);
1061 
1062  cpl_vector_unwrap(y);
1063  cpl_matrix_unwrap(x);
1064 
1065  if (ecode != CPL_ERROR_NONE) {
1066  cpl_polynomial_delete(gfit);
1067  cpl_array_delete(gy);
1068  cpl_array_delete(gx);
1069 
1070  cpl_msg_debug(__func__, "Fitting the gain relation failed. Fitting "
1071  "first order polynomial to gain data failed!");
1072 
1073  aGain->gain = 0.;
1074  return CPL_ERROR_ILLEGAL_OUTPUT;
1075  }
1076 
1077  double rms = 0.;
1078  cpl_size ix;
1079  for (ix = 0; ix < nx; ++ix) {
1080  double rx = cpl_array_get_double(gx, ix, NULL);
1081  double ry = cpl_array_get_double(gy, ix, NULL);
1082  double ryf = cpl_polynomial_eval_1d(gfit, rx, NULL);
1083  rms += (ry - ryf) * (ry - ryf);
1084  }
1085  rms = sqrt(rms);
1086 
1087  cpl_array_delete(gy);
1088  cpl_array_delete(gx);
1089 
1090  const cpl_size order = 0;
1091  aGain->gain = cpl_polynomial_get_coeff(gfit, &order);
1092  aGain->rms = rms;
1093 
1094  cpl_polynomial_delete(gfit);
1095 
1096  return CPL_ERROR_NONE;
1097 }
1098 
1099 /*---------------------------------------------------------------------------*/
1111 /*---------------------------------------------------------------------------*/
1112 static cpl_error_code
1113 muse_lingain_quadrant_get_nonlinearity(muse_linearity_fit_t *aFit,
1114  const cpl_table *aGainTable,
1115  const cpl_array *aWindows,
1116  double aGain,
1117  muse_lingain_params_t *aParams)
1118 {
1119 
1120  /* Build the log10(signal) grid. Using these signal bins to *
1121  * determine the set of measurement windows falling into each *
1122  * bin from the exposure with the longest exposure time. The *
1123  * selected windows are then used for the analysis of all *
1124  * exposure times */
1125 
1126  cpl_size ntimes = cpl_table_get_nrow(aGainTable);
1127  cpl_size irow;
1128  cpl_error_code ecode = cpl_table_get_column_maxpos(aGainTable, "ExpTime",
1129  &irow);
1130  if (ecode != CPL_ERROR_NONE) {
1131  cpl_msg_debug(__func__, "Maximum exposure time cannot be determined from "
1132  "linearity data set of IFU %u!", aParams->nifu);
1133  return ecode;
1134  }
1135 
1136  if (irow != ntimes - 1) {
1137  cpl_msg_debug(__func__, "Linearity data set of IFU %u is not sorted in "
1138  "ascending order of the exposure time!", aParams->nifu);
1139  return CPL_ERROR_ILLEGAL_INPUT;
1140  }
1141 
1142  /* Get the signal array of the longest exposure and use this signal *
1143  * (in log10(ADU)) to select measurement windows for each signal *
1144  * bin. Correct for the gain, since bin boundaries are given in *
1145  * log10(counts). */
1146  cpl_array *sigref =
1147  muse_lingain_quadrant_extract_data(aGainTable, aWindows, "Signal", irow);
1148  cpl_array_logarithm(sigref, 10.);
1149 
1150  cpl_size nsignal = cpl_array_get_size(sigref);
1151  cpl_size nlevels = (cpl_size)((aParams->ctsmax - aParams->ctsmin) /
1152  aParams->ctsbin + 0.5);
1153 
1154  cpl_array **counts = cpl_calloc(nlevels, sizeof *counts);
1155  const double qgain = log10(aGain);
1156  cpl_size klevel = 0;
1157  cpl_size ilevel;
1158  for (ilevel = 0; ilevel < nlevels; ++ilevel) {
1159  double shigh = aParams->ctsmax - ilevel * aParams->ctsbin;
1160  double slow = shigh - aParams->ctsbin;
1161 
1162  shigh -= qgain;
1163  slow -= qgain;
1164 
1165  /* For each bin create an array of window indices which satisfy *
1166  * the selection criterion. */
1167  cpl_size *sindex = cpl_malloc(nsignal * sizeof *sindex);
1168  cpl_size kdata = 0;
1169  cpl_size idata;
1170  for (idata = 0; idata < nsignal; ++idata) {
1171  int invalid = 0;
1172  double s = cpl_array_get_double(sigref, idata, &invalid);
1173 
1174  if (!invalid && ((s >= slow) && (s < shigh))) {
1175  sindex[kdata] = cpl_array_get_cplsize(aWindows, idata, NULL);
1176  ++kdata;
1177  }
1178  }
1179 
1180  if (kdata == 0) {
1181  cpl_free(sindex);
1182  cpl_msg_debug(__func__, "No valid measurement found within signal "
1183  "range [%.6f, %.6f[ [log10(counts)] of the linearity "
1184  "data set of IFU %u!", slow + qgain, shigh + qgain,
1185  aParams->nifu);
1186  continue;
1187  } else {
1188 
1189  /* Compute the mean signal [ADU] from the selected windows for *
1190  * each exposure time individually. If no valid signal data is *
1191  * present for an exposure time the respective mean signal *
1192  * element is marked as invalid to be able to exclude it *
1193  * during the following analysis. */
1194  cpl_array *smean = cpl_array_new(ntimes, CPL_TYPE_DOUBLE);
1195  cpl_array *windex = cpl_array_wrap_cplsize(sindex, kdata);
1196  cpl_size itime;
1197  for (itime = 0; itime < ntimes; ++itime) {
1198  cpl_array *qsignal =
1199  muse_lingain_quadrant_extract_data(aGainTable, windex,
1200  "Signal", itime);
1201  if (qsignal && (cpl_array_count_invalid(qsignal) < kdata)) {
1202  double mean = cpl_array_get_mean(qsignal);
1203  cpl_array_set_double(smean, itime, mean);
1204  } else {
1205  cpl_array_set_invalid(smean, itime);
1206  }
1207  cpl_array_delete(qsignal);
1208  }
1209  cpl_array_unwrap(windex);
1210 
1211  /* Note that here counts is still in ADU, despite counts is usually *
1212  * used after the signal is converted to electrons! */
1213  counts[klevel] = smean;
1214  ++klevel;
1215  }
1216  cpl_free(sindex);
1217  }
1218 
1219  cpl_array_delete(sigref);
1220 
1221  /* Get the array of the exposure times and add time offset */
1222  cpl_array *exptime = cpl_array_new(ntimes, CPL_TYPE_DOUBLE);
1223 
1224  cpl_size itime;
1225  for (itime = 0; itime < ntimes; ++itime) {
1226  double _exptime = cpl_table_get_double(aGainTable, "ExpTime",
1227  itime, NULL);
1228  cpl_array_set_double(exptime, itime, _exptime);
1229  }
1230  cpl_array_add_scalar(exptime, aParams->toffset);
1231 
1232  /* Compute count rate from the determined mean signals */
1233  cpl_array **crate = cpl_calloc(klevel, sizeof *crate);
1234  for (ilevel = 0; ilevel < klevel; ++ilevel) {
1235  crate[ilevel] = cpl_array_duplicate(counts[ilevel]);
1236  cpl_array_divide(crate[ilevel], exptime);
1237  }
1238 
1239  /* Select count rates located within the limits of the desired linear *
1240  * range, and select the corresponding exposure times. The count rates *
1241  * should be constant for all exposure times. Thus, a constant is *
1242  * fitted to the relation exposure time vs. count rate, and the *
1243  * residuals are computed. */
1244 
1245  /* Convert linear range limit to ADU */
1246  double linmin = pow(10., aParams->linearmin - qgain);
1247  double linmax = pow(10., aParams->linearmax - qgain);
1248 
1249  cpl_array **cresidual = cpl_malloc(klevel * sizeof *cresidual);
1250 
1251  for (ilevel = 0; ilevel < klevel; ++ilevel) {
1252  cpl_array *_exptime = cpl_array_duplicate(exptime);
1253  cpl_array *_crate = cpl_array_duplicate(crate[ilevel]);
1254 
1255  for (itime = 0; itime < ntimes; ++itime) {
1256  int invalid = 0;
1257  double cnts = cpl_array_get_double(counts[ilevel], itime, &invalid);
1258 
1259  if (invalid || ((cnts < linmin) || (cnts >= linmax))) {
1260  cpl_array_set_invalid(_exptime, itime);
1261  cpl_array_set_invalid(_crate, itime);
1262  }
1263  }
1264 
1265  if (cpl_array_get_size(_crate) == cpl_array_count_invalid(_crate)) {
1266 
1267  /* If all data at this count level is invalid mark the residuals *
1268  * as unavailable. */
1269  cresidual[ilevel] = NULL;
1270 
1271  cpl_array_delete(_crate);
1272  cpl_array_delete(_exptime);
1273 
1274  } else {
1275 
1276  /* XXX: Fitting a zero order polynomial with constant weights in *
1277  * a least square sense is equivalent to calculating the mean! *
1278  * Any reason for using a true fit? */
1279 #ifdef USE_POLYFIT
1280 
1281  /* Remove invalid data prior to fitting a constant to the count rates */
1282  muse_cplarray_erase_invalid(_exptime);
1284 
1285  cpl_size nt = cpl_array_get_size(_exptime);
1286  cpl_size nc = cpl_array_get_size(_crate);
1287 
1288  /* By design the 2 arrays should have the same number of elements. *
1289  * Thus the following condition should never be true! */
1290  if (nt != nc) {
1291  cpl_msg_debug(__func__, "Fitting a constant to the count rate data "
1292  "failed. Exposure time and count rate data sets do not "
1293  "match!");
1294  cpl_array_delete(_crate);
1295  cpl_array_delete(_exptime);
1296 
1297  muse_vfree((void **)cresidual, klevel, (muse_free_func)cpl_array_delete);
1298  muse_vfree((void **)crate, klevel, (muse_free_func)cpl_array_delete);
1299  muse_vfree((void **)counts, nlevels, (muse_free_func)cpl_array_delete);
1300 
1301  cpl_array_delete(exptime);
1302 
1303  return CPL_ERROR_INCOMPATIBLE_INPUT;
1304  }
1305 
1306  /* Fit a constant to the cleaned, count rate values. */
1307  double *_et = cpl_array_get_data_double(_exptime);
1308  double *_cr = cpl_array_get_data_double(_crate);
1309 
1310  cpl_matrix *et = cpl_matrix_wrap(1, nt, _et);
1311  cpl_vector *cr = cpl_vector_wrap(nc, _cr);
1312  cpl_boolean symetric = CPL_FALSE;
1313  const cpl_size mindeg = 0;
1314  const cpl_size maxdeg = 0;
1315 
1316  cpl_polynomial *crfit = cpl_polynomial_new(1);
1317  cpl_error_code ecode = cpl_polynomial_fit(crfit, et, &symetric, cr, NULL,
1318  CPL_FALSE, &mindeg, &maxdeg);
1319 
1320  cpl_vector_unwrap(et);
1321  cpl_matrix_unwrap(cr);
1322 
1323  cpl_array_delete(_crate);
1324  cpl_array_delete(_exptime);
1325 
1326  if (ecode != CPL_ERROR_NONE) {
1327  cpl_polynomial_delete(crfit);
1328  cpl_msg_debug(__func__, "Fitting a constant to the count rate level "
1329  "failed. Fitting a zero order polynomial to count rate "
1330  "data failed!");
1331 
1332  muse_vfree((void **)cresidual, klevel, (muse_free_func)cpl_array_delete);
1333  muse_vfree((void **)crate, klevel, (muse_free_func)cpl_array_delete);
1334  muse_vfree((void **)counts, nlevels, (muse_free_func)cpl_array_delete);
1335 
1336  cpl_array_delete(exptime);
1337 
1338  return CPL_ERROR_ILLEGAL_OUTPUT;
1339  }
1340 
1341  const cpl_size order = 0;
1342  double crlevel = cpl_polynomial_get_coeff(crfit, &order);
1343  cpl_polynomial_delete(crfit);
1344 #else
1345  double crlevel = cpl_array_get_mean(_crate);
1346 
1347  cpl_array_delete(_crate);
1348  cpl_array_delete(_exptime);
1349 #endif
1350  /* Calculate the count rate residuals */
1351  cresidual[ilevel] = cpl_array_duplicate(crate[ilevel]);
1352  cpl_array_subtract_scalar(cresidual[ilevel], crlevel);
1353  cpl_array_divide(cresidual[ilevel], crate[ilevel]);
1354  }
1355  }
1356 
1357  cpl_array_delete(exptime);
1358 
1359 
1360  /* Fit the final non-linearity relation, i.e. signal in ADU vs. residual *
1361  * non-linearity. */
1362 
1363  /* Create signal bins for which the mean residual will be computed. *
1364  * Reuse the bin settings used for the gain computation. */
1365  double signalmax = aParams->signalmax;
1366  double signalmin = aParams->signalmin;
1367  double signalbin = aParams->signalbin;
1368 
1369  cpl_size nbins = (cpl_size)((signalmax - signalmin) / signalbin + 0.5);
1370  cpl_array *lx = cpl_array_new(nbins, CPL_TYPE_DOUBLE);
1371  cpl_array *ly = cpl_array_new(nbins, CPL_TYPE_DOUBLE);
1372  cpl_array *lyse = cpl_array_new(nbins, CPL_TYPE_DOUBLE);
1373 
1374  cpl_size ibin;
1375  for (ibin = 0; ibin < nbins; ++ibin) {
1376  double bmin = signalmin + ibin * signalbin;
1377  double bmax = bmin + signalbin;
1378  double bmid = 0.5 * (bmin + bmax);
1379 
1380  cpl_array_set_double(lx, ibin, bmid);
1381 
1382  bmin = pow(10., bmin);
1383  bmax = pow(10., bmax);
1384 
1385  double *rdata = cpl_malloc((klevel * ntimes) * sizeof *rdata);
1386  cpl_size kresidual = 0;
1387  for (ilevel = 0; ilevel < klevel; ++ilevel) {
1388  cpl_array *residual = cresidual[ilevel];
1389 
1390  if (residual != NULL) {
1391  for (itime = 0; itime < ntimes; ++itime) {
1392  int cinvalid = 0;
1393  int rinvalid = 0;
1394  double _counts = cpl_array_get_double(counts[ilevel], itime, &cinvalid);
1395  double _residual = cpl_array_get_double(residual, itime, &rinvalid);
1396 
1397  if (!(cinvalid || rinvalid) && ((_counts >= bmin) && (_counts < bmax))) {
1398  rdata[kresidual] = _residual;
1399  ++kresidual;
1400  }
1401  }
1402  }
1403  }
1404  if (kresidual == 0) {
1405  cpl_array_set_invalid(lx, ibin);
1406  cpl_array_set_invalid(ly, ibin);
1407  cpl_array_set_invalid(lyse, ibin);
1408  } else {
1409  cpl_array *r = cpl_array_wrap_double(rdata, kresidual);
1410 
1411  double rmean = cpl_array_get_mean(r);
1412  double rsdev = cpl_array_get_stdev(r);
1413  cpl_array_set_double(ly, ibin, rmean);
1414  cpl_array_set_double(lyse, ibin, rsdev);
1415 
1416  cpl_array_unwrap(r);
1417  }
1418  cpl_free(rdata);
1419  }
1420 
1421  muse_vfree((void **)cresidual, klevel, (muse_free_func)cpl_array_delete);
1422  muse_vfree((void **)crate, klevel, (muse_free_func)cpl_array_delete);
1423  muse_vfree((void **)counts, nlevels, (muse_free_func)cpl_array_delete);
1424 
1425  /* Fit a polynomial to the final signal vs. mean residual non-linearity */
1429 
1430  cpl_size nx = cpl_array_get_size(lx);
1431  cpl_size ny = cpl_array_get_size(ly);
1432 
1433  if ((nx == 0) || (ny == 0)) {
1434 
1435  cpl_msg_debug(__func__, "Fitting the non-linearity relation failed. "
1436  "No data available!");
1437  cpl_array_delete(lyse);
1438  cpl_array_delete(ly);
1439  cpl_array_delete(lx);
1440 
1441  return CPL_ERROR_DATA_NOT_FOUND;
1442  }
1443 
1444  if (nx != ny) {
1445  cpl_msg_debug(__func__, "Fitting the non-linearity relation failed. "
1446  "Linearity residual and signal data sets do not match!");
1447  cpl_array_delete(lyse);
1448  cpl_array_delete(ly);
1449  cpl_array_delete(lx);
1450 
1451  return CPL_ERROR_INCOMPATIBLE_INPUT;
1452  }
1453 
1454  /* One extra data point is required so that the residuals can be computed! */
1455  if (nx < aParams->order + 2) {
1456  cpl_msg_debug(__func__, "Fitting the non-linearity relation failed. "
1457  "Insufficient data points; cannot fit a %d order "
1458  "polynomial!", aParams->order);
1459  cpl_array_delete(lyse);
1460  cpl_array_delete(ly);
1461  cpl_array_delete(lx);
1462 
1463  return CPL_ERROR_DATA_NOT_FOUND;
1464  }
1465 
1466  /* Fit a polynomial to the cleaned, mean residual non-linearity values. */
1467  double *_x = cpl_array_get_data_double(lx);
1468  double *_y = cpl_array_get_data_double(ly);
1469  double *_yse = cpl_array_get_data_double(lyse);
1470 
1471  cpl_matrix *x = cpl_matrix_wrap(1, nx, _x);
1472  cpl_vector *y = cpl_vector_wrap(ny, _y);
1473  cpl_vector *yse = cpl_vector_wrap(ny, _yse);
1474  cpl_boolean symetric = CPL_FALSE;
1475  const cpl_size mindeg = 0;
1476  const cpl_size maxdeg = aParams->order;
1477 
1478  cpl_polynomial *lfit = cpl_polynomial_new(1);
1479  ecode = cpl_polynomial_fit(lfit, x, &symetric, y, NULL, CPL_FALSE,
1480  &mindeg, &maxdeg);
1481 
1482  cpl_vector_unwrap(yse);
1483  cpl_vector_unwrap(y);
1484  cpl_matrix_unwrap(x);
1485 
1486  if (ecode != CPL_ERROR_NONE) {
1487  cpl_polynomial_delete(lfit);
1488  cpl_array_delete(lyse);
1489  cpl_array_delete(ly);
1490  cpl_array_delete(lx);
1491 
1492  cpl_msg_debug(__func__, "Fitting the non-linearity relation failed. "
1493  "Fitting %d order polynomial to residual non-linearity "
1494  "data failed!", aParams->order);
1495  aFit->coefficients = NULL;
1496  aFit->range[0] = 0;
1497  aFit->range[1] = 0;
1498 
1499  return CPL_ERROR_ILLEGAL_OUTPUT;
1500  }
1501 
1502  /* Save the results of the polynomial fit: coefficients and range */
1503  cpl_size ncoeff = aParams->order + 1;
1504  cpl_array *coefficients = cpl_array_new(ncoeff, CPL_TYPE_DOUBLE);
1505 
1506  cpl_size icoeff;
1507  for (icoeff = 0; icoeff < ncoeff; ++icoeff) {
1508  const cpl_size order = icoeff;
1509  double coeff = cpl_polynomial_get_coeff(lfit, &order);
1510  cpl_array_set_double(coefficients, icoeff, coeff);
1511  }
1512 
1513  double rms = 0.;
1514  cpl_size ix;
1515  for (ix = 0; ix < nx; ++ix) {
1516  double rx = cpl_array_get_double(lx, ix, NULL);
1517  double ry = cpl_array_get_double(ly, ix, NULL);
1518  double ryf = cpl_polynomial_eval_1d(lfit, rx, NULL);
1519  rms += (ry - ryf) * (ry - ryf);
1520  }
1521  rms = sqrt(rms);
1522  cpl_polynomial_delete(lfit);
1523 
1524  aFit->coefficients = coefficients;
1525  aFit->range[0] = cpl_array_get_min(lx);
1526  aFit->range[1] = cpl_array_get_max(lx);
1527  aFit->rms = rms;
1528 
1529  cpl_array_delete(lyse);
1530  cpl_array_delete(ly);
1531  cpl_array_delete(lx);
1532 
1533  return CPL_ERROR_NONE;
1534 }
1535 
1536 /*---------------------------------------------------------------------------*/
1558 /*---------------------------------------------------------------------------*/
1559 static cpl_table *
1560 muse_lingain_compute_gain(const cpl_table *aGainTable, cpl_table *aWindowList,
1561  muse_lingain_params_t *aParams)
1562 {
1563  const unsigned char nquadrants = 4;
1564  unsigned char iquadrant;
1565 
1566  cpl_table *gain = cpl_table_new(nquadrants);
1567 
1568  if (!gain) {
1569  return NULL;
1570  }
1571 
1572  cpl_table_new_column(gain, "Gain", CPL_TYPE_DOUBLE);
1573  cpl_table_new_column(gain, "FitRms", CPL_TYPE_DOUBLE);
1574 
1575  for (iquadrant = 0; iquadrant < nquadrants; ++iquadrant) {
1576  unsigned int quadrant = iquadrant + 1;
1577 
1578  /* Get window indices for the current detector quadrant */
1579  cpl_array *windex = muse_lingain_quadrant_get_windows(aWindowList, quadrant);
1580 
1581  if (!windex) {
1582  cpl_msg_warning(__func__, "No measurement windows defined for "
1583  "quadrant %hhu of IFU %u!", quadrant, aParams->nifu);
1584  cpl_table_set_invalid(gain, "Gain", iquadrant);
1585  cpl_table_set_invalid(gain, "FitRms", iquadrant);
1586  continue;
1587  }
1588 
1589  /* Compute the gain for the current detector quadrant, from the window *
1590  * indices and the gain measurements table. */
1591 
1592  muse_gain_fit_t qgain = {0., 0.};
1593  cpl_error_code ecode = muse_lingain_quadrant_get_gain(&qgain, aGainTable,
1594  windex, aParams);
1595  if (ecode != CPL_ERROR_NONE) {
1596  cpl_msg_warning(__func__, "Detector gain value for quadrant %hhu of "
1597  "IFU %u is invalid!", quadrant, aParams->nifu);
1598  cpl_table_set_invalid(gain, "Gain", iquadrant);
1599  cpl_table_set_invalid(gain, "FitRms", iquadrant);
1600  } else {
1601  cpl_msg_info(__func__, "Detector gain value for quadrant %hhu of "
1602  "IFU %u: %.6f [counts/ADU]", quadrant, aParams->nifu,
1603  qgain.gain);
1604  }
1605 
1606  cpl_table_set_double(gain, "Gain", iquadrant, qgain.gain);
1607  cpl_table_set_double(gain, "FitRms", iquadrant, qgain.rms);
1608  cpl_array_delete(windex);
1609  }
1610 
1611  return gain;
1612 }
1613 
1614 /*---------------------------------------------------------------------------*/
1629 /*---------------------------------------------------------------------------*/
1630 static cpl_table *
1631 muse_lingain_compute_nonlinearity(const cpl_table *aGainTable,
1632  cpl_table *aWindowList,
1633  const cpl_table *aGain,
1634  muse_lingain_params_t *aParams)
1635 {
1636  const unsigned char nquadrants = 4;
1637  unsigned char iquadrant;
1638 
1639  cpl_table *linearity = cpl_table_new(nquadrants);
1640  cpl_table_new_column(linearity, "Gain", CPL_TYPE_DOUBLE);
1641  cpl_table_new_column(linearity, "SignalMin", CPL_TYPE_DOUBLE);
1642  cpl_table_new_column(linearity, "SignalMax", CPL_TYPE_DOUBLE);
1643  cpl_table_new_column_array(linearity, "Coefficients", CPL_TYPE_DOUBLE,
1644  aParams->order + 1);
1645  cpl_table_new_column(linearity, "FitRms", CPL_TYPE_DOUBLE);
1646 
1647  cpl_table_set_column_unit(linearity, "Gain", "counts/ADU)");
1648  cpl_table_set_column_unit(linearity, "SignalMin", "log10(ADU)");
1649  cpl_table_set_column_unit(linearity, "SignalMax", "log10(ADU)");
1650 
1651  for (iquadrant = 0; iquadrant < nquadrants; ++iquadrant) {
1652  unsigned int quadrant = iquadrant + 1;
1653 
1654  /* Get window indices for the current detector quadrant */
1655  cpl_array *windex = muse_lingain_quadrant_get_windows(aWindowList, quadrant);
1656 
1657  if (!windex) {
1658  cpl_msg_warning(__func__, "No measurement windows defined for "
1659  "quadrant %hhu of IFU %u!", quadrant, aParams->nifu);
1660  continue;
1661  }
1662 
1663  /* Determine the detector non-linearity for the current detector *
1664  * quadrant, from the window and the gain measurements table. */
1665 
1666  int invalid = 0;
1667  double qgain = cpl_table_get_double(aGain, "Gain", iquadrant, &invalid);
1668 
1669  if (invalid) {
1670  cpl_msg_warning(__func__, "Got invalid gain for quadrant %hhu of IFU "
1671  "%u! Skipping non-linearity computation!", quadrant,
1672  aParams->nifu);
1673  cpl_array_delete(windex);
1674  continue;
1675  }
1676 
1677  muse_linearity_fit_t qlinearity = {NULL, {0., 0.}, 0.};
1678  cpl_error_code ecode =
1679  muse_lingain_quadrant_get_nonlinearity(&qlinearity, aGainTable,
1680  windex, qgain, aParams);
1681 
1682  if (ecode != CPL_ERROR_NONE) {
1683  cpl_msg_warning(__func__, "Computation of the detector non-linearity "
1684  "for quadrant %hhu of IFU %u failed!", quadrant,
1685  aParams->nifu);
1686  cpl_table_set_invalid(linearity, "Gain", iquadrant);
1687  cpl_table_set_invalid(linearity, "SignalMin", iquadrant);
1688  cpl_table_set_invalid(linearity, "SignalMax", iquadrant);
1689  cpl_table_set_invalid(linearity, "Coefficients", iquadrant);
1690  cpl_table_set_invalid(linearity, "FitRms", iquadrant);
1691  } else {
1692  cpl_msg_info(__func__, "RMS of residual non-linearity model over "
1693  "signal range [%.4f, %.4f] [log10(ADU)] for quadrant "
1694  "%hhu of IFU %u: %.6e", qlinearity.range[0],
1695  qlinearity.range[1], quadrant, aParams->nifu,
1696  qlinearity.rms);
1697 
1698  cpl_table_set_double(linearity, "Gain", iquadrant, qgain);
1699  cpl_table_set_double(linearity, "SignalMin", iquadrant,
1700  qlinearity.range[0]);
1701  cpl_table_set_double(linearity, "SignalMax", iquadrant,
1702  qlinearity.range[1]);
1703  cpl_table_set_array(linearity, "Coefficients", iquadrant,
1704  qlinearity.coefficients);
1705  cpl_table_set_double(linearity, "FitRms", iquadrant,
1706  qlinearity.rms);
1707  }
1708  muse_linearity_fit_clear(&qlinearity);
1709  cpl_array_delete(windex);
1710  }
1711 
1712  return linearity;
1713 }
1714 
1715 /*----------------------------------------------------------------------------*/
1723 /*----------------------------------------------------------------------------*/
1724 int
1725 muse_lingain_compute(muse_processing *aProcessing,
1726  muse_lingain_params_t *aParams)
1727 {
1728  /* Validate provided recipe parameters */
1729  if (!muse_lingain_validate_parameters(aParams)) {
1730  return -1;
1731  }
1732 
1733  /* Load trace table */
1734  cpl_table *trace = muse_processing_load_ctable(aProcessing, MUSE_TAG_TRACE_TABLE,
1735  aParams->nifu);
1736 
1737  if (!trace) {
1738  cpl_msg_error(__func__, "%s could not be loaded!", MUSE_TAG_TRACE_TABLE);
1739  return -1;
1740  }
1741 
1742  /* Load and pre-process the raw images. */
1743 
1744  /* XXX: This includes the bias subtraction, which will be applied to the *
1745  * two linearity bias frames too. In principle this is not an issue, since *
1746  * only their standard deviation is used as RON, but the bias subtraction *
1747  * adds noise. Thus, check whether the biases need a special treatment in *
1748  * muse_lingain_load_images()! */
1749 
1750  muse_imagelist *images = muse_lingain_load_images(aProcessing, aParams->nifu);
1751  if (!images) {
1752  cpl_msg_error(__func__, "Loading and basic processing of the raw input "
1753  "images failed for IFU %u!", aParams->nifu);
1754  cpl_table_delete(trace);
1755  return -1;
1756  }
1757 
1758  /* Create the sorted index table for the exposures of the linearity *
1759  * sequence. */
1760  cpl_table *exposures = muse_lingain_sort_images(images);
1761  if (!exposures) {
1762  cpl_msg_error(__func__, "Creating a sorted list of exposures failed "
1763  "for IFU %u!", aParams->nifu);
1764  muse_imagelist_delete(images);
1765  cpl_table_delete(trace);
1766  return -1;
1767  }
1768 
1769  /* Create list of measurement windows to be used for sampling the flat *
1770  * field data */
1771  cpl_table *windowlist = muse_lingain_create_windowlist(trace, exposures,
1772  images, aParams);
1773 
1774  if (!windowlist) {
1775  cpl_msg_error(__func__, "Creating the list of measurement windows failed "
1776  "for IFU %u!", aParams->nifu);
1777  cpl_table_delete(exposures);
1778  muse_imagelist_delete(images);
1779  cpl_table_delete(trace);
1780  return -1;
1781  }
1782  cpl_table_delete(trace);
1783 
1784  /* Measure per window read-out noise on the first 2 bias frames */
1785  cpl_array *ron = muse_lingain_compute_ron(windowlist, exposures, images,
1786  aParams);
1787  if (!ron) {
1788  cpl_msg_error(__func__, "Calculating the RON for individual measurement "
1789  "windows failed for IFU %u!", aParams->nifu);
1790  cpl_table_delete(windowlist);
1791  cpl_table_delete(exposures);
1792  muse_imagelist_delete(images);
1793  return -1;
1794  }
1795 
1796  /* Measure gain value for each measurement window and each flat field *
1797  * pair of the exposure series. */
1798  cpl_table *gaindata = muse_lingain_window_get_gain(windowlist, ron,
1799  exposures, images,
1800  aParams);
1801  if (!gaindata) {
1802  cpl_msg_error(__func__, "Calculating the gain for individual measurement "
1803  "windows failed for IFU %u!", aParams->nifu);
1804  cpl_array_delete(ron);
1805  cpl_table_delete(windowlist);
1806  cpl_table_delete(exposures);
1807  muse_imagelist_delete(images);
1808  return -1;
1809  }
1810  cpl_array_delete(ron);
1811  cpl_table_delete(exposures);
1812 
1813 
1814  /* Compute a gain value for each detector quadrant. */
1815  cpl_table *gain = muse_lingain_compute_gain(gaindata, windowlist, aParams);
1816 
1817  /* XXX: Check error handling here! Right now terminate the recipe if *
1818  * any gain value is invalid, but this may be relaxed with respect *
1819  * to operational needs. */
1820 
1821  if (cpl_table_count_invalid(gain, "Gain") != 0) {
1822  cpl_msg_error(__func__, "Calculating the detector gain failed for IFU %u!",
1823  aParams->nifu);
1824  cpl_table_delete(gain);
1825  cpl_table_delete(gaindata);
1826  cpl_table_delete(windowlist);
1827  muse_imagelist_delete(images);
1828  return -1;
1829  }
1830 
1831  /* Fit the residual detector non-linearity for each quadrant */
1832  cpl_table *linearity = muse_lingain_compute_nonlinearity(gaindata, windowlist,
1833  gain, aParams);
1834  if (cpl_table_has_invalid(linearity, "Coefficients")) {
1835  cpl_msg_error(__func__, "Computing the detector residual non-linearity "
1836  "failed for IFU %u!", aParams->nifu);
1837  cpl_table_delete(linearity);
1838  cpl_table_delete(gain);
1839  cpl_table_delete(gaindata);
1840  cpl_table_delete(windowlist);
1841  muse_imagelist_delete(images);
1842  return -1;
1843  }
1844 
1845  cpl_table_delete(gaindata);
1846  cpl_table_delete(windowlist);
1847 
1848  /* Create the linearity table product */
1849  cpl_propertylist *header =
1850  cpl_propertylist_duplicate(muse_imagelist_get(images, 0)->header);
1851 
1852  muse_imagelist_delete(images);
1853 
1854  cpl_propertylist_erase_regexp(header,
1855  "^SIMPLE$|^BITPIX$|^NAXIS|^EXTEND$|^XTENSION$|"
1856  "^DATASUM$|^DATAMIN$|^DATAMAX$|^DATAMD5$|"
1857  "^PCOUNT$|^GCOUNT$|^HDUVERS$|^BLANK$|"
1858  "^BZERO$|^BSCALE$|^BUNIT$|^CHECKSUM$|^INHERIT$|"
1859  "^PIPEFILE$|^ESO PRO ", 0);
1860 
1861  const unsigned char nquadrant = 4;
1862  unsigned char iquadrant;
1863  for (iquadrant = 0; iquadrant < nquadrant; ++iquadrant) {
1864  char keyword[KEYWORD_LENGTH];
1865  char comment[KEYWORD_LENGTH];
1866  unsigned char _iquadrant = iquadrant + 1;
1867  int ncoefficient = (int)cpl_table_get_column_depth(linearity,
1868  "Coefficients");
1869  snprintf(keyword, KEYWORD_LENGTH, "ESO DET OUT%d GAIN", _iquadrant);
1870  cpl_propertylist_update_double(header, keyword,
1871  cpl_table_get_double(gain, "Gain",
1872  iquadrant, NULL));
1873 
1874  /* XXX: Should the units of CONAD and GAIN be corrected in the *
1875  * product, or should they be wrong but consistent with the *
1876  * raw data? */
1877 #if 0
1878  cpl_propertylist_set_comment(header, keyword,
1879  "[e-/ADU] Conversion ADUs to electrons");
1880  snprintf(keyword, KEYWORD_LENGTH, "ESO DET OUT%d CONAD", _iquadrant);
1881  cpl_propertylist_set_comment(header, keyword,
1882  "[ADU/e-] Conversion electrons to ADUs");
1883 #endif
1884 
1885  snprintf(keyword, KEYWORD_LENGTH, MUSE_HDR_NONLINn_LLO, _iquadrant);
1886  cpl_propertylist_append_double(header, keyword,
1887  cpl_table_get_double(linearity, "SignalMin",
1888  iquadrant, NULL));
1889  cpl_propertylist_set_comment(header, keyword,
1890  "[log10(ADU)] Minimum signal used for fit");
1891 
1892  snprintf(keyword, KEYWORD_LENGTH, MUSE_HDR_NONLINn_LHI, _iquadrant);
1893  cpl_propertylist_append_double(header, keyword,
1894  cpl_table_get_double(linearity, "SignalMax",
1895  iquadrant, NULL));
1896  cpl_propertylist_set_comment(header, keyword,
1897  "[log10(ADU)] Maximum signal used for fit");
1898 
1899  snprintf(keyword, KEYWORD_LENGTH, MUSE_HDR_NONLINn_ORDER, _iquadrant);
1900  cpl_propertylist_append_double(header, keyword, ncoefficient - 1);
1901  cpl_propertylist_set_comment(header, keyword,
1902  "Order of the polynomial fit");
1903 
1904  const cpl_array *coefficients =
1905  cpl_table_get_array(linearity, "Coefficients", iquadrant);
1906  int icoefficient;
1907  for (icoefficient = 0; icoefficient < ncoefficient; ++icoefficient) {
1908  double c = cpl_array_get_double(coefficients, icoefficient, NULL);
1909  snprintf(keyword, KEYWORD_LENGTH, MUSE_HDR_NONLINn_COEFFo,
1910  _iquadrant, (unsigned char)icoefficient);
1911  cpl_propertylist_append_double(header, keyword, c);
1912 
1913  snprintf(comment, KEYWORD_LENGTH,
1914  "%d order coefficient of the polynomial fit", icoefficient);
1915  cpl_propertylist_set_comment(header, keyword, comment);
1916  }
1917 
1918  snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_GFITi_RMS, _iquadrant);
1919  cpl_propertylist_append_double(header, keyword,
1920  cpl_table_get_double(gain, "FitRms",
1921  iquadrant, NULL));
1922  snprintf(keyword, KEYWORD_LENGTH, QC_LINGAIN_NLFITi_RMS, _iquadrant);
1923  cpl_propertylist_append_double(header, keyword,
1924  cpl_table_get_double(linearity, "FitRms",
1925  iquadrant, NULL));
1926  }
1927  cpl_table_erase_column(linearity, "FitRms");
1928 
1929  muse_processing_save_table(aProcessing, aParams->nifu, linearity, header,
1930  MUSE_TAG_NONLINGAIN, MUSE_TABLE_TYPE_CPL);
1931 
1932  cpl_propertylist_delete(header);
1933  cpl_table_delete(linearity);
1934  cpl_table_delete(gain);
1935 
1936  return 0;
1937 } /* muse_lingain_compute() */
const char * muse_pfits_get_dpr_type(const cpl_propertylist *aHeaders)
find out the DPR type
Definition: muse_pfits.c:105
cpl_error_code muse_cplarray_erase_invalid(cpl_array *aArray)
Erase all invalid values from an array.
muse_imagelist * muse_basicproc_load(muse_processing *aProcessing, unsigned char aIFU, muse_basicproc_params *aBPars)
Load the raw input files from disk and do basic processing.
cpl_polynomial ** muse_trace_table_get_polys_for_slice(const cpl_table *aTable, const unsigned short aSlice)
construct polynomial from the trace table entry for the given slice
Structure definition for a collection of muse_images.
cpl_size * muse_quadrants_get_window(const muse_image *aImage, unsigned char aQuadrant)
Determine the data window of a given quadrant on the CCD.
cpl_image * data
the data extension
Definition: muse_image.h:46
Structure to hold the parameters of the muse_lingain recipe.
void muse_imagelist_delete(muse_imagelist *aList)
Free the memory of the MUSE image list.
double signalmin
Minimum signal value in log(ADU) used for the gain analysis and the non-linearity polynomial model...
double linearmin
Lower limit of desired linear range in log10(counts).
double sigma
Sigma value used for signal value clipping.
int nifu
IFU to handle. If set to 0, all IFUs are processed serially. If set to -1, all IFUs are processed in ...
Structure definition of MUSE three extension FITS file.
Definition: muse_image.h:40
void muse_basicproc_params_delete(muse_basicproc_params *aBPars)
Free a structure of basic processing parameters.
cpl_propertylist * header
the FITS header
Definition: muse_image.h:72
double signalbin
Size of a signal bin in log10(ADU) used for the gain analysis and the non-linearity polynomial model...
void muse_trace_polys_delete(cpl_polynomial *aPolys[])
Delete the multi-polynomial array created in relation to tracing.
muse_image * muse_imagelist_get(muse_imagelist *aList, unsigned int aIdx)
Get the muse_image of given list index.
double signalmax
Maximum signal value in log(ADU) used for the gain analysis and the non-linearity polynomial model...
double fluxtol
Tolerance value for the overall flux consistency check of a pair of flat fields. The value is the max...
int order
Order of the polynomial used to fit the non-linearity residuals.
double toffset
Exposure time offset in seconds to apply to linearity flat fields.
double gainsigma
Sigma value for gain value clipping.
double linearmax
Upper limit of desired linear range in log10(counts).
cpl_table * muse_processing_load_ctable(muse_processing *aProcessing, const char *aTag, unsigned char aIFU)
Load a CPL table according to its tag and IFU/channel number.
double ctsmin
Minimum signal value in log(counts) to consider for the non-linearity analysis.
double ctsbin
Size of a signal bin in log10(counts) used for the non-linearity analysis.
double muse_pfits_get_mjdobs(const cpl_propertylist *aHeaders)
find out the Julian Date of the observation
Definition: muse_pfits.c:346
cpl_error_code muse_image_reject_from_dq(muse_image *aImage)
Reject pixels of a muse_image depending on its DQ data.
Definition: muse_image.c:863
int xborder
Extra offset from the detector edge used for the selection of slices.
double ctsmax
Maximum signal value in log(counts) to consider for the non-linearity analysis.
cpl_frameset * inframes
muse_basicproc_params * muse_basicproc_params_new_from_propertylist(const cpl_propertylist *aHeader)
Create a structure of basic processing parameters from a FITS header.
double muse_pfits_get_exptime(const cpl_propertylist *aHeaders)
find out the exposure time
Definition: muse_pfits.c:382
int xgap
Extra offset from tracing edge.
cpl_error_code muse_processing_save_table(muse_processing *aProcessing, int aIFU, void *aTable, cpl_propertylist *aHeader, const char *aTag, muse_table_type aType)
Save a computed table to disk.
Structure of basic processing parameters.
double gainlimit
Minimum signal value [ADU] used for fitting the gain relation.
unsigned int size
int ybox
Size of windows along the traces of the slices.
cpl_frame * muse_frameset_find_master(const cpl_frameset *aFrames, const char *aTag, unsigned char aIFU)
find the master frame according to its CCD number and tag
Definition: muse_utils.c:505