MUSE Pipeline Reference Manual  2.1.1
muse_wavecalib.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  * This file is part of the MUSE Instrument Pipeline
5  * Copyright (C) 2005-2017 European Southern Observatory
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  */
21 
22 #ifdef HAVE_CONFIG_H
23 #include <config.h>
24 #endif
25 
26 /*----------------------------------------------------------------------------*
27  * Includes *
28  *----------------------------------------------------------------------------*/
29 #if HAVE_POPEN && HAVE_PCLOSE
30 #define _BSD_SOURCE /* force popen/pclose, mkdtemp definitions from stdio/stdlib */
31 #endif
32 #include <cpl.h>
33 #include <math.h>
34 #include <string.h>
35 
36 #include "muse_wavecalib.h"
37 #include "muse_instrument.h"
38 
39 #include "muse_combine.h"
40 #include "muse_cplwrappers.h"
41 #include "muse_data_format_z.h"
42 #include "muse_dfs.h"
43 #include "muse_tracing.h"
44 #include "muse_utils.h"
45 
46 /*----------------------------------------------------------------------------*
47  * Debugging Macros *
48  * Set these to 1 or higher for (lots of) debugging output *
49  *----------------------------------------------------------------------------*/
50 #define SEARCH_DEBUG 0 /* debugging in muse_wave_lines_search(), 1 or higher */
51 #define SEARCH_DEBUG_FILES 0 /* save different versions of the columns into *
52  * FITS files in muse_wave_lines_search() */
53 #define DEBUG_GAUSSFIT 0 /* debugging the Gaussian fit in muse_wave_lines_search() */
54 #define MUSE_WAVE_LINES_SEARCH_SHIFT_WARN 3.0 /* [pix] output debug message if *
55  * a shift of more than this occurs */
56 #define MUSE_WAVE_LINE_FIT_MAXSHIFT 2.0 /* [pix] don't use the fit when a shift *
57  * of more than this is detected */
58 #define MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT 0.25 /* [pix] maximum shift between *
59  * adjacent CCD columns */
60 
61 /*----------------------------------------------------------------------------*/
65 /*----------------------------------------------------------------------------*/
66 
69 /*----------------------------------------------------------------------------*/
94 /*----------------------------------------------------------------------------*/
96  { "lampno", CPL_TYPE_INT, "", "%d", "Number of the lamp", CPL_TRUE },
97  { "lampname", CPL_TYPE_STRING, "", "%s", "Name of the lamp", CPL_TRUE },
98  { "x", CPL_TYPE_DOUBLE, "pix", "%.2f", "x-position on CCD", CPL_TRUE },
99  { "y", CPL_TYPE_DOUBLE, "pix", "%.2f", "first-guess y-position on CCD", CPL_TRUE },
100  { "peak", CPL_TYPE_DOUBLE, "pix", "%g", "Peak of line", CPL_TRUE },
101  { "center", CPL_TYPE_DOUBLE, "pix", "%.4f", "Gaussian line center", CPL_TRUE },
102  { "cerr", CPL_TYPE_DOUBLE, "pix", "%.4f", "error estimate of line center", CPL_TRUE },
103  { "sigma", CPL_TYPE_DOUBLE, "pix", "%.3f", "Gaussian sigma", CPL_TRUE },
104  { "fwhm", CPL_TYPE_DOUBLE, "pix", "%.3f", "Gaussian FWHM", CPL_TRUE },
105  { "flux", CPL_TYPE_DOUBLE, "count", "%g", "Gaussian area (flux)", CPL_TRUE },
106  { "bg", CPL_TYPE_DOUBLE, "count", "%.2f", "background level", CPL_TRUE },
107  { "mse", CPL_TYPE_DOUBLE, "", "%e", "mean squared error", CPL_TRUE },
108  { "lambda", CPL_TYPE_DOUBLE, "Angstrom", "%9.3f",
109  "identified wavelength of the line", CPL_TRUE },
110  { NULL, 0, NULL, NULL, NULL, CPL_FALSE }
111 };
112 
113 /*----------------------------------------------------------------------------*/
117 /*----------------------------------------------------------------------------*/
119  { "slice", CPL_TYPE_INT, "", "%02d", "slice number", CPL_TRUE},
120  { "iteration", CPL_TYPE_INT, "", "%d", "iteration", CPL_TRUE},
121  { "x", CPL_TYPE_INT, "pix", "%04d", "x-position on CCD", CPL_TRUE},
122  { "y", CPL_TYPE_FLOAT, "pix", "%8.3f", "y-position on CCD", CPL_TRUE},
123  { "lambda", CPL_TYPE_FLOAT, "Angstrom", "%8.3f", "wavelength", CPL_TRUE},
124  { "residual", CPL_TYPE_DOUBLE, "Angstrom", "%.4e", "residual at this point", CPL_TRUE},
125  { "rejlimit", CPL_TYPE_DOUBLE, "Angstrom", "%.4e",
126  "rejection limit for this iteration", CPL_TRUE},
127  { NULL, 0, NULL, NULL, NULL, CPL_FALSE }
128 };
129 
130 /* corresponds to muse_wave_weighting_type, *
131  * keep in sync with those values! */
132 const char *muse_wave_weighting_string[] = {
133  "uniform",
134  "centroid error",
135  "per-line RMS scatter",
136  "centroid error plus per-line RMS scatter"
137 };
138 
139 /*----------------------------------------------------------------------------*/
156 /*----------------------------------------------------------------------------*/
158 muse_wave_params_new(cpl_propertylist *aHeader)
159 {
160  muse_wave_params *p = cpl_malloc(sizeof(muse_wave_params));
161  /* set defaults */
162  p->xorder = 2;
163  p->yorder = 6;
164  p->detsigma = 1.;
165  p->ddisp = 0.05; /* from 1.20...1.30 Angstrom/pixel */
166  p->tolerance = 0.1;
167  p->linesigma = -1.;
168  p->rflag = CPL_FALSE; /* do not create a residuals table by default */
169  p->residuals = NULL;
170  p->fitsigma = -1.;
172  p->targetrms = 0.03;
173  if (aHeader) { /* otherwise p->mode stays zero which means WFM-NOAO-N */
174  p->mode = muse_pfits_get_mode(aHeader);
175  cpl_msg_debug(__func__, "Set mode %s (%d)", muse_pfits_get_insmode(aHeader),
176  p->mode);
177  }
178  return p;
179 }
180 
181 /*----------------------------------------------------------------------------*/
191 /*----------------------------------------------------------------------------*/
192 void
194 {
195  if (!aParams) {
196  return;
197  }
198  cpl_table_delete(aParams->residuals);
199  aParams->residuals = NULL;
200  memset(aParams, 0, sizeof(muse_wave_params)); /* null out everything */
201  cpl_free(aParams);
202 }
203 
204 /*----------------------------------------------------------------------------*/
216 /*----------------------------------------------------------------------------*/
217 static void
218 muse_wave_calib_qc_peaks(cpl_table *aLines, cpl_propertylist *aHeader,
219  const unsigned short aSlice)
220 {
221  char keyword[KEYWORD_LENGTH];
222  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_PEAK_MEAN, aSlice);
223  cpl_propertylist_append_float(aHeader, keyword,
224  cpl_table_get_column_mean(aLines, "peak"));
225  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_PEAK_STDEV, aSlice);
226  cpl_propertylist_append_float(aHeader, keyword,
227  cpl_table_get_column_stdev(aLines, "peak"));
228  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_PEAK_MIN, aSlice);
229  cpl_propertylist_append_float(aHeader, keyword,
230  cpl_table_get_column_min(aLines, "peak"));
231  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_PEAK_MAX, aSlice);
232  cpl_propertylist_append_float(aHeader, keyword,
233  cpl_table_get_column_max(aLines, "peak"));
234  if (aSlice != 20) {
235  return;
236  }
237  /* if we are in slice 20 (a slice that is near the middle of a stack, so
238  * should never be vignetted loop through all possible lamps */
239  int n, nlamps = muse_pfits_get_lampnum(aHeader);
240  for (n = 1; n < nlamps; n++) {
241  /* select and extract all table entries matching the lamp */
242  cpl_table_unselect_all(aLines);
243  cpl_table_or_selected_int(aLines, "lampno", CPL_EQUAL_TO, n);
244  cpl_table *lamplines = cpl_table_extract_selected(aLines);
245  if (cpl_table_get_nrow(lamplines) < 1) {
246  cpl_table_delete(lamplines);
247  continue;
248  }
249  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LAMPl_LINES_PEAK_MEAN,
250  aSlice, n);
251  cpl_propertylist_append_float(aHeader, keyword,
252  cpl_table_get_column_mean(lamplines, "peak"));
253  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LAMPl_LINES_PEAK_STDEV,
254  aSlice, n);
255  cpl_propertylist_append_float(aHeader, keyword,
256  cpl_table_get_column_stdev(lamplines, "peak"));
257  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LAMPl_LINES_PEAK_MAX,
258  aSlice, n);
259  cpl_propertylist_append_float(aHeader, keyword,
260  cpl_table_get_column_max(lamplines, "peak"));
261  cpl_table_delete(lamplines);
262  } /* for n (all lamps) */
263 } /* muse_wave_calib_qc_peaks() */
264 
265 /*----------------------------------------------------------------------------*/
274 /*----------------------------------------------------------------------------*/
275 static void
276 muse_wave_calib_qc_fwhm_old(cpl_table *aLines, cpl_propertylist *aHeader,
277  cpl_vector *aRefLam, const unsigned short aSlice)
278 {
279  int nlines = cpl_table_get_nrow(aLines);
280 
281  /* track the minimum and maximum resolutions and their wavelengths */
282  double Rmin = DBL_MAX, Rmax = -DBL_MAX, wlmin = DBL_MAX, wlmax = -DBL_MAX;
283 
284  /* convert the properties of the relevant lines from the respective *
285  * table columns into vectors for easy statistics computation */
286  cpl_vector *fwhms = cpl_vector_new(nlines),
287  *resol = cpl_vector_new(cpl_vector_get_size(aRefLam));
288  int i, iresol = 0;
289  for (i = 0; i < nlines; i++) {
290  /* FWHM is directly accessible, but needs to be converted to Angstrom */
291  cpl_errorstate prestate = cpl_errorstate_get();
292  double fwhm = cpl_table_get(aLines, "fwhm", i, NULL), /* in [pix] */
293  sampling = kMuseSpectralSamplingA, /* sensible default in [A/pix] */
294  lambda = cpl_table_get(aLines, "lambda", i, NULL),
295  s1 = (lambda - cpl_table_get(aLines, "lambda", i - 1, NULL))
296  / (cpl_table_get(aLines, "center", i, NULL)
297  - cpl_table_get(aLines, "center", i - 1, NULL)),
298  s2 = (cpl_table_get(aLines, "lambda", i + 1, NULL)
299  - cpl_table_get(aLines, "lambda", i, NULL))
300  / (cpl_table_get(aLines, "center", i + 1, NULL)
301  - cpl_table_get(aLines, "center", i, NULL));
302  if (!cpl_errorstate_is_equal(prestate)) {
303  cpl_errorstate_set(prestate); /* reset "Access beyond boundaries" errors */
304  }
305  if (i == 0) { /* no lower one */
306  sampling = s2;
307  } else if (i == nlines - 1) { /* no higher one */
308  sampling = s1;
309  } else {
310  sampling = (s1 + s2) / 2.;
311  }
312  fwhm *= sampling; /* now in [A] */
313  cpl_vector_set(fwhms, i, fwhm);
314  /* spectral resolution R for this line R = lambda / dlambda */
315  double R = lambda / fwhm;
316  /* compare wavelength to those lines in the FWHM reference list */
317  if (fabs(cpl_vector_get(aRefLam, cpl_vector_find(aRefLam, lambda)) - lambda)
318  < FLT_EPSILON) {
319  if (R < Rmin) {
320  Rmin = R;
321  wlmin = lambda;
322  }
323  if (R > Rmax) {
324  Rmax = R;
325  wlmax = lambda;
326  }
327  cpl_vector_set(resol, iresol++, R);
328  }
329  } /* for i (all arc lines) */
330  cpl_vector_set_size(resol, iresol);
331 
332  char keyword[KEYWORD_LENGTH];
333  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MEAN, aSlice);
334  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_mean(fwhms));
335  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_STDEV, aSlice);
336  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_stdev(fwhms));
337  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MIN, aSlice);
338  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_min(fwhms));
339  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MAX, aSlice);
340  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_max(fwhms));
341  cpl_vector_delete(fwhms);
342 
343  cpl_msg_debug(__func__, "Average spectral resolution in IFU %hhu is R=%4.0f, "
344  "ranging from %4.0f (at %6.1fA) to %4.0f (at %6.1fA)",
345  muse_utils_get_ifu(aHeader), cpl_vector_get_mean(resol),
346  Rmin, wlmin, Rmax, wlmax);
347  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_RESOL, aSlice);
348  cpl_propertylist_append_float(aHeader, keyword, cpl_vector_get_mean(resol));
349  cpl_vector_delete(resol);
350 } /* muse_wave_calib_qc_fwhm_old() */
351 
352 /*----------------------------------------------------------------------------*/
360 /*----------------------------------------------------------------------------*/
361 static void
362 muse_wave_calib_qc_fwhm(cpl_table *aFWHM, cpl_propertylist *aHeader,
363  const unsigned short aSlice)
364 {
365  /* compute the spectral resolution at each wavelength, as *
366  * R = lambda / fwhm *
367  * where both lambda and fwhm are [Angstrom] */
368  cpl_table_duplicate_column(aFWHM, "R", aFWHM, "lambda");
369  cpl_table_divide_columns(aFWHM, "R", "fwhm");
370  cpl_table_set_column_unit(aFWHM, "R", "");
371 
372 #if 0 /* AIT Gaussian FWHM */
373  /* range < 600 nm (as in PR5) */
374  cpl_table_unselect_all(aFWHM);
375  cpl_table_or_selected_double(aFWHM, "lambda", CPL_LESS_THAN, 6000.);
376  cpl_table *tblue = cpl_table_extract_selected(aFWHM);
377  /* range > 800 nm (as in PR5) */
378  cpl_table_unselect_all(aFWHM);
379  cpl_table_or_selected_double(aFWHM, "lambda", CPL_GREATER_THAN, 8000.);
380  cpl_table *tred = cpl_table_extract_selected(aFWHM);
381  /* range 600 - 800 nm (as in PR5) */
382  cpl_table_unselect_all(aFWHM);
383  cpl_table_or_selected_double(aFWHM, "lambda", CPL_NOT_LESS_THAN, 6000.);
384  cpl_table_and_selected_double(aFWHM, "lambda", CPL_NOT_GREATER_THAN, 8000.);
385  cpl_table *tgreen = cpl_table_extract_selected(aFWHM);
386 #if 0
387  printf("all:\n");
388  cpl_table_dump(aFWHM, 0, 100000, stdout);
389  fflush(stdout);
390  printf("blue:\n");
391  cpl_table_dump(tblue, 0, 100000, stdout);
392  fflush(stdout);
393  printf("green:\n");
394  cpl_table_dump(tgreen, 0, 100000, stdout);
395  fflush(stdout);
396  printf("red:\n");
397  cpl_table_dump(tred, 0, 100000, stdout);
398  fflush(stdout);
399 #endif
400 #endif /* AIT Gaussian FWHM */
401 
402  /* fill the QC parameters */
403  char keyword[KEYWORD_LENGTH];
404  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_RESOL, aSlice);
405  double rmean = cpl_table_get_column_mean(aFWHM, "R");
406  cpl_propertylist_update_float(aHeader, keyword, rmean);
407 
408  double fmean = cpl_table_get_column_mean(aFWHM, "fwhm"),
409  fstdev = cpl_table_get_column_stdev(aFWHM, "fwhm"),
410  flo = cpl_table_get_column_min(aFWHM, "fwhm"),
411  fhi = cpl_table_get_column_max(aFWHM, "fwhm");
412  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MEAN, aSlice);
413  cpl_propertylist_update_float(aHeader, keyword, fmean);
414  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_STDEV, aSlice);
415  cpl_propertylist_update_float(aHeader, keyword, fstdev);
416  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MIN, aSlice);
417  cpl_propertylist_update_float(aHeader, keyword, flo);
418  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_FWHM_MAX, aSlice);
419  cpl_propertylist_update_float(aHeader, keyword, fhi);
420 
421 #if 0 /* AIT Gaussian FWHM */
422  /* output all the stuff as debug messages */
423  cpl_msg_debug(__func__, "Gaussian FWHM [%s]:\n"
424  "\tFWHM all (%d values)\t%.3f +/- %.3f (%.3f) %.3f...%.3f\n"
425  "\tFWHM blue (%d values)\t%.3f +/- %.3f (%.3f)\n"
426  "\tFWHM green (%d values)\t%.3f +/- %.3f (%.3f)\n"
427  "\tFWHM red (%d values)\t%.3f +/- %.3f (%.3f)",
428  cpl_table_get_column_unit(aFWHM, "fwhm"),
429  (int)cpl_table_get_nrow(aFWHM), fmean, fstdev, flo, fhi,
430  cpl_table_get_column_median(aFWHM, "fwhm"),
431  (int)cpl_table_get_nrow(tblue),
432  cpl_table_get_column_mean(tblue, "fwhm"),
433  cpl_table_get_column_stdev(tblue, "fwhm"),
434  cpl_table_get_column_median(tblue, "fwhm"),
435  (int)cpl_table_get_nrow(tgreen),
436  cpl_table_get_column_mean(tgreen, "fwhm"),
437  cpl_table_get_column_stdev(tgreen, "fwhm"),
438  cpl_table_get_column_median(tgreen, "fwhm"),
439  (int)cpl_table_get_nrow(tred),
440  cpl_table_get_column_mean(tred, "fwhm"),
441  cpl_table_get_column_stdev(tred, "fwhm"),
442  cpl_table_get_column_median(tred, "fwhm"));
443  cpl_msg_debug(__func__, "Gaussian spectral resolution R:\n"
444  "\tR all (%d values)\t%.1f +/- %.1f (%.1f) %.1f...%.1f\n"
445  "\tR blue (%d values)\t%.1f +/- %.1f (%.1f)\n"
446  "\tR green (%d values)\t%.1f +/- %.1f (%.1f)\n"
447  "\tR red (%d values)\t%.1f +/- %.1f (%.1f)",
448  (int)cpl_table_get_nrow(aFWHM), rmean,
449  cpl_table_get_column_stdev(aFWHM, "R"),
450  cpl_table_get_column_median(aFWHM, "R"),
451  cpl_table_get_column_min(aFWHM, "R"),
452  cpl_table_get_column_max(aFWHM, "R"),
453  (int)cpl_table_get_nrow(tblue),
454  cpl_table_get_column_mean(tblue, "R"),
455  cpl_table_get_column_stdev(tblue, "R"),
456  cpl_table_get_column_median(tblue, "R"),
457  (int)cpl_table_get_nrow(tgreen),
458  cpl_table_get_column_mean(tgreen, "R"),
459  cpl_table_get_column_stdev(tgreen, "R"),
460  cpl_table_get_column_median(tgreen, "R"),
461  (int)cpl_table_get_nrow(tred),
462  cpl_table_get_column_mean(tred, "R"),
463  cpl_table_get_column_stdev(tred, "R"),
464  cpl_table_get_column_median(tred, "R"));
465  cpl_table_delete(tblue);
466  cpl_table_delete(tgreen);
467  cpl_table_delete(tred);
468 #endif /* AIT Gaussian FWHM */
469  cpl_table_erase_column(aFWHM, "R");
470 } /* muse_wave_calib_qc_fwhm() */
471 
472 /*----------------------------------------------------------------------------*/
484 /*----------------------------------------------------------------------------*/
485 static void
486 muse_wave_calib_qc_lambda(muse_image *aImage, const unsigned short aSlice,
487  cpl_polynomial **aTrace, cpl_polynomial *aFit)
488 {
489  int ny = cpl_image_get_size_y(aImage->data);
490  unsigned char ifu = muse_utils_get_ifu(aImage->header);
491 
492  /* wavelength differences at bottom and top of each slice as QC parameters */
493  double xbotl = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_LEFT], 1, NULL),
494  xbotr = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_RIGHT], 1, NULL),
495  xtopl = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_LEFT], ny, NULL),
496  xtopr = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_RIGHT], ny, NULL);
497  cpl_vector *pos = cpl_vector_new(2);
498  cpl_vector_set(pos, 0, xbotl);
499  cpl_vector_set(pos, 1, 1);
500  double wlbotl = cpl_polynomial_eval(aFit, pos);
501  cpl_vector_set(pos, 0, xbotr);
502  cpl_vector_set(pos, 1, 1);
503  double wlbotr = cpl_polynomial_eval(aFit, pos);
504  cpl_vector_set(pos, 0, xtopl);
505  cpl_vector_set(pos, 1, ny);
506  double wltopl = cpl_polynomial_eval(aFit, pos);
507  cpl_vector_set(pos, 0, xtopr);
508  cpl_vector_set(pos, 1, ny);
509  double wltopr = cpl_polynomial_eval(aFit, pos);
510 #if 0
511  cpl_msg_debug(__func__, "Wavelengths at slice corners in slice %hu of IFU %hhu: "
512  "botl(%.2f,1)=%f, botr(%.2f,1)=%f, topl(%.2f,%d)=%f, topr(%.2f,%d)"
513  "=%f", aSlice, ifu, xbotl, wlbotl, xbotr, wlbotr,
514  xtopl, ny, wltopl, xtopr, ny, wltopr);
515 #endif
516  char *keyword = cpl_sprintf(QC_WAVECAL_SLICEj_DWLEN_BOT, aSlice);
517  cpl_propertylist_append_float(aImage->header, keyword, wlbotl - wlbotr);
518  cpl_free(keyword);
519  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_DWLEN_TOP, aSlice);
520  cpl_propertylist_append_float(aImage->header, keyword, wltopl - wltopr);
521  cpl_free(keyword);
522 
523 #define DWLEN_WARN_LIMIT_N 6.5 /* 5.7 Angstrom differences can happen... */
524 #define DWLEN_WARN_LIMIT_E 5.9 /* ... at the bottom in both modes */
525  double limit = DWLEN_WARN_LIMIT_E;
526  cpl_errorstate prestate = cpl_errorstate_get();
527  if (muse_pfits_get_mode(aImage->header) != MUSE_MODE_WFM_NONAO_E &&
528  muse_pfits_get_mode(aImage->header) != MUSE_MODE_WFM_AO_E) {
529  limit = DWLEN_WARN_LIMIT_N;
530  }
531  cpl_errorstate_set(prestate); /* swallow possible error about missing INS MODE */
532  if (fabs(wlbotl - wlbotr) > limit) {
533  cpl_msg_warning(__func__, "Wavelength differences at bottom of slice %hu "
534  "of IFU %hhu are large (%f Angstrom)!", aSlice, ifu,
535  wlbotl - wlbotr);
536  }
537  if (fabs(wltopl - wltopr) > DWLEN_WARN_LIMIT_E) { /* strict limit at red end */
538  cpl_msg_warning(__func__, "Wavelength differences at top of slice %hu of IFU "
539  "%hhu are large (%f Angstrom)!", aSlice, ifu, wltopl - wltopr);
540  }
541 
542  /* generate some arbitrary evaluation point, approx. at *
543  * the slice center, and compute the wavelength there. */
544  const double yc = (1. + ny) / 2.,
545  xc = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_CENTER], yc, NULL);
546  cpl_vector_set(pos, 0, xc);
547  cpl_vector_set(pos, 1, yc);
548  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_WLPOS, aSlice);
549  cpl_propertylist_append_float(aImage->header, keyword, yc);
550  cpl_free(keyword);
551  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_WLEN, aSlice);
552  cpl_propertylist_append_float(aImage->header, keyword,
553  cpl_polynomial_eval(aFit, pos));
554  cpl_free(keyword);
555  cpl_vector_delete(pos);
556 } /* muse_wave_calib_qc_lambda() */
557 
558 /*----------------------------------------------------------------------------*/
571 /*----------------------------------------------------------------------------*/
572 static void
573 muse_wave_calib_output_summary(const cpl_propertylist *aHeader,
574  const cpl_table *aWave)
575 {
576  unsigned char ifu = muse_utils_get_ifu(aHeader);
577  if (!aWave) {
578  cpl_msg_warning(__func__, "Wavelength solution missing for IFU %hhu, no "
579  "summary!", ifu);
580  return;
581  }
582  cpl_vector *found = cpl_vector_new(kMuseSlicesPerCCD),
583  *ident = cpl_vector_new(kMuseSlicesPerCCD),
584  *used = cpl_vector_new(kMuseSlicesPerCCD),
585  *resolution = cpl_vector_new(kMuseSlicesPerCCD);
586  unsigned short islice;
587  for (islice = 0; islice < kMuseSlicesPerCCD; islice++) {
588  char keyword[KEYWORD_LENGTH];
589  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_NDET,
590  (unsigned short)(islice + 1));
591  cpl_vector_set(found, islice, cpl_propertylist_get_int(aHeader, keyword));
592  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_LINES_NID,
593  (unsigned short)(islice + 1));
594  cpl_vector_set(ident, islice, cpl_propertylist_get_int(aHeader, keyword));
595  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_FIT_NLINES,
596  (unsigned short)(islice + 1));
597  cpl_vector_set(used, islice, cpl_propertylist_get_int(aHeader, keyword));
598  snprintf(keyword, KEYWORD_LENGTH, QC_WAVECAL_SLICEj_RESOL,
599  (unsigned short)(islice + 1));
600  cpl_vector_set(resolution, islice, cpl_propertylist_get_float(aHeader, keyword));
601  } /* for islice (all kMuseSlicesPerCCD) */
602  cpl_msg_info(__func__, "Summary of wavelength solution for IFU %hhu:\n"
603  "\tDetections per slice: %d ... %d\n"
604  "\tIdentified lines per slice: %d ... %d\n"
605  "\tLines per slice used in fit: %d ... %d\n"
606  "\tRMS of fit [Angstrom]: %5.3f +/- %5.3f (%5.3f ... %5.3f)\n"
607  "\tMean spectral resolution R: %6.1f +/- %5.1f", ifu,
608  (int)cpl_vector_get_min(found), (int)cpl_vector_get_max(found),
609  (int)cpl_vector_get_min(ident), (int)cpl_vector_get_max(ident),
610  (int)cpl_vector_get_min(used), (int)cpl_vector_get_max(used),
611  sqrt(cpl_table_get_column_mean(aWave, MUSE_WAVECAL_TABLE_COL_MSE)),
612  sqrt(cpl_table_get_column_stdev(aWave, MUSE_WAVECAL_TABLE_COL_MSE)),
613  sqrt(cpl_table_get_column_min(aWave, MUSE_WAVECAL_TABLE_COL_MSE)),
614  sqrt(cpl_table_get_column_max(aWave, MUSE_WAVECAL_TABLE_COL_MSE)),
615  cpl_vector_get_mean(resolution), cpl_vector_get_stdev(resolution));
616  cpl_vector_delete(found);
617  cpl_vector_delete(ident);
618  cpl_vector_delete(used);
619  cpl_vector_delete(resolution);
620 } /* muse_wave_calib_output_summary() */
621 
622 /*----------------------------------------------------------------------------*/
668 /*----------------------------------------------------------------------------*/
669 cpl_table *
670 muse_wave_calib(muse_image *aImage, cpl_table *aTrace, cpl_table *aLinelist,
671  muse_wave_params *aParams)
672 {
673  if (!aImage) {
674  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "arc image missing!");
675  return NULL;
676  }
677  unsigned char ifu = muse_utils_get_ifu(aImage->header);
678  /* variance always has to be larger than zero */
679  double minstat = cpl_image_get_min(aImage->stat);
680  if (minstat <= 0.) {
681  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_INPUT, "arc image %d does"
682  " not have valid STAT extension in IFU %hhu (minimum "
683  "is %e)!", 0, ifu, minstat);
684  return NULL;
685  }
686 
687  if (!aTrace) {
688  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "trace table missing "
689  "for IFU %hhu, cannot create wavelength calibration!",
690  ifu);
691  return NULL;
692  } else if (cpl_table_get_nrow(aTrace) != kMuseSlicesPerCCD) {
693  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "trace table "
694  "not valid for this dataset of IFU %hhu!", ifu);
695  return NULL;
696  }
697 
698  if (!aLinelist) {
699  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "no arc line list "
700  "supplied for IFU %hhu!", ifu);
701  return NULL;
702  }
703  if (!aParams) {
704  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "wavelength "
705  "calibration parameters missing for IFU %hhu!", ifu);
706  return NULL;
707  }
709  cpl_error_set_message(__func__, CPL_ERROR_UNSUPPORTED_MODE,
710  "unknown weighting scheme for IFU %hhu", ifu);
711  return NULL;
712  }
713 
714  /* do not support smaller mock datasets any more */
715  int nx = cpl_image_get_size_x(aImage->data),
716  ny = cpl_image_get_size_y(aImage->data);
717  if (ny < 4000) {
718  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_INPUT, "this dataset of "
719  "IFU %hhu is too small (%dx%d pix) to be supported",
720  ifu, nx, ny);
721  return NULL;
722  }
723 
724  cpl_msg_info(__func__, "Using polynomial orders %hu (x) and %hu (y), %s "
725  "weighting, assuming initial sampling of %.3f +/- %.3f Angstrom"
726  "/pix, in IFU %hhu", aParams->xorder, aParams->yorder,
727  muse_wave_weighting_string[aParams->fitweighting],
728  kMuseSpectralSamplingA, aParams->ddisp, ifu);
729 
730  cpl_vector *vreflam = muse_wave_lines_get(aLinelist, 5, 0.);
731  if (!vreflam) {
732  cpl_error_set_message(__func__, CPL_ERROR_BAD_FILE_FORMAT, "could not "
733  "create list of FWHM reference arc wavelengths for "
734  "IFU %hhu", ifu);
735  return NULL;
736  }
737 
738  cpl_propertylist_erase_regexp(aImage->header, "^ESO QC", 0);
739  cpl_table *wavecaltable = NULL;
740 
741  int debug = getenv("MUSE_DEBUG_WAVECAL")
742  ? atoi(getenv("MUSE_DEBUG_WAVECAL")) : 0;
743  char fn_debug[100];
744  FILE *fp_debug = NULL;
745  if (debug >= 3) {
746  snprintf(fn_debug, 99, "MUSE_DEBUG_WAVE_LINES-%02hhu.ascii", ifu);
747  cpl_msg_info(__func__, "Will write all single line fits to \"%s\"", fn_debug);
748  fp_debug = fopen(fn_debug, "w");
749  if (fp_debug) {
750  fprintf(fp_debug, "#slice x y lambda lambdaerr\n");
751  }
752  }
753 
754  /* loop over all slices */
755  unsigned short islice;
756  for (islice = 0; islice < kMuseSlicesPerCCD; islice++) {
757  if (debug > 0) {
758  printf("\n\nSlice %d of IFU %hhu\n", (int)islice + 1, ifu);
759  fflush(stdout);
760  }
761  /* get the tracing polynomials for this slice */
762  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
763  islice + 1);
764  if (!ptrace) {
765  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: tracing polynomials "
766  "missing!", (int)islice + 1, ifu);
767  continue;
768  }
769 
770  /* detect all lines in the center of the slice */
771  int imid = lround(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER],
772  ny/2., NULL));
773  if (imid < 1 || imid > nx) {
774  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: faulty trace polynomial"
775  " detected", (int)islice + 1, ifu);
776  muse_trace_polys_delete(ptrace);
777  continue; /* next slice */
778  }
779  /* create muse_image from the spectrum in which we want to search lines */
780  const int kWidth = 3;
781  int k, kcol1 = imid - kWidth/2, kcol2 = kcol1 + kWidth;
782  muse_imagelist *collist = muse_imagelist_new();
783  for (k = kcol1; k <= kcol2; k++) {
784  muse_image *column = muse_image_new();
785  column->data = cpl_image_extract(aImage->data, k, 1, k, ny);
786  column->dq = cpl_image_extract(aImage->dq, k, 1, k, ny);
787  column->stat = cpl_image_extract(aImage->stat, k, 1, k, ny);
788  column->header = cpl_propertylist_new();
789  muse_imagelist_set(collist, column, k - kcol1);
790  }
791  muse_image *column = muse_combine_median_create(collist);
792  cpl_propertylist_append_string(column->header, "BUNIT",
793  muse_pfits_get_bunit(aImage->header));
794  muse_imagelist_delete(collist);
795 
796  /* now search, measure, and store the lines */
797  cpl_table *detlines = muse_wave_lines_search(column, aParams->detsigma,
798  islice + 1, ifu);
799  if (!detlines) {
800  cpl_error_set_message(__func__, CPL_ERROR_DATA_NOT_FOUND, "problem when "
801  "searching for arc lines in slice %d of IFU %hhu, "
802  "columns %d..%d", (int)islice + 1, ifu, kcol1, kcol2);
803  muse_image_delete(column);
804  cpl_vector_delete(vreflam);
805  muse_trace_polys_delete(ptrace);
806  cpl_table_delete(wavecaltable);
807  cpl_table_delete(aParams->residuals);
808  aParams->residuals = NULL;
809  return NULL;
810  }
811  /* clean up only here, to not disturb error message above */
812  muse_image_delete(column);
813  if (debug >= 2) {
814  printf("Detected arc lines in slice %d of IFU %hhu:\n", (int)islice + 1, ifu);
815  cpl_table_dump(detlines, 0, cpl_table_get_nrow(detlines), stdout);
816  fflush(stdout);
817  }
818 
819  /* identify the lines that we detected, and see if there are enough *
820  * for the polynomial order in y-direction to fit a polynomial */
821  int nfound = cpl_table_get_nrow(detlines);
822  /* get all arc lines, taking into account the detection limit */
823  double minflux = cpl_table_get_column_min(detlines, "flux") * 1.15;
824  cpl_vector *vlambda = muse_wave_lines_get(aLinelist, 1, minflux);
825  muse_wave_lines_identify(detlines, vlambda, aParams);
826  cpl_vector_delete(vlambda);
827  int nlines = cpl_table_get_nrow(detlines);
828  cpl_msg_debug(__func__, "Identified %d of %d arc lines in slice %d of IFU "
829  "%hhu (column %d, %d..%d)", nlines, nfound, (int)islice + 1, ifu,
830  imid, kcol1, kcol2);
831  if (debug >= 2) {
832  printf("Identified arc lines with wavelengths in slice %d of IFU %hhu:\n",
833  (int)islice + 1, ifu);
834  cpl_table_dump(detlines, 0, cpl_table_get_nrow(detlines), stdout);
835  fflush(stdout);
836  }
837 
838  /* the first QC parameter does even make sense without lines */
839  char *keyword = cpl_sprintf(QC_WAVECAL_SLICEj_LINES_NDET,
840  (unsigned short)(islice + 1));
841  cpl_propertylist_append_int(aImage->header, keyword, nfound);
842  cpl_free(keyword);
843 
844  if (nlines < aParams->yorder + 1) {
845  /* could apparently not identify enough lines, or an error occured */
846  cpl_msg_error(__func__, "Could not identify enough arc lines in slice %d"
847  " of IFU %hhu (%d of %d, required %d)", (int)islice + 1, ifu,
848  nlines, nfound, (int)aParams->yorder + 1);
849  muse_trace_polys_delete(ptrace);
850  cpl_table_delete(detlines);
851  continue; /* work on next slice immediately */
852  }
853 
854  /* the next set of QC parameters */
855  muse_wave_calib_qc_peaks(detlines, aImage->header, islice + 1);
856  muse_wave_calib_qc_fwhm_old(detlines, aImage->header, vreflam, islice + 1);
857  /* above we wrote the number of detected lines, now is the number of *
858  * arc lines really used in the fit, i.e. the identified ones */
859  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_LINES_NID, (unsigned short)(islice + 1));
860  cpl_propertylist_append_int(aImage->header, keyword, nlines);
861  cpl_free(keyword);
862 
863  /* positions matrix and wavelengths vector to be used for the 2D polynomial *
864  * fit; just set and initial size of 1, it has to be resized with every line */
865  cpl_matrix *xypos = cpl_matrix_new(2, 1);
866  cpl_vector *lambdas = cpl_vector_new(1),
867  *dlambdas = NULL;
868  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
869  dlambdas = cpl_vector_new(1);
870  }
871  int ientry = 0; /* index for these two structures */
872 
873  /* for all identified arc lines, work on the original input data */
874  int j;
875  for (j = 0; j < nlines; j++) {
876  /* convenient access to some properties of this identified line */
877  double lambda = cpl_table_get(detlines, "lambda", j, NULL);
878  double ypos = cpl_table_get(detlines, "center", j, NULL);
879  /* better fix the Gaussian sigmas, otherwise *
880  * the fit might run astray in low S/N cases */
881  double sigma = cpl_table_get(detlines, "sigma", j, NULL);
882  int n = 0,
883  halfwidth = 2.*cpl_table_get(detlines, "fwhm", j, NULL); /* 2*FWHM */
884  /* get both slice edges and the center */
885  double dleft = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT],
886  ypos, NULL),
887  dright = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT],
888  ypos, NULL),
889  dmid = (dleft + dright) / 2.;
890  int ileft = ceil(dleft),
891  iright = floor(dright);
892 #if 0
893  cpl_msg_debug(__func__, "limits at y=%f: %f < %f < %f", ypos, dleft, dmid,
894  dright);
895 #endif
896 
897  /* table to store line fits for this one arc line */
898  cpl_table *fittable = muse_cpltable_new(muse_wavelines_def,
899  (int)kMuseSliceHiLikelyWidth + 5);
900 
901  /* From the center of the slice move outwards and fit the line *
902  * until we have arrived at the edge of the slice */
903  int i;
904  for (i = dmid; i >= ileft; i--) {
905  cpl_error_code rc = muse_wave_line_fit_single(aImage, i, ypos, halfwidth,
906  sigma, fittable, ++n);
907  if (rc != CPL_ERROR_NONE) { /* do not count this line */
908  --n;
909  }
910  }
911 #if 0
912  printf("arc line j=%d, columns i=%d...", j + 1, i);
913 #endif
914  for (i = dmid + 1; i <= iright; i++) {
915  cpl_error_code rc = muse_wave_line_fit_single(aImage, i, ypos, halfwidth,
916  sigma, fittable, ++n);
917  if (rc != CPL_ERROR_NONE) { /* do not count this line */
918  --n;
919  }
920  }
921  /* now remove rows with invalid entries, i.e. those that were not *
922  * filled with the properties of the fit -- cpl_table_erase_invalid() *
923  * does not work, it deletes all columns */
924  cpl_table_select_all(fittable);
925  cpl_table_and_selected_invalid(fittable, "center");
926  cpl_table_erase_selected(fittable);
927 #if 0
928  printf("line %d, %d line fits\n", j + 1, i);
929  cpl_table_dump(fittable, 0, n, stdout);
930  fflush(stdout);
931 #endif
932  cpl_errorstate state = cpl_errorstate_get();
933  muse_wave_line_fit_iterate(fittable, -1, aParams);
934  int npos = cpl_table_get_nrow(fittable);
935  if (npos <= aParams->xorder) {
936  cpl_msg_debug(__func__, "Polynomial fit failed in slice %d of IFU %hhu"
937  " for line at %.3fA (y-position near %.2f pix): %s",
938  (int)islice + 1, ifu, lambda, ypos, cpl_error_get_message());
939  cpl_errorstate_set(state);
940  }
941 #if 0
942  else {
943  printf("%s: line %2d, %d line fits, %d with low residuals:\n", __func__,
944  j + 1, i, npos);
945  cpl_table_dump(fittable, 0, npos, stdout);
946  fflush(stdout);
947  }
948 #endif
949 
950  /* resize matrix/vector to be able to fit all new entries */
951  cpl_matrix_resize(xypos, 0, 0,
952  0, ientry + npos - cpl_matrix_get_ncol(xypos));
953  cpl_vector_set_size(lambdas, ientry + npos);
954  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
955  cpl_vector_set_size(dlambdas, ientry + npos);
956  }
957 
958  /* now add the final fit positions */
959  int ipos;
960  for (ipos = 0; ipos < npos; ipos++) {
961  /* set x-position (CCD column) in the first matrix row */
962  cpl_matrix_set(xypos, 0, ientry, cpl_table_get(fittable, "x", ipos, NULL));
963  /* y-position on CCD (center of Gauss fit) into second matrix row */
964  cpl_matrix_set(xypos, 1, ientry, cpl_table_get(fittable, "center", ipos, NULL));
965  /* the vector has to contain as many lambda entries */
966  cpl_vector_set(lambdas, ientry, lambda);
967  /* pretend that errors in Gaussian fit are errors in wavelength, *
968  * scale according to the nominal Angstrom/pix sampling of MUSE, *
969  * everything else would be too complicated for little gain */
970  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
971  cpl_vector_set(dlambdas, ientry, cpl_table_get(fittable, "cerr", ipos, NULL)
972  * kMuseSpectralSamplingA);
973  }
974 
975  ientry++; /* next position in this matrix/vector combo */
976  if (fp_debug) {
977  fprintf(fp_debug, " %02d %04d %9.4f %10.4f %e\n", (int)islice + 1,
978  (int)cpl_table_get(fittable, "x", ipos, NULL),
979  cpl_table_get(fittable, "center", ipos, NULL), lambda,
980  cpl_table_get(fittable, "cerr", ipos, NULL) * kMuseSpectralSamplingA);
981  }
982  }
983  cpl_table_delete(fittable);
984  } /* for j (all identified arc lines) */
985  cpl_table_delete(detlines);
986 
987  /* Compute two-dimensional wavelength solution for each slice. */
988  cpl_polynomial *poly = NULL; /* polynomial for wavelength solution */
989  double mse = 0; /* mean squared error */
990  cpl_error_code rc = muse_wave_poly_fit(xypos, lambdas, dlambdas,
991  &poly, &mse, aParams, islice + 1);
992  cpl_matrix_delete(xypos);
993  int nfinal = muse_cplvector_count_unique(lambdas);
994  cpl_vector_delete(lambdas);
995  cpl_vector_delete(dlambdas);
996  /* above we wrote detected and identified lines, here *
997  * save the ones that actually survived the fit */
998  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_FIT_NLINES, (unsigned short)(islice + 1));
999  cpl_propertylist_append_int(aImage->header, keyword, nfinal);
1000  cpl_free(keyword);
1001  /* collect QC on wavelengths */
1002  muse_wave_calib_qc_lambda(aImage, islice + 1, ptrace, poly);
1003  /* trace polynomials are not needed further */
1004  muse_trace_polys_delete(ptrace);
1005 
1006  if (rc != CPL_ERROR_NONE) { /* failure */
1007  cpl_msg_warning(__func__, "Something went wrong while fitting in slice "
1008  "%d of IFU %hhu: %s", (int)islice + 1, ifu,
1009  cpl_error_get_message_default(rc));
1010  cpl_polynomial_delete(poly);
1011  continue; /* try next slice immediately */
1012  }
1013 
1014  if (!wavecaltable) {
1015  /* create output table with one row for each slice; no need to *
1016  * check return code, it can only fail for negative lengths */
1017  wavecaltable = muse_wave_table_create(kMuseSlicesPerCCD,
1018  aParams->xorder, aParams->yorder);
1019  }
1020  rc = muse_wave_table_add_poly(wavecaltable, poly, mse,
1021  aParams->xorder, aParams->yorder, islice);
1022 
1023  /* use the number of wavelengths as indicator of the lines used for the *
1024  * fit and record it as QC parameter */
1025  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_FIT_RMS, (unsigned short)(islice + 1));
1026  cpl_propertylist_append_float(aImage->header, keyword, sqrt(mse));
1027  cpl_free(keyword);
1028 
1029  if (rc != CPL_ERROR_NONE) {
1030  cpl_msg_warning(__func__, "Could not write polynomial to wavecal table "
1031  "for slice %d of IFU %hhu: %s", (int)islice + 1, ifu,
1032  cpl_error_get_message_default(rc));
1033  }
1034  /* clean up, ignore any failure with this */
1035  cpl_polynomial_delete(poly);
1036  } /* for islice (all kMuseSlicesPerCCD) */
1037  cpl_vector_delete(vreflam);
1038  if (fp_debug) {
1039  cpl_msg_info(__func__, "Done writing line fits to \"%s\"", fn_debug);
1040  fclose(fp_debug);
1041  }
1042 
1043  muse_wave_calib_output_summary(aImage->header, wavecaltable);
1044 #if 0
1045  if (cpl_msg_get_level() == CPL_MSG_DEBUG) {
1046  cpl_table_dump(wavecaltable, 0, 3, stdout); fflush(stdout);
1047  }
1048 #endif
1049 
1050  return wavecaltable;
1051 } /* muse_wave_calib() */
1052 
1053 /*----------------------------------------------------------------------------*/
1117 /*----------------------------------------------------------------------------*/
1118 cpl_table *
1119 muse_wave_calib_lampwise(muse_imagelist *aImages, cpl_table *aTrace,
1120  cpl_table *aLinelist, muse_wave_params *aParams)
1121 {
1122  if (!aImages || muse_imagelist_get_size(aImages) < 1) {
1123  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "arc imagelist is "
1124  "missing or empty!");
1125  return NULL;
1126  }
1127  unsigned char ifu = muse_utils_get_ifu(muse_imagelist_get(aImages, 0)->header);
1128  if (!aTrace) {
1129  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "trace table missing "
1130  "for IFU %hhu, cannot create wavelength calibration!",
1131  ifu);
1132  return NULL;
1133  } else if (cpl_table_get_nrow(aTrace) != kMuseSlicesPerCCD) {
1134  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "trace table "
1135  "not valid for this dataset of IFU %hhu!", ifu);
1136  return NULL;
1137  }
1138  if (!aLinelist) {
1139  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "no arc line list "
1140  "supplied for IFU %hhu!", ifu);
1141  return NULL;
1142  }
1143  if (!aParams) {
1144  cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT, "wavelength "
1145  "calibration parameters missing for IFU %hhu!", ifu);
1146  return NULL;
1147  }
1149  cpl_error_set_message(__func__, CPL_ERROR_UNSUPPORTED_MODE,
1150  "unknown weighting scheme for IFU %hhu", ifu);
1151  return NULL;
1152  }
1153 
1154  muse_image *firstimage = muse_imagelist_get(aImages, 0);
1155  int nx = cpl_image_get_size_x(firstimage->data),
1156  ny = cpl_image_get_size_y(firstimage->data);
1157  /* do not support binned exposures */
1158  if (ny < 4000) {
1159  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_INPUT, "this dataset of "
1160  "IFU %hhu is too small (%dx%d pix) to be supported",
1161  ifu, nx, ny);
1162  return NULL;
1163  }
1164 
1165  cpl_msg_info(__func__, "Using polynomial orders %hu (x) and %hu (y), %s "
1166  "weighting, assuming initial sampling of %.3f +/- %.3f Angstrom"
1167  "/pix, in IFU %hhu", aParams->xorder, aParams->yorder,
1168  muse_wave_weighting_string[aParams->fitweighting],
1169  kMuseSpectralSamplingA, aParams->ddisp, ifu);
1170 
1171  cpl_propertylist_erase_regexp(firstimage->header, "^ESO QC", 0);
1172  char *lamp = muse_utils_header_get_lamp_names(firstimage->header, ',');
1173  cpl_msg_debug(__func__, "Image 1 was taken with lamp %s", lamp);
1174  cpl_free(lamp);
1175  unsigned int k;
1176  for (k = 1; k < muse_imagelist_get_size(aImages); k++) {
1177  int nxk = cpl_image_get_size_x(muse_imagelist_get(aImages, k)->data),
1178  nyk = cpl_image_get_size_y(muse_imagelist_get(aImages, k)->data);
1179  if (nxk != nx || nyk != ny) {
1180  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "arc "
1181  "imagelist is not uniform (image %u is %dx%d, "
1182  "first image is %dx%d)", k + 1, nxk, nyk, nx, ny);
1183  return NULL;
1184  }
1185  cpl_propertylist_erase_regexp(muse_imagelist_get(aImages, k)->header,
1186  "^ESO QC", 0);
1187  lamp = muse_utils_header_get_lamp_names(muse_imagelist_get(aImages, k)->header, ',');
1188  cpl_msg_debug(__func__, "Image %d was taken with lamp %s", k + 1, lamp);
1189  cpl_free(lamp);
1190  } /* for k (all images in list except first) */
1191 
1192  cpl_vector *vreflam = muse_wave_lines_get(aLinelist, 5, 0.);
1193  if (!vreflam) {
1194  cpl_error_set_message(__func__, CPL_ERROR_BAD_FILE_FORMAT, "could not "
1195  "create list of FWHM reference arc wavelengths for "
1196  "IFU %hhu", ifu);
1197  return NULL;
1198  }
1199  cpl_table *wavecal = NULL;
1200  int debug = getenv("MUSE_DEBUG_WAVECAL")
1201  ? atoi(getenv("MUSE_DEBUG_WAVECAL")) : 0;
1202  char fn_debug[100];
1203  FILE *fp_debug = NULL;
1204  if (debug >= 3) {
1205  snprintf(fn_debug, 99, "MUSE_DEBUG_WAVE_LINES-%02hhu.ascii", ifu);
1206  cpl_msg_info(__func__, "Will write all single line fits to \"%s\"", fn_debug);
1207  fp_debug = fopen(fn_debug, "w");
1208  if (fp_debug) {
1209  fprintf(fp_debug, "#slice x y lambda lambdaerr\n");
1210  }
1211  }
1212 
1213  /* loop over all slices */
1214  unsigned short islice;
1215  for (islice = 0; islice < kMuseSlicesPerCCD; islice++) {
1216  if (debug > 0) {
1217  printf("\n\nSlice %d of IFU %hhu\n", (int)islice + 1, ifu);
1218  fflush(stdout);
1219  }
1220  /* get the tracing polynomials for this slice */
1221  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
1222  islice + 1);
1223  if (!ptrace) {
1224  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: tracing polynomials "
1225  "missing!", (int)islice + 1, ifu);
1226  continue;
1227  }
1228  /* detect all lines in the center of the slice */
1229  const int imid = lround(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER],
1230  ny/2., NULL)),
1231  iWidth = 3,
1232  icol1 = imid - iWidth/2,
1233  icol2 = icol1 + iWidth;
1234  if (imid < 1 || imid > nx) {
1235  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: faulty trace polynomial"
1236  "detected", (int)islice + 1, ifu);
1237  muse_trace_polys_delete(ptrace);
1238  continue; /* next slice */
1239  }
1240  cpl_table *detlines = NULL;
1241  int ndet = 0;
1242  for (k = 0; k < muse_imagelist_get_size(aImages); k++) {
1243  muse_image *arc = muse_imagelist_get(aImages, k);
1244  /* create muse_image from the spectrum in which we want to search lines */
1245  muse_imagelist *collist = muse_imagelist_new();
1246  int i;
1247  for (i = icol1; i <= icol2; i++) {
1248  muse_image *column = muse_image_new();
1249  column->data = cpl_image_extract(arc->data, i, 1, i, ny);
1250  column->dq = cpl_image_extract(arc->dq, i, 1, i, ny);
1251  column->stat = cpl_image_extract(arc->stat, i, 1, i, ny);
1252  muse_imagelist_set(collist, column, i - icol1);
1253  } /* for i (neighboring image columns) */
1254  muse_image *column = muse_combine_median_create(collist);
1255  cpl_propertylist_append_string(column->header, "BUNIT",
1257  muse_imagelist_delete(collist);
1258 
1259  /* now search, measure, and store the lines */
1260  cpl_table *detections = muse_wave_lines_search(column, aParams->detsigma,
1261  islice + 1, ifu);
1262  muse_image_delete(column);
1263  int nlampdet = cpl_table_get_nrow(detections); /* det. with this lamp */
1264  ndet += nlampdet; /* detections of all lamps */
1265  char *lampname = muse_utils_header_get_lamp_names(arc->header, ',');
1266  cpl_table_fill_column_window_string(detections, "lampname", 0, nlampdet,
1267  lampname);
1268  cpl_array *lampnumbers = muse_utils_header_get_lamp_numbers(arc->header);
1269  cpl_table_fill_column_window_int(detections, "lampno", 0, nlampdet,
1270  cpl_array_get_int(lampnumbers, 0, NULL));
1271  cpl_array_delete(lampnumbers);
1272  /* get arc lines for this lamp from original list */
1273  double minflux = cpl_table_get_column_min(detections, "flux") * 1.15;
1274  cpl_vector *ionlambdas = muse_wave_lines_get_for_lamp(aLinelist, lampname,
1275  1, minflux);
1276  cpl_free(lampname);
1277  muse_wave_lines_identify(detections, ionlambdas, aParams);
1278  cpl_vector_delete(ionlambdas);
1279 
1280  if (!detlines) {
1281  detlines = detections;
1282  } else {
1283  cpl_table_insert(detlines, detections, cpl_table_get_nrow(detlines));
1284  cpl_table_delete(detections);
1285  }
1286  } /* for k (all images in list) */
1287  cpl_propertylist *order = cpl_propertylist_new();
1288  cpl_propertylist_append_bool(order, "y", CPL_FALSE);
1289  cpl_table_sort(detlines, order);
1290  cpl_propertylist_delete(order);
1291  int nlines = cpl_table_get_nrow(detlines);
1292  if (debug >= 2) {
1293  printf("Detected and identified %d arc lines in slice %d of IFU %hhu:\n",
1294  nlines, (int)islice + 1, ifu);
1295  cpl_table_dump(detlines, 0, cpl_table_get_nrow(detlines), stdout);
1296  fflush(stdout);
1297  }
1298  /* the first QC parameter does even make sense without lines */
1299  char *keyword = cpl_sprintf(QC_WAVECAL_SLICEj_LINES_NDET, (unsigned short)(islice + 1));
1300  cpl_propertylist_append_int(firstimage->header, keyword, ndet);
1301  cpl_free(keyword);
1302 
1303  if (nlines < aParams->yorder + 1) {
1304  /* could apparently not identify enough lines, or an error occured */
1305  cpl_error_set_message(__func__, CPL_ERROR_DATA_NOT_FOUND, "could not "
1306  "detect and/or identify enough arc lines in slice "
1307  "%d of IFU %hhu (%d of %d, required %d)",
1308  (int)islice + 1, ifu, nlines, ndet, (int)aParams->yorder + 1);
1309  muse_trace_polys_delete(ptrace);
1310  cpl_table_delete(detlines);
1311  continue; /* work on next slice immediately */
1312  }
1313  cpl_msg_info(__func__, "Identified %d of %d detected arc lines in slice %d"
1314  " of IFU %hhu (column %d, %d..%d)", nlines, ndet, (int)islice + 1,
1315  ifu, imid, icol1, icol2);
1316 
1317  /* the next set of QC parameters */
1318  muse_wave_calib_qc_peaks(detlines, firstimage->header, islice + 1);
1319  /* above we wrote the number of detected lines, now is the number of *
1320  * arc lines really used in the fit, i.e. the identified ones */
1321  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_LINES_NID, (unsigned short)(islice + 1));
1322  cpl_propertylist_append_int(firstimage->header, keyword, nlines);
1323  cpl_free(keyword);
1324 
1325  /* convert y and lambda into matrix and vector for an initial, *
1326  * vertical, inverse 1D fit */
1327  cpl_vector *cen = cpl_vector_new(nlines);
1328  cpl_matrix *lbda = cpl_matrix_new(1, nlines);
1329  int idx;
1330  for (idx = 0; idx < nlines; idx++) {
1331  cpl_vector_set(cen, idx, cpl_table_get(detlines, "center", idx, NULL));
1332  cpl_matrix_set(lbda, 0, idx, cpl_table_get(detlines, "lambda", idx, NULL));
1333  }
1334 #if 0
1335  char *fn = cpl_sprintf("slice%02d_lbda1.dat", (int)islice + 1);
1336  FILE *file = fopen(fn, "w");
1337  cpl_matrix_dump(lbda, file);
1338  fclose(file);
1339  cpl_free(fn);
1340 #endif
1341  double mse1d, chisq1d;
1342  cpl_polynomial *fit
1343  = muse_utils_iterate_fit_polynomial(lbda, cen, NULL, detlines,
1344  aParams->yorder, 3.,
1345  &mse1d, &chisq1d);
1346 #if 0
1347  cpl_vector *res = cpl_vector_new(cpl_vector_get_size(cen));
1348  cpl_vector_fill_polynomial_fit_residual(res, cen, NULL, fit, lbda, NULL);
1349  cpl_bivector *biv = cpl_bivector_wrap_vectors(cen, res);
1350  cpl_plot_bivector(NULL, NULL, NULL, biv);
1351  cpl_bivector_unwrap_vectors(biv);
1352  cpl_vector_delete(res);
1353 #endif
1354  if (debug >= 1) {
1355  printf("Initial (inverse) polynomial fit in slice %2d of IFU %hhu "
1356  "(RMS = %f, chisq = %e, %d of %d input points left)\n", (int)islice + 1,
1357  ifu, sqrt(mse1d), chisq1d, (int)cpl_vector_get_size(cen), nlines);
1358  cpl_polynomial_dump(fit, stdout);
1359  fflush(stdout);
1360  }
1361  nlines = cpl_vector_get_size(cen);
1362 #if 0
1363  fn = cpl_sprintf("slice%02hu_lbda2.dat", islice + 1);
1364  file = fopen(fn, "w");
1365  cpl_matrix_dump(lbda, file);
1366  fclose(file);
1367  cpl_free(fn);
1368 #endif
1369  cpl_vector_delete(cen);
1370  cpl_matrix_delete(lbda);
1371 
1372  /* positions matrix and wavelengths vector to be used for the 2D polynomial *
1373  * fit; just set and initial size of 1, it has to be resized with every line */
1374  cpl_matrix *xypos = cpl_matrix_new(2, 1);
1375  cpl_vector *lambdas = cpl_vector_new(1),
1376  *dlambdas = NULL;
1377  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
1378  dlambdas = cpl_vector_new(1);
1379  }
1380  int ientry = 0; /* index for these two structures */
1381 
1382  /* now measure all bright, single lines taken from the input linelist */
1383  cpl_table *fwhmtable = muse_cpltable_new(muse_wavelines_def, 0);
1384  cpl_table_set_column_unit(fwhmtable, "fwhm", "Angstrom");
1385  int j, narclines = cpl_table_get_nrow(aLinelist);
1386  for (j = 0; j < narclines; j++) {
1387  int quality = cpl_table_get_int(aLinelist, MUSE_LINE_CATALOG_QUALITY, j, NULL);
1388  if (quality <= 1) {
1389  continue; /* skip unwanted line */
1390  }
1391 
1392  /* use the lamp name to find the corresponding exposure for arc line */
1393  const char *lampname = muse_wave_lines_get_lampname(aLinelist, j);
1394  muse_image *arc = NULL;
1395  for (k = 0; k < muse_imagelist_get_size(aImages); k++) {
1396  arc = muse_imagelist_get(aImages, k);
1397  char *thislamp = muse_utils_header_get_lamp_names(arc->header, ',');
1398  cpl_boolean match = CPL_FALSE;
1399  if (lampname && thislamp) {
1400  match = strncmp(lampname, thislamp, strlen(lampname)) == 0;
1401  }
1402  cpl_free(thislamp);
1403  if (match) {
1404  break;
1405  }
1406  } /* for k (all images in list) */
1407 
1408  cpl_table *fittable = NULL;
1409  if (quality == 2) { /* multiplet */
1410  fittable = muse_wave_line_handle_multiplet(arc, aLinelist, j, fit, ptrace,
1411  aParams, islice + 1, debug);
1412  } else {
1413  fittable = muse_wave_line_handle_singlet(arc, aLinelist, j, fit, ptrace,
1414  aParams, islice + 1, debug);
1415  }
1416  if (!fittable) {
1417  continue;
1418  }
1419  int nnew = cpl_table_get_nrow(fittable);
1420 
1421  /* resize matrix/vector to be able to fit all new entries */
1422  cpl_matrix_resize(xypos, 0, 0,
1423  0, ientry + nnew - cpl_matrix_get_ncol(xypos));
1424  cpl_vector_set_size(lambdas, ientry + nnew);
1425  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
1426  cpl_vector_set_size(dlambdas, ientry + nnew);
1427  }
1428 
1429  /* now add the final fit positions */
1430  cpl_table_set_column_unit(fittable, "fwhm", ""); /* ongoing unit conversion */
1431  int ipos;
1432  for (ipos = 0; ipos < nnew; ipos++) {
1433  /* set x-position (CCD column) in the first matrix row */
1434  cpl_matrix_set(xypos, 0, ientry, cpl_table_get(fittable, "x", ipos, NULL));
1435  /* y-position on CCD (center of Gauss fit) into second matrix row */
1436  cpl_matrix_set(xypos, 1, ientry, cpl_table_get(fittable, "center", ipos, NULL));
1437  /* the vector has to contain as many lambda entries */
1438  double lambda = cpl_table_get(fittable, "lambda", ipos, NULL);
1439  cpl_vector_set(lambdas, ientry, lambda);
1440  /* Pretend that errors in Gaussian fit are errors in wavelength, *
1441  * use the first-guess 1D solution to convert from pix to Angstrom. */
1442  double dlbda;
1443  cpl_polynomial_eval_1d(fit, lambda, &dlbda); /* not interested in result */
1444  double cerr = 0.;
1445  if (aParams->fitweighting != MUSE_WAVE_WEIGHTING_UNIFORM) {
1446  cerr = cpl_table_get(fittable, "cerr", ipos, NULL) / dlbda;
1447  cpl_vector_set(dlambdas, ientry, cerr);
1448  }
1449  /* now that we have the actual sampling, also convert FWHM to Angstrom */
1450  int err;
1451  double fwhm = cpl_table_get(fittable, "fwhm", ipos, &err) / dlbda;
1452  if (!err) {
1453  cpl_table_set(fittable, "fwhm", ipos, fwhm);
1454  }
1455  ientry++; /* next position in this matrix/vector combo */
1456  if (fp_debug) {
1457  fprintf(fp_debug, " %02d %04d %9.4f %10.4f %e\n", (int)islice + 1,
1458  (int)cpl_table_get(fittable, "x", ipos, NULL),
1459  cpl_table_get(fittable, "center", ipos, NULL), lambda, cerr);
1460  }
1461  } /* for ipos (all fittable rows) */
1462  cpl_table_set_column_unit(fittable, "fwhm", "Angstrom");
1463  /* append relevant lines to extra table to compute FWHM and R later */
1464  cpl_table_select_all(fittable);
1465  cpl_table_and_selected_invalid(fittable, "fwhm");
1466  cpl_table_erase_selected(fittable);
1467  cpl_table_insert(fwhmtable, fittable, cpl_table_get_nrow(fwhmtable));
1468  cpl_table_delete(fittable);
1469  } /* for j (all arc lines) */
1470  /* reset original line quality info */
1471  cpl_table_abs_column(aLinelist, MUSE_LINE_CATALOG_QUALITY);
1472  cpl_table_delete(detlines);
1473  cpl_polynomial_delete(fit);
1474  muse_wave_calib_qc_fwhm(fwhmtable, firstimage->header, islice + 1);
1475  cpl_table_delete(fwhmtable);
1476 
1477  /* Compute two-dimensional wavelength solution for each slice. */
1478  cpl_polynomial *poly = NULL; /* polynomial for wavelength solution */
1479  double mse2d;
1480  cpl_error_code rc = muse_wave_poly_fit(xypos, lambdas, dlambdas,
1481  &poly, &mse2d, aParams, islice + 1);
1482  cpl_msg_info(__func__, "Polynomial fit of %"CPL_SIZE_FORMAT" of %d positions"
1483  " in slice %d of IFU %hhu gave an RMS of %f Angstrom",
1484  cpl_vector_get_size(lambdas), ientry, (int)islice + 1, ifu,
1485  sqrt(mse2d));
1486  cpl_matrix_delete(xypos);
1487  int nfinal = muse_cplvector_count_unique(lambdas);
1488  cpl_vector_delete(lambdas);
1489  cpl_vector_delete(dlambdas);
1490  /* above we wrote detected and identified lines, here *
1491  * save the ones that actually survived the fit */
1492  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_FIT_NLINES, (unsigned short)(islice + 1));
1493  cpl_propertylist_append_int(firstimage->header, keyword, nfinal);
1494  cpl_free(keyword);
1495  /* collect QC on wavelengths */
1496  muse_wave_calib_qc_lambda(firstimage, islice + 1, ptrace, poly);
1497  /* trace polynomials are not needed further */
1498  muse_trace_polys_delete(ptrace);
1499 
1500  if (rc != CPL_ERROR_NONE) { /* failure of the polynomial fit */
1501  cpl_error_set_message(__func__, rc, "a failure while computing the two-di"
1502  "mensional polynomial fit in slice %d of IFU %hhu",
1503  (int)islice + 1, ifu);
1504  cpl_polynomial_delete(poly);
1505  continue; /* try next slice immediately */
1506  }
1507 
1508  if (!wavecal) {
1509  /* create output table with one row for each slice; no need to *
1510  * check return code, it can only fail for negative lengths */
1511  wavecal = muse_wave_table_create(kMuseSlicesPerCCD,
1512  aParams->xorder, aParams->yorder);
1513  }
1514  rc = muse_wave_table_add_poly(wavecal, poly, mse2d,
1515  aParams->xorder, aParams->yorder, islice);
1516 
1517  /* use the number of wavelengths as indicator of the lines used for the *
1518  * fit and record it as QC parameter */
1519  keyword = cpl_sprintf(QC_WAVECAL_SLICEj_FIT_RMS, (unsigned short)(islice + 1));
1520  cpl_propertylist_append_float(firstimage->header, keyword, sqrt(mse2d));
1521  cpl_free(keyword);
1522 
1523  if (rc != CPL_ERROR_NONE) {
1524  cpl_msg_warning(__func__, "Could not write polynomial to wavecal table "
1525  "for slice %d of IFU %hhu: %s", (int)islice + 1, ifu,
1526  cpl_error_get_message_default(rc));
1527  }
1528  /* clean up, ignore any failure with this */
1529  cpl_polynomial_delete(poly);
1530  } /* for islice (all kMuseSlicesPerCCD) */
1531  cpl_vector_delete(vreflam);
1532  if (fp_debug) {
1533  cpl_msg_info(__func__, "Done writing line fits to \"%s\"", fn_debug);
1534  fclose(fp_debug);
1535  }
1536 
1537  muse_wave_calib_output_summary(firstimage->header, wavecal);
1538  /* copy the QC parameters to the other image headers */
1539  for (k = 1; k < muse_imagelist_get_size(aImages); k++) {
1540  cpl_propertylist *header = muse_imagelist_get(aImages, k)->header;
1541  cpl_propertylist_erase_regexp(header, "^ESO QC", 0);
1542  cpl_propertylist_copy_property_regexp(header, firstimage->header, "^ESO QC", 0);
1543  } /* for k (all images in list except first) */
1544 
1545  return wavecal;
1546 } /* muse_wave_calib_lampwise() */
1547 
1548 /*----------------------------------------------------------------------------*/
1565 /*----------------------------------------------------------------------------*/
1566 cpl_boolean
1568 {
1569  cpl_ensure(aTable && aTable->table && aTable->header, CPL_ERROR_NULL_INPUT,
1570  CPL_FALSE);
1571  /* check the table */
1572  int nrow = cpl_table_get_nrow(aTable->table);
1573  cpl_ensure(nrow > 0, CPL_ERROR_NULL_INPUT, CPL_FALSE);
1574  cpl_ensure(muse_cpltable_check(aTable->table, muse_line_catalog_def)
1575  == CPL_ERROR_NONE, CPL_ERROR_DATA_NOT_FOUND, CPL_FALSE);
1576 
1577  /* check the version */
1578  if (!cpl_propertylist_has(aTable->header, MUSE_HDR_LINE_CATALOG_VERSION)) {
1579  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "%s does not "
1580  "contain a VERSION header entry!",
1581  MUSE_TAG_LINE_CATALOG);
1582  return CPL_FALSE;
1583  }
1584  int version = cpl_propertylist_get_int(aTable->header,
1585  MUSE_HDR_LINE_CATALOG_VERSION);
1586 #define LINE_CATALOG_EXPECTED_VERSION 3
1587  if (version != LINE_CATALOG_EXPECTED_VERSION) {
1588  cpl_error_set_message(__func__, CPL_ERROR_BAD_FILE_FORMAT, "VERSION = %d "
1589  "is wrong, we need a %s with VERSION = %d", version,
1590  MUSE_TAG_LINE_CATALOG, LINE_CATALOG_EXPECTED_VERSION);
1591  return CPL_FALSE;
1592  }
1593  return CPL_TRUE;
1594 } /* muse_wave_lines_check() */
1595 
1596 /*----------------------------------------------------------------------------*/
1618 /*----------------------------------------------------------------------------*/
1619 cpl_vector *
1620 muse_wave_lines_get(cpl_table *aTable, int aGoodnessLimit, double aFluxLimit)
1621 {
1622  cpl_ensure(aTable, CPL_ERROR_NULL_INPUT, NULL);
1623  int nrow = cpl_table_get_nrow(aTable);
1624  cpl_ensure(nrow > 0, CPL_ERROR_NULL_INPUT, NULL);
1625 
1626  cpl_ensure(cpl_table_has_column(aTable, MUSE_LINE_CATALOG_LAMBDA) == 1
1627  && cpl_table_has_column(aTable, MUSE_LINE_CATALOG_QUALITY) == 1,
1628  CPL_ERROR_DATA_NOT_FOUND, NULL);
1629 
1630  /* convert the usable wavelengths into a vector */
1631  cpl_vector *lambdas = cpl_vector_new(nrow);
1632  int i, pos = 0;
1633  for (i = 0; i < nrow; i++) {
1634  double lambda = cpl_table_get(aTable, MUSE_LINE_CATALOG_LAMBDA, i, NULL),
1635  flux = cpl_table_get(aTable, MUSE_LINE_CATALOG_FLUX, i, NULL);
1636 
1637  /* make sure that the arc line table is sorted, to simplify debugging */
1638  if (i > 0 && lambda < cpl_table_get(aTable, MUSE_LINE_CATALOG_LAMBDA, i-1,
1639  NULL)) {
1640  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT, "%s is not "
1641  "sorted by increasing lambda (at row %d)!",
1642  MUSE_TAG_LINE_CATALOG, i+1);
1643  cpl_vector_delete(lambdas);
1644  return NULL;
1645  }
1646 
1647  if (cpl_table_get(aTable, MUSE_LINE_CATALOG_QUALITY, i, NULL)
1648  < aGoodnessLimit || flux < aFluxLimit) {
1649  /* this is apparently a line that should not be used */
1650  continue;
1651  }
1652  cpl_vector_set(lambdas, pos, lambda);
1653  pos++;
1654  } /* for i (all table rows) */
1655  if (!pos) { /* setting the size to zero does not work! */
1656  cpl_vector_delete(lambdas);
1657  cpl_error_set_message(__func__, CPL_ERROR_DATA_NOT_FOUND, "No lines with "
1658  "%s >= %d found", MUSE_LINE_CATALOG_QUALITY,
1659  aGoodnessLimit);
1660  return NULL;
1661  }
1662  /* cut vector size to contain the usable wavelengths */
1663  cpl_vector_set_size(lambdas, pos);
1664 
1665  /* remove lines that are too close together, i.e. within 1.5 Angstrom */
1666  for (i = 0; i < cpl_vector_get_size(lambdas) - 1; i++) {
1667  double l1 = cpl_vector_get(lambdas, i),
1668  l2 = cpl_vector_get(lambdas, i + 1);
1669  if ((l2 - l1) < 1.5) {
1670  cpl_msg_debug(__func__, "Excluding lines at %.3f and %.3f (d = %.3f) "
1671  "Angstrom", l1, l2, l2 - l1);
1672  muse_cplvector_erase_element(lambdas, i + 1);
1673  muse_cplvector_erase_element(lambdas, i);
1674  i--; /* stay at this index to see what moved here */
1675  }
1676  } /* for i (all vector entries) */
1677 #if 0
1678  printf("%d arc lines in input list are usable:\n", pos);
1679  cpl_vector_dump(lambdas, stdout);
1680 #endif
1681  cpl_msg_debug(__func__, "Using a list of %d %s arc lines (from %6.1f to %6.1f "
1682  "Angstrom)", pos,
1683  aGoodnessLimit == 1 ? "good"
1684  : (aGoodnessLimit == 5 ? "FWHM reference"
1685  : "all"),
1686  cpl_vector_get(lambdas, 0),
1687  cpl_vector_get(lambdas, cpl_vector_get_size(lambdas) - 1));
1688 
1689  return lambdas;
1690 } /* muse_wave_lines_get() */
1691 
1692 /*----------------------------------------------------------------------------*/
1709 /*----------------------------------------------------------------------------*/
1710 cpl_vector *
1711 muse_wave_lines_get_for_lamp(cpl_table *aTable, const char *aLamp,
1712  int aGoodnessLimit, double aFluxLimit)
1713 {
1714  cpl_ensure(aTable && aLamp, CPL_ERROR_NULL_INPUT, NULL);
1715  int nrow = cpl_table_get_nrow(aTable);
1716  cpl_ensure(nrow > 0, CPL_ERROR_NULL_INPUT, NULL);
1717 
1718  cpl_table_unselect_all(aTable);
1719  int i;
1720  for (i = 0; i < nrow; i++) {
1721  const char *lampname = muse_wave_lines_get_lampname(aTable, i);
1722  if (!strcmp(lampname, aLamp)) {
1723  cpl_table_select_row(aTable, i);
1724  }
1725  } /* for i (all table rows) */
1726  cpl_table *lamplines = cpl_table_extract_selected(aTable);
1727 #if 0
1728  printf("lamplines for %s\n", aLamp);
1729  cpl_table_dump(lamplines, 0, 10000, stdout);
1730  fflush(stdout);
1731 #endif
1732  cpl_vector *lamplambdas = muse_wave_lines_get(lamplines, aGoodnessLimit,
1733  aFluxLimit);
1734  cpl_table_delete(lamplines);
1735  return lamplambdas;
1736 } /* muse_wave_lines_get_for_lamp() */
1737 
1738 /*----------------------------------------------------------------------------*/
1752 /*----------------------------------------------------------------------------*/
1753 const char *
1754 muse_wave_lines_get_lampname(cpl_table *aTable, const int aIdx)
1755 {
1756  cpl_ensure(aTable, CPL_ERROR_NULL_INPUT, "Unknown_Lamp");
1757  const char *ion = cpl_table_get_string(aTable, "ion", aIdx);
1758  cpl_ensure(ion, CPL_ERROR_ILLEGAL_INPUT, "Unknown_Lamp");
1759 
1760  /* for the HgCd lamp we have "HgI", "CdI", and "Hg_or_Cd" */
1761  if (!strncmp(ion, "Hg", 2) || !strncmp(ion, "Cd", 2)) {
1762  return "HgCd";
1763  }
1764  /* for the Ne lamp we have only "NeI" */
1765  if (!strncmp(ion, "Ne", 2)) {
1766  return "Ne";
1767  }
1768  /* for the Xe lamp we have "XeI" and "XeII" */
1769  if (!strncmp(ion, "Xe", 2)) {
1770  return "Xe";
1771  }
1772  return "Unknown_Lamp";
1773 } /* muse_wave_lines_get_lampname() */
1774 
1775 /*----------------------------------------------------------------------------*/
1782 /*----------------------------------------------------------------------------*/
1783 cpl_boolean
1784 muse_wave_lines_covered_by_data(double aLambda, muse_ins_mode aMode)
1785 {
1786  if ((aLambda > kMuseUsefulLambdaMax) || (aLambda < kMuseUsefulELambdaMin)) {
1787  return CPL_FALSE;
1788  }
1789  /* first check the modes without a gap */
1790  if ((aMode <= MUSE_MODE_WFM_NONAO_N) && (aLambda >= kMuseUsefulNLambdaMin)) {
1791  return CPL_TRUE;
1792  }
1793  if ((aMode == MUSE_MODE_WFM_NONAO_E) && (aLambda >= kMuseUsefulELambdaMin)) {
1794  return CPL_TRUE;
1795  }
1796  /* now the AO modes, with gap */
1797  if ((aMode == MUSE_MODE_WFM_AO_E) && (aLambda >= kMuseUsefulELambdaMin) &&
1798  ((aLambda <= kMuseNaLambdaMin) || (aLambda >= kMuseNaLambdaMax))) {
1799  return CPL_TRUE;
1800  }
1801  if ((aMode >= MUSE_MODE_WFM_AO_N) && (aLambda >= kMuseUsefulAOLambdaMin) &&
1802  ((aLambda <= kMuseNa2LambdaMin) || (aLambda >= kMuseNa2LambdaMax))) {
1803  return CPL_TRUE;
1804  }
1805  /* whatever is left now, has to be outside the common wavelength range */
1806  return CPL_FALSE;
1807 } /* muse_wave_lines_covered_by_data() */
1808 
1809 /*----------------------------------------------------------------------------*/
1836 /*----------------------------------------------------------------------------*/
1837 cpl_table *
1838 muse_wave_lines_search(muse_image *aColumnImage, double aSigma,
1839  const unsigned short aSlice, const unsigned char aIFU)
1840 {
1841 #if !SEARCH_DEBUG_FILES
1842  UNUSED_ARGUMENT(aSlice);
1843 #endif
1844  cpl_ensure(aColumnImage, CPL_ERROR_NULL_INPUT, NULL);
1845  cpl_ensure(cpl_image_get_min(aColumnImage->stat) > 0.,
1846  CPL_ERROR_DATA_NOT_FOUND, NULL);
1847  cpl_ensure(aSigma > 0., CPL_ERROR_ILLEGAL_INPUT, NULL);
1848 
1849 #define SEARCH_SUBTRACT_BG 1
1850 #if SEARCH_SUBTRACT_BG
1851  /* Subtract any large scale background using a median-filtered spectrum. *
1852  * In the longer run, maybe a polynomial fit to the median-smoothed data *
1853  * should be used instead, because it does not create extra noise. */
1854  cpl_image *bgmedian = cpl_image_duplicate(aColumnImage->data);
1855  cpl_image_fill_noise_uniform(bgmedian, -FLT_MIN, FLT_MIN);
1856 #define BG_KERNEL_WIDTH 51
1857  cpl_mask *filter = cpl_mask_new(1, BG_KERNEL_WIDTH);
1858  cpl_mask_not(filter);
1859  cpl_image_filter_mask(bgmedian, aColumnImage->data, filter, CPL_FILTER_MEDIAN,
1860  CPL_BORDER_FILTER);
1861  cpl_mask_delete(filter);
1862 #if SEARCH_DEBUG_FILES
1863  char fn[FILENAME_MAX];
1864  sprintf(fn, "column_slice%02hu.fits", aSlice);
1865  muse_image_save(aColumnImage, fn);
1866  sprintf(fn, "column_slice%02hu_bgmedian.fits", aSlice);
1867  cpl_image_save(bgmedian, fn, CPL_TYPE_FLOAT, NULL, CPL_IO_CREATE);
1868 #endif
1869  cpl_image *bgsub = cpl_image_subtract_create(aColumnImage->data, bgmedian);
1870  cpl_image_delete(bgmedian);
1871 #if SEARCH_DEBUG_FILES
1872  sprintf(fn, "column_slice%02hu_bgsub.fits", aSlice);
1873  cpl_image_save(bgsub, fn, CPL_TYPE_FLOAT, NULL, CPL_IO_CREATE);
1874 #endif
1875 #endif
1876 
1877  /* create the S/N image */
1878  cpl_image *N = cpl_image_power_create(aColumnImage->stat, 0.5), /* noise */
1879 #if SEARCH_SUBTRACT_BG
1880  *SN = cpl_image_divide_create(bgsub, N); /* S/N image */
1881  cpl_image_delete(bgsub);
1882 #else
1883  *SN = cpl_image_divide_create(aColumnImage->data, N); /* S/N image */
1884 #endif
1885  cpl_image_delete(N);
1886 #if SEARCH_DEBUG_FILES
1887  sprintf(fn, "column_slice%02hu_SN.fits", aSlice);
1888  cpl_image_save(SN, fn, CPL_TYPE_FLOAT, NULL, CPL_IO_CREATE);
1889 #endif
1890 
1891  /* use median statistics of the S/N image to determine the detection limit */
1892  double mdev, median = cpl_image_get_median_dev(SN, &mdev),
1893  limitSN = fmax(0.1, median + aSigma*mdev);
1894  cpl_mask *mask = cpl_mask_threshold_image_create(SN, limitSN, FLT_MAX);
1895  cpl_size nlines = 0; /* number of detected lines */
1896  cpl_image *label = cpl_image_labelise_mask_create(mask, &nlines);
1897  cpl_mask_delete(mask);
1898 #if SEARCH_DEBUG_FILES
1899  sprintf(fn, "column_slice%02hu_label.fits", aSlice);
1900  cpl_image_save(label, fn, CPL_TYPE_USHORT, NULL, CPL_IO_CREATE);
1901 #endif
1902 #if SEARCH_DEBUG || SEARCH_DEBUG_FILES
1903  double mean = cpl_image_get_mean(SN),
1904  stdev = cpl_image_get_stdev(SN),
1905  min = cpl_image_get_min(SN),
1906  max = cpl_image_get_max(SN);
1907  cpl_msg_debug(__func__, "%"CPL_SIZE_FORMAT" lines found; parameters: sigma=%f, "
1908  "med=%f+/-%f " "mean=%f+/-%f min/max=%f/%f limit(S/N)=%f", nlines,
1909  aSigma, median, mdev, mean, stdev, min, max, limitSN);
1910 #endif
1911  cpl_image_delete(SN);
1912 
1913  /* create table to store the fit results of all the lines */
1914  cpl_table *linesresult = muse_cpltable_new(muse_wavelines_def, nlines);
1915 
1916  /* now loop through all the detected lines and measure them */
1917  int i;
1918  for (i = 0; i < nlines; i++) {
1919  /* create mask for this label to isolate line (note that i starts at 0!) */
1920  cpl_mask *linemask = cpl_mask_threshold_image_create(label, i+0.5, i+1.5);
1921  int masksize = cpl_mask_get_size_y(linemask);
1922 
1923  /* determine the edges of the labeled region and the center of the line */
1924  int j = 1;
1925  while (j <= masksize && cpl_mask_get(linemask, 1, j) == CPL_BINARY_0) {
1926  j++;
1927  }
1928  int lopix = j;
1929  while (j <= masksize && cpl_mask_get(linemask, 1, j) == CPL_BINARY_1) {
1930  j++;
1931  }
1932  int hipix = j - 1;
1933  double cenpix = (hipix + lopix) / 2.;
1934 #if SEARCH_DEBUG
1935  int err;
1936  cpl_msg_debug(__func__, "line %d lo/hi/cen=%d,%d,%f: values=%f,%f", i + 1,
1937  lopix, hipix, cenpix,
1938  cpl_image_get(aColumnImage->data, 1, lopix, &err),
1939  cpl_image_get(aColumnImage->data, 1, hipix, &err));
1940 #endif
1941  cpl_mask_delete(linemask);
1942  if (lopix == hipix) {
1943  /* one pixel is not enough for a solid detection: *
1944  * set to some very low area / flux to be deleted below */
1945  cpl_table_set(linesresult, "flux", i, -1.);
1946  continue;
1947  }
1948 
1949  /* Enlarge region to be sure to measure the whole line (correct *
1950  * width!). As long as the pixel values are smaller than on the *
1951  * edges, and we don't go too far this should be OK. */
1952 #define MAXENLARGE 5
1953  /* reference pixel value (lower edge of mask) */
1954  int err0, err1 = 0, err2 = 0;
1955  double valref = cpl_image_get(aColumnImage->data, 1, lopix, &err0);
1956  cpl_errorstate prestate = cpl_errorstate_get();
1957  j = lopix;
1958  double value = -FLT_MAX;
1959  while (err1 == 0 && value < valref && (lopix - j) <= MAXENLARGE) {
1960  j--; /* enlarge downwards */
1961  value = cpl_image_get(aColumnImage->data, 1, j, &err1);
1962  }
1963  lopix = j + 1; /* set new lower edge */
1964 
1965  /* reference pixel value (lower edge of mask) */
1966  valref = cpl_image_get(aColumnImage->data, 1, hipix, &err2);
1967  j = hipix;
1968  value = -FLT_MAX;
1969  while (err2 == 0 && value < valref && (j - hipix) <= MAXENLARGE) {
1970  j++; /* enlarge upwards */
1971  value = cpl_image_get(aColumnImage->data, 1, j, &err2);
1972  }
1973  hipix = j - 1; /* set new upper edge */
1974  if (lopix > hipix) {
1975  /* guard against weird data, which would cause vector creation to fail */
1976  continue;
1977  }
1978 #if SEARCH_DEBUG
1979  cpl_msg_debug(__func__, "region=%d...%d, size=%d", lopix, hipix,
1980  hipix - lopix + 1);
1981 #endif
1982  if (err1 < 0 || err2 < 0) {
1983  cpl_errorstate_set(prestate); /* clean possible "Access beyond boundaries" */
1984  }
1985 
1986  /* fill vectors */
1987  cpl_vector *positions = cpl_vector_new(hipix - lopix + 1), /* for positions */
1988  *values = cpl_vector_new(hipix - lopix + 1), /* for pixel values */
1989  *valuessigma = cpl_vector_new(hipix - lopix + 1); /* sigma values */
1990  int k;
1991  for (j = lopix, k = 0; j <= hipix; j++, k++) {
1992  cpl_vector_set(positions, k, j);
1993  cpl_vector_set(values, k, cpl_image_get(aColumnImage->data, 1, j, &err0));
1994  /* when setting pixel value from stat extension, use *
1995  * square root, here we need the sigma not the variance: */
1996  cpl_vector_set(valuessigma, k, sqrt(cpl_image_get(aColumnImage->stat,
1997  1, j, &err0)));
1998  }
1999 
2000 #if SEARCH_DEBUG
2001  cpl_bivector *biv = cpl_bivector_wrap_vectors(positions, values);
2002  cpl_bivector_dump(biv, stdout);
2003  cpl_bivector_unwrap_vectors(biv);
2004 #endif
2005 
2006  /* fit Gaussian */
2007  /* compute position, sigma, area, background offset, and mean squared error */
2008  double center, cerr, sigma, area, bglevel, mse;
2009  cpl_matrix *covariance;
2010  prestate = cpl_errorstate_get();
2011  cpl_error_code rc = cpl_vector_fit_gaussian(positions, NULL, values,
2012  valuessigma, CPL_FIT_ALL,
2013  &center, &sigma, &area, &bglevel,
2014  &mse, NULL, &covariance);
2015 
2016  /* try to treat some possible problems */
2017  if (rc == CPL_ERROR_CONTINUE) { /* fit didn't converge */
2018  /* estimate position error as sigma^2/area as CPL docs suggest *
2019  * no warning necessary in this case, print a debug message */
2020  cerr = sqrt(sigma * sigma / area);
2021  cpl_msg_debug(__func__, "Gaussian fit in slice %hu of IFU %hhu around "
2022  "position %6.1f: %s", aSlice, aIFU, cenpix,
2023  cpl_error_get_message());
2024 #if DEBUG_GAUSSFIT
2025  if (cpl_msg_get_log_level() == CPL_MSG_DEBUG) {
2026  /* CPL_ERROR_CONTINUE often occurs if there is another peak on the edge *
2027  * of the input positions/data array, so print it out in debug mode */
2028  cpl_bivector *bv = cpl_bivector_wrap_vectors(positions, values);
2029  printf("Gaussian fit: %f+/-%f, %f, %f, %f %f\nand the input data:\n",
2030  center, cerr, bglevel, area, sigma, mse);
2031  cpl_bivector_dump(bv, stdout);
2032  cpl_bivector_unwrap_vectors(bv);
2033  fflush(stdout);
2034  }
2035 #endif
2036  } else if (rc == CPL_ERROR_SINGULAR_MATRIX || !covariance) {
2037  cerr = sqrt(sigma * sigma / area);
2038  cpl_msg_debug(__func__, "Gaussian fit in slice %hu of IFU %hhu around "
2039  "position %6.1f: %s", aSlice, aIFU, cenpix,
2040  cpl_error_get_message());
2041  } else if (rc != CPL_ERROR_NONE) {
2042  /* although this is unlikely to occur, better print some warning *
2043  * and do something to the data to give this fit less weight in *
2044  * the final computation of the solution */
2045  cpl_msg_debug(__func__, "Gaussian fit in slice %hu of IFU %hhu around "
2046  "position %6.1f: %s", aSlice, aIFU, cenpix,
2047  cpl_error_get_message());
2048  cerr = 100.; /* set a high centering error */
2049  } else {
2050  /* no error returned, everything should have worked nicely *
2051  * take the centering error from the covariance matrix */
2052  cerr = sqrt(cpl_matrix_get(covariance, 0, 0));
2053 #if SEARCH_DEBUG
2054  cpl_matrix_dump(covariance, stdout);
2055 #endif
2056  cpl_matrix_delete(covariance);
2057  }
2058  cpl_errorstate_set(prestate); /* now we can reset the status */
2059  if (fabs(center - cenpix) > MUSE_WAVE_LINES_SEARCH_SHIFT_WARN) {
2060  cpl_msg_debug(__func__, "Large shift in Gaussian centering in slice %hu "
2061  "of IFU %hhu: initial %7.2f, fit %7.2f", aSlice, aIFU,
2062  cenpix, center);
2063  }
2064 
2065  /* save all results of this fit into the output table *
2066  * (no need to fill the x-position into the "x" column) */
2067  cpl_table_set(linesresult, "y", i, cenpix);
2068  /* set actual peak value in the table using cpl_vector_get_max() is safe, *
2069  * because the vector is constructed to be small enough to not include *
2070  * neighboring lines and so the likelihood of including cosmic rays is low */
2071  cpl_table_set(linesresult, "peak", i, cpl_vector_get_max(values));
2072  cpl_table_set(linesresult, "center", i, center);
2073  cpl_table_set(linesresult, "cerr", i, cerr);
2074  cpl_table_set(linesresult, "fwhm", i, CPL_MATH_FWHM_SIG * sigma);
2075  cpl_table_set(linesresult, "sigma", i, sigma);
2076  cpl_table_set(linesresult, "flux", i, area);
2077  cpl_table_set(linesresult, "bg", i, bglevel);
2078  cpl_table_set(linesresult, "mse", i, mse);
2079 
2080  cpl_vector_delete(positions);
2081  cpl_vector_delete(values);
2082  cpl_vector_delete(valuessigma);
2083 #if SEARCH_DEBUG
2084  printf("%s: results of fit stored in table row %d:\n", __func__, i + 1);
2085  cpl_table_dump(linesresult, i, 1, stdout);
2086  fflush(stdout);
2087 #endif
2088  } /* for i (all detected lines) */
2089  cpl_image_delete(label);
2090 
2091  /* again loop through all the table lines, and remove unlikely detections */
2092  cpl_table_unselect_all(linesresult);
2093  for (i = 0; i < cpl_table_get_nrow(linesresult); i++) {
2094  if (cpl_table_get(linesresult, "cerr", i, NULL) > kMuseArcMaxCenteringError ||
2095  cpl_table_get(linesresult, "fwhm", i, NULL) < kMuseArcMinFWHM ||
2096  cpl_table_get(linesresult, "fwhm", i, NULL) > kMuseArcMaxFWHM ||
2097  cpl_table_get(linesresult, "flux", i, NULL) < kMuseArcMinFlux) {
2098  cpl_table_select_row(linesresult, i);
2099  }
2100  } /* for i (all detected lines) */
2101  cpl_table_erase_selected(linesresult);
2102 
2103  return linesresult;
2104 } /* muse_wave_lines_search() */
2105 
2106 /*----------------------------------------------------------------------------*/
2126 /*----------------------------------------------------------------------------*/
2127 cpl_error_code
2128 muse_wave_lines_identify(cpl_table *aLines, cpl_vector *aLambdas,
2129  const muse_wave_params *aParams)
2130 {
2131  cpl_ensure_code(aLines && aLambdas, CPL_ERROR_NULL_INPUT);
2132 
2133  /* convert detected lines from the table into a simple vector */
2134  int i, nlines = cpl_table_get_nrow(aLines);
2135  cpl_vector *vcenter = cpl_vector_new(nlines);
2136  for (i = 0; i < nlines; i++) {
2137  cpl_vector_set(vcenter, i, cpl_table_get(aLines, "center", i, NULL));
2138  } /* for i (all lines) */
2139 #if 0
2140  cpl_vector_dump(vcenter, stdout);
2141 #endif
2142  double dmin = kMuseSpectralSamplingA - kMuseSpectralSamplingA * aParams->ddisp,
2143  dmax = kMuseSpectralSamplingA + kMuseSpectralSamplingA * aParams->ddisp;
2144  cpl_bivector *id_lines = cpl_ppm_match_positions(vcenter, aLambdas, dmin, dmax,
2145  aParams->tolerance, NULL, NULL);
2146 #if 0
2147  cpl_msg_debug(__func__, "%"CPL_SIZE_FORMAT" identified lines (of %"
2148  CPL_SIZE_FORMAT" detected lines and %"CPL_SIZE_FORMAT
2149  " from linelist)", cpl_bivector_get_size(id_lines),
2150  cpl_vector_get_size(vcenter), cpl_vector_get_size(aLambdas));
2151  cpl_bivector_dump(id_lines, stdout);
2152  fflush(stdout);
2153 #endif
2154  cpl_vector_delete(vcenter);
2155 
2156 #if 0
2157  printf("detected lines before cleanup:\n");
2158  cpl_table_dump(aLines, 0, nlines, stdout);
2159  fflush(stdout);
2160 #endif
2161  /* now go through the detected lines and remove all unidentified */
2162  const double *id_center = cpl_bivector_get_x_data_const(id_lines),
2163  *id_lambda = cpl_bivector_get_y_data_const(id_lines);
2164  cpl_table_unselect_all(aLines);
2165  int j, nid = cpl_bivector_get_size(id_lines);
2166  for (i = 0, j = 0; i < cpl_table_get_nrow(aLines) && id_center && id_lambda; i++) {
2167 #if 0
2168  cpl_msg_debug(__func__, "c=%f, l=%f, c_det=%f", id_center[j], id_lambda[j],
2169  cpl_table_get(aLines, "center", i, NULL));
2170 #endif
2171  if (j < nid && fabs(id_center[j] - cpl_table_get(aLines, "center", i, NULL))
2172  < DBL_EPSILON) {
2173  /* found the same line, append the wavelength */
2174  cpl_table_set(aLines, "lambda", i, id_lambda[j]);
2175  j++;
2176  continue;
2177  }
2178  cpl_table_select_row(aLines, i); /* not matching, delete this line */
2179  } /* for i (detected lines) and j (identified lines) */
2180  cpl_table_erase_selected(aLines);
2181  cpl_bivector_delete(id_lines);
2182  int debug = getenv("MUSE_DEBUG_WAVECAL")
2183  ? atoi(getenv("MUSE_DEBUG_WAVECAL")) : 0;
2184  if (debug >= 2) {
2185  printf("identified %d lines, %"CPL_SIZE_FORMAT" after cleanup:\n", nid,
2186  cpl_table_get_nrow(aLines));
2187  cpl_table_dump(aLines, 0, nid, stdout);
2188  fflush(stdout);
2189  }
2190  nid = cpl_table_get_nrow(aLines);
2191  /* check that after identification enough lines are still there */
2192  if (nid <= 0) {
2193  return CPL_ERROR_DATA_NOT_FOUND;
2194  } else if (nid < aParams->yorder + 1) {
2195  return CPL_ERROR_ILLEGAL_OUTPUT;
2196  }
2197  return CPL_ERROR_NONE;
2198 } /* muse_wave_lines_identify() */
2199 
2200 /*----------------------------------------------------------------------------*/
2232 /*----------------------------------------------------------------------------*/
2233 cpl_table *
2234 muse_wave_line_handle_singlet(muse_image *aImage, cpl_table *aLinelist,
2235  unsigned int aIdx, cpl_polynomial *aPoly,
2236  cpl_polynomial **aTrace,
2237  const muse_wave_params *aParams,
2238  const unsigned short aSlice, int aDebug)
2239 {
2240  cpl_ensure(aImage && aLinelist && aPoly && aTrace, CPL_ERROR_NULL_INPUT, NULL);
2241 
2242  /* Fix the Gaussian sigmas for lines that are not FWHM reference lines, *
2243  * otherwise the fit might run astray in low S/N cases. A negative sigma *
2244  * signifies constant value to fitting routine. */
2245  double sigma = kMuseSliceSlitWidthA / kMuseSpectralSamplingA / CPL_MATH_FWHM_SIG;
2246  /* if not reference line, set it negative, so that it is fixed in the fit */
2247  if (cpl_table_get(aLinelist, MUSE_LINE_CATALOG_QUALITY, aIdx, NULL) != 5) {
2248  sigma *= -1;
2249  }
2250  int halfwidth = 3.*kMuseSliceSlitWidthA / kMuseSpectralSamplingA; /* 3*FWHM */
2251 
2252  /* convenient access to some properties of the current line */
2253  double lambda = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_LAMBDA, aIdx, NULL),
2254  ypos = cpl_polynomial_eval_1d(aPoly, lambda, NULL);
2255  if ((ypos - halfwidth) < 1 ||
2256  (ypos + halfwidth) > cpl_image_get_size_y(aImage->data)) {
2257  if (aDebug >= 2) {
2258  cpl_msg_debug(__func__, "%f is supposed to lie near %.3f in slice %2hu of"
2259  " IFU %hhu, i.e. outside!", lambda, ypos, aSlice,
2260  muse_utils_get_ifu(aImage->header));
2261  }
2262  return NULL;
2263  }
2264  if (aDebug >= 2) {
2265  cpl_msg_debug(__func__, "%f is supposed to lie near %.3f in slice %2hu of "
2266  "IFU %hhu", lambda, ypos, aSlice,
2267  muse_utils_get_ifu(aImage->header));
2268  }
2269  /* get both slice edges and the center */
2270  double dleft = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_LEFT],
2271  ypos, NULL),
2272  dright = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_RIGHT],
2273  ypos, NULL),
2274  dmid = (dleft + dright) / 2.;
2275  int ileft = ceil(dleft),
2276  iright = floor(dright);
2277 #if 0
2278  cpl_msg_debug(__func__, "limits at y=%f: %f < %f < %f", ypos, dleft, dmid,
2279  dright);
2280 #endif
2281 
2282  /* table to store line fits for this one arc line */
2283  cpl_table *fittable = muse_cpltable_new(muse_wavelines_def,
2284  (int)kMuseSliceHiLikelyWidth + 5);
2285 
2286  /* From the center of the slice move outwards and fit the line *
2287  * until we have arrived at the edge of the slice */
2288  double y = ypos;
2289  int i, n = 0;
2290  for (i = dmid; i >= ileft; i--) {
2291  cpl_error_code rc = muse_wave_line_fit_single(aImage, i, y, halfwidth,
2292  sigma, fittable, ++n);
2293  if (rc != CPL_ERROR_NONE) { /* do not count this line */
2294  --n;
2295  } else {
2296  double ynew = cpl_table_get(fittable, "center", n - 1, NULL);
2297  if (fabs(y - ynew) < MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT) {
2298  y = ynew;
2299  }
2300  }
2301  } /* for i (columns to left of slice middle) */
2302 #if 0
2303  printf("arc line aIdx=%u, columns i=%d...", aIdx + 1, i);
2304 #endif
2305  y = ypos; /* start at middle y position again */
2306  for (i = dmid + 1; i <= iright; i++) {
2307  cpl_error_code rc = muse_wave_line_fit_single(aImage, i, ypos, halfwidth,
2308  sigma, fittable, ++n);
2309  if (rc != CPL_ERROR_NONE) { /* do not count this line */
2310  --n;
2311  } else {
2312  double ynew = cpl_table_get(fittable, "center", n - 1, NULL);
2313  if (fabs(y - ynew) < MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT) {
2314  y = ynew;
2315  }
2316  }
2317  } /* for i (columns to right of slice middle) */
2318  /* now remove rows with invalid entries, i.e. those that were not *
2319  * filled with the properties of the fit -- cpl_table_erase_invalid() *
2320  * does not work, it deletes all columns */
2321  cpl_table_select_all(fittable);
2322  cpl_table_and_selected_invalid(fittable, "center");
2323  cpl_table_erase_selected(fittable);
2324  cpl_table_fill_column_window(fittable, "lambda", 0,
2325  cpl_table_get_nrow(fittable), lambda);
2326 #if 0
2327  printf("line %u, %d line fits\n", aIdx + 1, n);
2328  cpl_table_dump(fittable, 0, n, stdout);
2329  fflush(stdout);
2330 #endif
2331  /* and reject deviant fits */
2332  muse_wave_line_fit_iterate(fittable, lambda, aParams);
2333  int npos = cpl_table_get_nrow(fittable);
2334  if (npos <= 0) {
2335  cpl_msg_warning(__func__, "Polynomial fit failed in slice %hu of IFU %hhu "
2336  "for line %u (y-position near %.2f pix): %s", aSlice,
2337  muse_utils_get_ifu(aImage->header), aIdx + 1, ypos,
2338  cpl_error_get_message());
2339  }
2340 #if 0
2341  else {
2342  printf("%s: line %2u, %d line fits, %d with low residuals:\n", __func__,
2343  aIdx + 1, n, npos);
2344  cpl_table_dump(fittable, 0, npos, stdout);
2345  fflush(stdout);
2346  }
2347 #endif
2348  return fittable;
2349 } /* muse_wave_line_handle_singlet() */
2350 
2351 /*----------------------------------------------------------------------------*/
2387 /*----------------------------------------------------------------------------*/
2388 cpl_table *
2389 muse_wave_line_handle_multiplet(muse_image *aImage, cpl_table *aLinelist,
2390  unsigned int aIdx, cpl_polynomial *aPoly,
2391  cpl_polynomial **aTrace,
2392  const muse_wave_params *aParams,
2393  const unsigned short aSlice, int aDebug)
2394 {
2395  cpl_ensure(aImage && aLinelist && aPoly && aTrace, CPL_ERROR_NULL_INPUT, NULL);
2396 
2397  /* search for the other line(s) of the multiplet */
2398 #define MULTIPLET_SEARCH_RANGE 40. /* 40 Angstrom for NeI 7024...7059 */
2399  unsigned int nlines = 1; /* for the moment we know of one line of the multiplet */
2400  double lbda1 = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_LAMBDA, aIdx, NULL);
2401  const char *lamp1 = muse_wave_lines_get_lampname(aLinelist, aIdx);
2402  cpl_vector *lambdas = cpl_vector_new(nlines),
2403  *fluxes = cpl_vector_new(nlines);
2404  cpl_vector_set(lambdas, 0, lbda1);
2405  cpl_vector_set(fluxes, 0, cpl_table_get(aLinelist, MUSE_LINE_CATALOG_FLUX,
2406  aIdx, NULL));
2407  double lambda;
2408  unsigned int j = aIdx;
2409  for (lambda = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_LAMBDA, ++j, NULL);
2410  fabs(lambda - lbda1) < MULTIPLET_SEARCH_RANGE;
2411  lambda = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_LAMBDA, ++j, NULL)) {
2412  int quality = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_QUALITY, j, NULL);
2413  const char *lamp = muse_wave_lines_get_lampname(aLinelist, j);
2414  if (quality == 2 && !strcmp(lamp1, lamp)) { /* duplet of the same lamp */
2415  nlines++;
2416  cpl_vector_set_size(lambdas, nlines);
2417  cpl_vector_set_size(fluxes, nlines);
2418  cpl_vector_set(lambdas, nlines - 1, lambda);
2419  double flux = cpl_table_get(aLinelist, MUSE_LINE_CATALOG_FLUX, j, NULL);
2420  cpl_vector_set(fluxes, nlines - 1, flux);
2421  /* invert the sign of the quality, so that this line is not found again */
2422  cpl_table_set(aLinelist, MUSE_LINE_CATALOG_QUALITY, j, -quality);
2423  }
2424  } /* for lambda */
2425 
2426  if (aDebug >= 2) {
2427  printf("found multiplet of lamp %s with %u lines:\n", lamp1, nlines);
2428  cpl_bivector *bv = cpl_bivector_wrap_vectors(lambdas, fluxes);
2429  cpl_bivector_dump(bv, stdout);
2430  cpl_bivector_unwrap_vectors(bv);
2431  fflush(stdout);
2432  }
2433 
2434  /* positive sigma: using negative value (= fixed parameter in fit) actually *
2435  * causes higher deviations, worse RMS in the global fit, so leave it free */
2436  double sigma = kMuseSliceSlitWidthA / kMuseSpectralSamplingA / CPL_MATH_FWHM_SIG;
2437  int halfwidth = 3.*kMuseSliceSlitWidthA / kMuseSpectralSamplingA; /* 3*FWHM */
2438 
2439  cpl_vector *ypos = cpl_vector_new(nlines);
2440  for (j = 0; j < nlines; j++) {
2441  double yp = cpl_polynomial_eval_1d(aPoly, cpl_vector_get(lambdas, j), NULL);
2442  cpl_vector_set(ypos, j, yp);
2443  } /* for j */
2444  double ypos1 = cpl_vector_get(ypos, 0),
2445  ypos2 = cpl_vector_get(ypos, nlines - 1);
2446  cpl_bivector *peaks = cpl_bivector_wrap_vectors(ypos, fluxes);
2447  if ((ypos1 - halfwidth) < 1 ||
2448  (ypos2 + halfwidth) > cpl_image_get_size_y(aImage->data)) {
2449  if (aDebug >= 2) {
2450  cpl_msg_debug(__func__, "%f is supposed to lie at %.3f..%.3f in slice "
2451  "%2hu of IFU %hhu, i.e. outside!", lambda, ypos1, ypos2,
2452  aSlice, muse_utils_get_ifu(aImage->header));
2453  }
2454  cpl_bivector_delete(peaks);
2455  cpl_vector_delete(lambdas);
2456  return NULL;
2457  }
2458  if (aDebug >= 2) {
2459  cpl_msg_debug(__func__, "%f is supposed to lie at %.3f..%.3f in slice %2hu "
2460  "of IFU %hhu", lambda, ypos1, ypos2, aSlice,
2461  muse_utils_get_ifu(aImage->header));
2462  }
2463  /* get both slice edges and the center */
2464  double dleft = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_LEFT],
2465  (ypos1 + ypos2)/2., NULL),
2466  dright = cpl_polynomial_eval_1d(aTrace[MUSE_TRACE_RIGHT],
2467  (ypos1 + ypos2)/2., NULL),
2468  dmid = (dleft + dright) / 2.;
2469  int ileft = ceil(dleft),
2470  iright = floor(dright);
2471 
2472  /* table to store line fits for this one arc line */
2473  cpl_table *fittable = muse_cpltable_new(muse_wavelines_def,
2474  ((int)kMuseSliceHiLikelyWidth + 5) * nlines);
2475 
2476  /* From the center of the slice move outwards and fit the *
2477  * line until we have arrived at the edge of the slice */
2478  cpl_bivector *bp = cpl_bivector_duplicate(peaks),
2479  *bpgood = cpl_bivector_duplicate(peaks);
2480  int i, n = 0;
2481  for (i = dmid; i >= ileft; i--) {
2482  n += nlines;
2483  cpl_error_code rc = muse_wave_line_fit_multiple(aImage, i, bp, lambdas,
2484  halfwidth, sigma, fittable, n);
2485  if (rc != CPL_ERROR_NONE) { /* do not count this fit */
2486  cpl_bivector_delete(bp);
2487  bp = cpl_bivector_duplicate(bpgood);
2488  n -= nlines;
2489  } else {
2490  cpl_vector *shifts = cpl_vector_duplicate(cpl_bivector_get_x(bp));
2491  cpl_vector_subtract(shifts, cpl_bivector_get_x(bpgood));
2492  double shift = cpl_vector_get_median(shifts); /* may be non-const! */
2493  cpl_vector_delete(shifts);
2494  if (fabs(shift) < MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT) {
2495  cpl_bivector_delete(bpgood);
2496  bpgood = cpl_bivector_duplicate(bp);
2497  } else {
2498  cpl_bivector_delete(bp);
2499  bp = cpl_bivector_duplicate(bpgood);
2500  }
2501  } /* else (good fit) */
2502  } /* for i (columns to left of slice middle) */
2503  cpl_bivector_delete(bp);
2504  cpl_bivector_delete(bpgood);
2505  bp = cpl_bivector_duplicate(peaks);
2506  bpgood = cpl_bivector_duplicate(peaks);
2507  for (i = dmid + 1; i <= iright; i++) {
2508  n += nlines;
2509  cpl_error_code rc = muse_wave_line_fit_multiple(aImage, i, bp, lambdas,
2510  halfwidth, sigma, fittable, n);
2511  if (rc != CPL_ERROR_NONE) { /* do not count this fit */
2512  cpl_bivector_delete(bp);
2513  bp = cpl_bivector_duplicate(bpgood);
2514  n -= nlines;
2515  } else {
2516  cpl_vector *shifts = cpl_vector_duplicate(cpl_bivector_get_x(bp));
2517  cpl_vector_subtract(shifts, cpl_bivector_get_x(bpgood));
2518  double shift = cpl_vector_get_median(shifts); /* may be non-const! */
2519  cpl_vector_delete(shifts);
2520  if (fabs(shift) < MUSE_WAVE_LINE_HANDLE_SHIFT_LIMIT) {
2521  cpl_bivector_delete(bpgood);
2522  bpgood = cpl_bivector_duplicate(bp);
2523  } else {
2524  cpl_bivector_delete(bp);
2525  bp = cpl_bivector_duplicate(bpgood);
2526  }
2527  } /* else (good fit) */
2528  } /* for i (columns to right of slice middle) */
2529  cpl_bivector_delete(bp);
2530  cpl_bivector_delete(bpgood);
2531  /* now remove rows with invalid entries, i.e. those that *
2532  * were not filled with the properties of the fit */
2533  cpl_table_select_all(fittable);
2534  cpl_table_and_selected_invalid(fittable, "center");
2535  cpl_table_erase_selected(fittable);
2536  cpl_bivector_delete(peaks);
2537  /* and reject deviant fits */
2538  for (j = 0; j < nlines; j++) {
2539  muse_wave_line_fit_iterate(fittable, cpl_vector_get(lambdas, j), aParams);
2540  } /* for i */
2541  cpl_vector_delete(lambdas);
2542 
2543  return fittable;
2544 } /* muse_wave_line_handle_multiplet() */
2545 
2546 /*----------------------------------------------------------------------------*/
2577 /*----------------------------------------------------------------------------*/
2578 cpl_error_code
2579 muse_wave_line_fit_single(muse_image *aImage, int aX, double aY, int aHalfWidth,
2580  double aSigma, cpl_table *aFitTable, int aRowsNeeded)
2581 {
2582  cpl_ensure_code(aImage && aImage->data && aImage->stat && aFitTable,
2583  CPL_ERROR_NULL_INPUT);
2584 #if 0
2585  cpl_msg_debug(__func__, "%d %d %d -> %d, %d", aX, (int)aY, aHalfWidth,
2586  2*aHalfWidth+1, aRowsNeeded);
2587 #endif
2588  /* determine area on the input image, copy area into the vectors */
2589  cpl_vector *positions = cpl_vector_new(2*aHalfWidth+1),
2590  *values = cpl_vector_new(2*aHalfWidth+1),
2591  *valuessigma = cpl_vector_new(2*aHalfWidth+1);
2592  int i, j, ny = cpl_image_get_size_y(aImage->data);
2593  for (i = (int)aY - aHalfWidth, j = 0; i <= (int)aY + aHalfWidth && i <= ny;
2594  i++, j++) {
2595  int err;
2596  cpl_vector_set(positions, j, i);
2597  cpl_vector_set(values, j, cpl_image_get(aImage->data, aX, i, &err));
2598  cpl_vector_set(valuessigma, j,
2599  sqrt(cpl_image_get(aImage->stat, aX, i, &err)));
2600  }
2601 #if 0
2602  printf("input vectors: positions, values\n");
2603  cpl_bivector *biv = cpl_bivector_wrap_vectors(positions, values);
2604  cpl_bivector_dump(biv, stdout);
2605  fflush(stdout);
2606  cpl_bivector_unwrap_vectors(biv);
2607 #if 0
2608  printf("input vectors: values, valuessigma\n");
2609  biv = cpl_bivector_wrap_vectors(values, valuessigma);
2610  cpl_bivector_dump(biv, stdout);
2611  fflush(stdout);
2612  cpl_bivector_unwrap_vectors(biv);
2613 #endif
2614 #endif
2615 
2616  /* do the Gaussfit */
2617  cpl_errorstate prestate = cpl_errorstate_get();
2618  double center, cerror, area, bglevel, mse;
2619  cpl_matrix *covariance = NULL;
2620  cpl_fit_mode mode = CPL_FIT_ALL;
2621  double sigma = aSigma;
2622  if (sigma < 0) {
2623  mode = CPL_FIT_CENTROID | CPL_FIT_AREA | CPL_FIT_OFFSET;
2624  sigma *= -1;
2625  }
2626  cpl_error_code rc
2627  = cpl_vector_fit_gaussian(positions, NULL, values, valuessigma, mode,
2628  &center, &sigma, &area, &bglevel, &mse, NULL,
2629  &covariance);
2630 #if 0
2631  printf("--> rc=%d (%d) %s %#x %#x\n", rc, CPL_ERROR_NONE,
2632  cpl_error_get_message(), covariance, &covariance);
2633  fflush(stdout);
2634 #endif
2635  cpl_vector_delete(positions);
2636  cpl_vector_delete(values);
2637  cpl_vector_delete(valuessigma);
2638  /* Determine goodness of fit and try to treat some possible problems. *
2639  * (This is partly copied from muse_wave_lines_search().) */
2640  /* if no error happened, take the centering error from the covariance matrix */
2641  if (covariance) {
2642  cerror = sqrt(cpl_matrix_get(covariance, 0, 0));
2643  cpl_matrix_delete(covariance);
2644  } else { /* a missing covariance matrix is bad */
2645  cpl_msg_debug(__func__, "Gauss fit produced no covariance matrix (y=%.3f in "
2646  "column=%d): %s", aY, aX, cpl_error_get_message());
2647  cpl_errorstate_set(prestate);
2648  return rc == CPL_ERROR_NONE ? CPL_ERROR_ILLEGAL_OUTPUT : rc;
2649  }
2650  if (rc == CPL_ERROR_CONTINUE) { /* fit didn't converge */
2651  /* estimate position error as sigma^2/area as CPL docs *
2652  * suggest and pretend that everything worked fine */
2653  cerror = sqrt(sigma * sigma / area);
2654  cpl_errorstate_set(prestate);
2655  rc = CPL_ERROR_NONE;
2656  }
2657  if (rc != CPL_ERROR_NONE) {
2658  /* some other, probably more serious error, don't use this fit */
2659  cpl_msg_debug(__func__, "Gauss fit failed with some error (y=%.3f in "
2660  "column=%d): %s", aY, aX, cpl_error_get_message());
2661  cpl_errorstate_set(prestate);
2662  return rc;
2663  }
2664  if (fabs(center - aY) > MUSE_WAVE_LINE_FIT_MAXSHIFT) {
2665  /* too big shift, don't use this fit */
2666  cpl_msg_debug(__func__, "Gauss fit gave unexpectedly large offset "
2667  "(shifted %.3f pix from y=%.3f in column=%d)",
2668  center - aY, aY, aX);
2669  return CPL_ERROR_ACCESS_OUT_OF_RANGE;
2670  }
2671 
2672  /* store the results in the table, after making sure it is large enough */
2673  if (cpl_table_get_nrow(aFitTable) < aRowsNeeded) {
2674  cpl_table_set_size(aFitTable, aRowsNeeded);
2675  }
2676  cpl_table_set(aFitTable, "center", aRowsNeeded - 1, center);
2677  cpl_table_set(aFitTable, "cerr", aRowsNeeded - 1, cerror);
2678  cpl_table_set(aFitTable, "sigma", aRowsNeeded - 1, sigma);
2679  if (mode == CPL_FIT_ALL) { /* is FWHM reference line, store the FWHM */
2680  cpl_table_set(aFitTable, "fwhm", aRowsNeeded - 1, CPL_MATH_FWHM_SIG * sigma);
2681  }
2682  cpl_table_set(aFitTable, "flux", aRowsNeeded - 1, area);
2683  cpl_table_set(aFitTable, "bg", aRowsNeeded - 1, bglevel);
2684  cpl_table_set(aFitTable, "mse", aRowsNeeded - 1, mse);
2685  cpl_table_set(aFitTable, "x", aRowsNeeded - 1, aX);
2686  cpl_table_set(aFitTable, "y", aRowsNeeded - 1, aY);
2687 
2688 #if 0
2689  printf("--> Gaussian fit: %f +/- %f, %f\n", center, cerror, area);
2690  fflush(stdout);
2691 #endif
2692  return rc;
2693 } /* muse_wave_line_fit_single() */
2694 
2695 /*----------------------------------------------------------------------------*/
2733 /*----------------------------------------------------------------------------*/
2734 cpl_error_code
2735 muse_wave_line_fit_multiple(muse_image *aImage, int aX, cpl_bivector *aPeaks,
2736  cpl_vector *aLambdas, int aHalfWidth, double aSigma,
2737  cpl_table *aFitTable, int aRowsNeeded)
2738 {
2739  cpl_ensure_code(aImage && aImage->data && aImage->stat && aFitTable,
2740  CPL_ERROR_NULL_INPUT);
2741  cpl_vector *ypeaks = cpl_bivector_get_x(aPeaks),
2742  *fluxes = cpl_bivector_get_y(aPeaks);
2743  int np = cpl_vector_get_size(ypeaks);
2744  double yp1 = cpl_vector_get(ypeaks, 0),
2745  yp2 = cpl_vector_get(ypeaks, np - 1);
2746  int i1 = floor(yp1),
2747  i2 = ceil(yp2);
2748 #if 0
2749  cpl_msg_debug(__func__, "%d %d..%d %d -> %d", aX, i1, i2, aHalfWidth,
2750  aRowsNeeded);
2751 #endif
2752  /* determine area on the input image, copy area into the vectors */
2753  int npoints = (i2 + aHalfWidth) - (i1 - aHalfWidth) + 1;
2754  cpl_vector *positions = cpl_vector_new(npoints),
2755  *values = cpl_vector_new(npoints),
2756  *valuessigma = cpl_vector_new(npoints);
2757  double minval = DBL_MAX; /* 1st-guess fit value */
2758  int i, j, ny = cpl_image_get_size_y(aImage->data);
2759  for (i = i1 - aHalfWidth, j = 0; i <= i2 + aHalfWidth && i <= ny; i++, j++) {
2760  int err;
2761  cpl_vector_set(positions, j, i);
2762  double value = cpl_image_get(aImage->data, aX, i, &err);
2763  cpl_vector_set(values, j, value);
2764  if (value < minval) {
2765  minval = value;
2766  }
2767  cpl_vector_set(valuessigma, j,
2768  sqrt(cpl_image_get(aImage->stat, aX, i, &err)));
2769  }
2770  minval = minval > 0 ? minval : 0.;
2771 #if 0
2772  printf("input vectors: positions, values\n");
2773  cpl_bivector *biv = cpl_bivector_wrap_vectors(positions, values);
2774  cpl_bivector_dump(biv, stdout);
2775  fflush(stdout);
2776  cpl_bivector_unwrap_vectors(biv);
2777 #endif
2778  cpl_bivector *bvalues = cpl_bivector_wrap_vectors(values, valuessigma);
2779 
2780  /* do the multi-Gauss fit */
2781  /* parameters: slope, background, sigma, np * center, np * flux */
2782  cpl_vector *poly = cpl_vector_new(2);
2783  cpl_vector_set(poly, 0, minval);
2784  cpl_vector_set(poly, 1, 0.);
2785  /* record the index of the expected strongest line, and its position */
2786  cpl_array *afluxes = cpl_array_wrap_double(cpl_vector_get_data(fluxes), np);
2787  cpl_size imaxflux;
2788  cpl_array_get_maxpos(afluxes, &imaxflux);
2789  double yposmax = cpl_vector_get(ypeaks, imaxflux);
2790  cpl_array_unwrap(afluxes);
2791  /* do the fit, store the errorstate before */
2792  cpl_errorstate prestate = cpl_errorstate_get();
2793  double sigma = aSigma,
2794  mse, chisq;
2795  cpl_matrix *covariance;
2796  cpl_error_code rc = muse_utils_fit_multigauss_1d(positions, bvalues, ypeaks,
2797  &sigma, fluxes, poly,
2798  &mse, &chisq, &covariance);
2799  cpl_vector_delete(positions);
2800  cpl_bivector_delete(bvalues);
2801 
2802  /* Determine goodness of fit and try to treat some possible problems. *
2803  * (This is partly copied from muse_wave_lines_search().) */
2804  /* if no error happened, take the centering error from the covariance matrix */
2805  if (!covariance) { /* a missing covariance matrix is bad */
2806  cpl_msg_debug(__func__, "Multi-Gauss fit produced no covariance matrix (y=%."
2807  "3f..%.3f in column=%d): %s", yp1, yp2, aX, cpl_error_get_message());
2808  cpl_errorstate_set(prestate);
2809  cpl_vector_delete(poly);
2810  return rc == CPL_ERROR_NONE ? CPL_ERROR_ILLEGAL_OUTPUT : rc;
2811  }
2812  if (rc != CPL_ERROR_NONE) {
2813  /* some other, probably more serious error, don't use this fit */
2814  cpl_msg_debug(__func__, "Multi-Gauss fit failed with some error (y=%.3f..%."
2815  "3f in column=%d): %s", yp1, yp2, aX, cpl_error_get_message());
2816  cpl_errorstate_set(prestate);
2817  cpl_matrix_delete(covariance);
2818  cpl_vector_delete(poly);
2819  return rc;
2820  }
2821  /* did the center of the brightest line shift too much? */
2822  double yfitmax = cpl_vector_get(ypeaks, imaxflux);
2823  if (fabs(yposmax - yfitmax) > MUSE_WAVE_LINE_FIT_MAXSHIFT) {
2824  cpl_msg_debug(__func__, "Multi-Gauss fit gave unexpectedly large offset "
2825  "(shifted %.3f pix from y=%.3f..%.3f in column=%d)",
2826  yposmax - yfitmax, yp1, yp2, aX);
2827  cpl_matrix_delete(covariance);
2828  cpl_vector_delete(poly);
2829  return CPL_ERROR_ACCESS_OUT_OF_RANGE;
2830  }
2831  /* are any of the fluxes negative? */
2832  double yfitmin = cpl_vector_get_min(fluxes);
2833  if (yfitmin < 0) {
2834  cpl_msg_debug(__func__, "Multi-Gauss fit gave negative flux (%e in "
2835  "multiplet from y=%.3f..%.3f in column=%d)", yfitmin,
2836  yp1, yp2, aX);
2837  cpl_matrix_delete(covariance);
2838  cpl_vector_delete(poly);
2839  return CPL_ERROR_ILLEGAL_OUTPUT;
2840  }
2841 
2842  /* store the results in the table, after making sure it is large enough */
2843  if (cpl_table_get_nrow(aFitTable) < aRowsNeeded) {
2844  cpl_table_set_size(aFitTable, aRowsNeeded);
2845  }
2846  /* first record the common values */
2847  cpl_table_fill_column_window(aFitTable, "mse", aRowsNeeded - np, np, mse);
2848  cpl_table_fill_column_window(aFitTable, "x", aRowsNeeded - np, np, aX);
2849  cpl_table_fill_column_window(aFitTable, "sigma", aRowsNeeded - np, np, sigma);
2850  /* multiplets are never FWHM reference lines, do not store FWHM */
2851  for (i = 0, j = aRowsNeeded - np; i < np; i++, j++) {
2852  cpl_table_set(aFitTable, "lambda", j, cpl_vector_get(aLambdas, i));
2853  cpl_table_set(aFitTable, "y", j, cpl_vector_get(ypeaks, i));
2854  double center = cpl_vector_get(ypeaks, i);
2855  cpl_table_set(aFitTable, "center", j, center);
2856  /* centroid errors are starting with row/column index 3 in the matrix, *
2857  * the first ones are the two polynomial coeffs and the sigma */
2858  double cerror = sqrt(cpl_matrix_get(covariance, 3 + i, 3 + i));
2859  cpl_table_set(aFitTable, "cerr", j, cerror);
2860  cpl_table_set(aFitTable, "flux", j, cpl_vector_get(fluxes, i));
2861  /* evaluate the linear background level at the fitted center */
2862  double bg = cpl_vector_get(poly, 0) + cpl_vector_get(poly, 1) * center;
2863  cpl_table_set(aFitTable, "bg", j, bg);
2864  } /* for i, j */
2865  cpl_vector_delete(poly);
2866  cpl_matrix_delete(covariance);
2867 #if 0
2868  printf("stored %d fitted multiplet values:\n", np);
2869  cpl_table_dump(aFitTable, aRowsNeeded - np - 2 < 0 ? 0 : aRowsNeeded - np - 2, np + 5, stdout);
2870  fflush(stdout);
2871 #endif
2872 
2873  return rc;
2874 } /* muse_wave_line_fit_multiple() */
2875 
2876 /*----------------------------------------------------------------------------*/
2915 /*----------------------------------------------------------------------------*/
2916 cpl_error_code
2917 muse_wave_line_fit_iterate(cpl_table *aFitTable, double aLambda,
2918  const muse_wave_params *aParams)
2919 {
2920  cpl_ensure_code(aFitTable, CPL_ERROR_NULL_INPUT);
2921  int npos = cpl_table_get_nrow(aFitTable);
2922  cpl_ensure_code(npos > 0, CPL_ERROR_ILLEGAL_INPUT);
2923  double rsigma = aParams->linesigma < 0 ? 2.5 : aParams->linesigma;
2924 
2925  /* select all entries with the requested wavelength */
2926  cpl_table *fittable = NULL;
2927  if (aLambda > 0) {
2928  cpl_table_unselect_all(aFitTable);
2929  cpl_table_or_selected_double(aFitTable, "lambda", CPL_EQUAL_TO, aLambda);
2930  npos = cpl_table_count_selected(aFitTable);
2931  cpl_ensure_code(npos > 0, CPL_ERROR_ILLEGAL_INPUT);
2932  fittable = cpl_table_extract_selected(aFitTable);
2933  cpl_table_erase_selected(aFitTable);
2934  } else {
2935  fittable = aFitTable;
2936  }
2937 
2938  /* convert "x" and "center" columns into matrix and vector for the fit */
2939  cpl_matrix *pos = cpl_matrix_new(1, npos);
2940  cpl_vector *val = cpl_vector_new(npos);
2941  int i;
2942  for (i = 0; i < npos; i++) {
2943  cpl_matrix_set(pos, 0, i, cpl_table_get(fittable, "x", i, NULL));
2944  cpl_vector_set(val, i, cpl_table_get(fittable, "center", i, NULL));
2945  }
2946 
2947  /* use the iterative polynomial field to throw out bad entries in fittable */
2948  cpl_errorstate state = cpl_errorstate_get();
2949  double mse;
2950  cpl_polynomial *fit = muse_utils_iterate_fit_polynomial(pos, val, NULL, fittable,
2951  aParams->xorder, rsigma,
2952  &mse, NULL);
2953  cpl_matrix_delete(pos);
2954  cpl_vector_delete(val);
2955  cpl_polynomial_delete(fit);
2956  if (!cpl_errorstate_is_equal(state)) {
2957  /* this implies less than xorder entries left, and is really bad; set the *
2958  * error to something high enough to basically exclude these points */
2959  cpl_table_fill_column_window(fittable, "cerr",
2960  0, cpl_table_get_nrow(fittable), 10.);
2961  } else if (aParams->fitweighting == MUSE_WAVE_WEIGHTING_SCATTER) {
2962  /* replace the original centroid error with the fit RMS */
2963  cpl_table_fill_column_window(fittable, "cerr",
2964  0, cpl_table_get_nrow(fittable), sqrt(mse));
2965  } else if (aParams->fitweighting == MUSE_WAVE_WEIGHTING_CERRSCATTER) {
2966  /* add RMS of fit to individual error in quadrature: *
2967  * cerr = sqrt(cerr^2 + mse) */
2968  cpl_table_power_column(fittable, "cerr", 2.); /* cerr^2 */
2969  cpl_table_add_scalar(fittable, "cerr", mse); /* + mse */
2970  cpl_table_power_column(fittable, "cerr", 0.5); /* sqrt() */
2971  }
2972 #if 0
2973  printf("line at %.3f Angstrom: rms = %f (mean error %f)\n", aLambda,
2974  sqrt(mse), cpl_table_get_column_mean(fittable, "cerr"));
2975  cpl_table_dump(fittable, 0, npos, stdout);
2976  fflush(stdout);
2977 #endif
2978  if (aLambda > 0) { /* need to copy results back into input table */
2979  cpl_table_insert(aFitTable, fittable, cpl_table_get_nrow(aFitTable));
2980  cpl_table_delete(fittable);
2981  }
2982  return CPL_ERROR_NONE;
2983 } /* muse_wave_line_fit_iterate() */
2984 
2985 /*----------------------------------------------------------------------------*/
2997 /*----------------------------------------------------------------------------*/
2998 static inline double
2999 muse_wave_ipow(double x, unsigned int y)
3000 {
3001  if (!y) {
3002  return 1;
3003  }
3004  if (y == 1) {
3005  return x;
3006  }
3007  double x2 = x*x;
3008  if (y == 2) {
3009  return x2;
3010  }
3011  if (y == 3) {
3012  return x2*x;
3013  }
3014  if (y == 4) {
3015  return x2*x2;
3016  }
3017  double result;
3018  /* Handle least significant bit in y here in order to avoid an unnecessary *
3019  * multiplication of x - which could cause an over- or underflow */
3020  result = y & 1 ? x : 1.; /* 0^0 is 1, while any other power of 0 is 0 */
3021  while (y >>= 1) {
3022  x *= x;
3023  if (y & 1) { /* Process least significant bit in y */
3024  result *= x;
3025  }
3026  }
3027  return result;
3028 } /* muse_wave_ipow() */
3029 
3030 /*----------------------------------------------------------------------------*/
3045 /*----------------------------------------------------------------------------*/
3046 static int
3047 muse_wave_poly_2d(const double xy[], const double p[], double *f)
3048 {
3049  unsigned short xorder = p[0],
3050  yorder = p[1];
3051  /* add up the polynomial orders */
3052  *f = 0;
3053  double x = xy[0],
3054  y = xy[1];
3055  unsigned int i, idx = 2;
3056  for (i = 0; i <= xorder; i++) {
3057  double xi = muse_wave_ipow(x, i);
3058  unsigned int j; /* y order */
3059  for (j = 0; j <= yorder; j++) {
3060  *f += p[idx++] * xi * muse_wave_ipow(y, j);
3061  } /* for j */
3062  } /* for i */
3063  return 0;
3064 } /* muse_wave_poly_2d() */
3065 
3066 /*----------------------------------------------------------------------------*/
3079 /*----------------------------------------------------------------------------*/
3080 static int
3081 muse_wave_dpoly_2d(const double xy[], const double p[], double f[])
3082 {
3083  unsigned short xorder = p[0],
3084  yorder = p[1];
3085  f[0] = f[1] = 0; /* derivatives regarding the polynomial orders are zero */
3086  /* now the real derivatives, they don't contain the coefficients any more */
3087  double x = xy[0],
3088  y = xy[1];
3089  unsigned int i, idx = 2;
3090  for (i = 0; i <= xorder; i++) {
3091  double xi = muse_wave_ipow(x, i);
3092  unsigned int j; /* y order */
3093  for (j = 0; j <= yorder; j++) {
3094  f[idx++] = xi * muse_wave_ipow(y, j);
3095  } /* for j */
3096  } /* for i */
3097  return 0;
3098 } /* muse_wave_poly_x2y5_deriv() */
3099 
3100 /*----------------------------------------------------------------------------*/
3142 /*----------------------------------------------------------------------------*/
3143 cpl_error_code
3144 muse_wave_poly_fit(cpl_matrix *aXYPos, cpl_vector *aLambdas, cpl_vector *aDLambdas,
3145  cpl_polynomial **aPoly, double *aMSE, muse_wave_params *aParams,
3146  const unsigned short aSlice)
3147 {
3148  cpl_ensure_code(aPoly && aXYPos && aLambdas, CPL_ERROR_NULL_INPUT);
3149 
3150  /* filter the input measurements by wavelengths, and exclude those *
3151  * outside the valid range, depending on the instrument mode */
3152  int ii, nin = cpl_vector_get_size(aLambdas),
3153  nout = 0;
3154 #if 0
3155  cpl_msg_debug(__func__, "Testing coverage for mode %d.", aParams->mode);
3156 #endif
3157  for (ii = 0; ii < cpl_vector_get_size(aLambdas); ii++) {
3158  double lambda = cpl_vector_get(aLambdas, ii);
3159  if (muse_wave_lines_covered_by_data(lambda, aParams->mode)) {
3160  nout++;
3161  continue;
3162  }
3163 #if 0
3164  cpl_msg_debug(__func__, "line at %.3f Angstrom outside range", lambda);
3165 #endif
3166  cpl_matrix_erase_columns(aXYPos, ii, 1);
3167  muse_cplvector_erase_element(aLambdas, ii);
3168  if (aDLambdas) {
3169  muse_cplvector_erase_element(aDLambdas, ii);
3170  }
3171  ii--; /* stay at this position */
3172  }
3173  cpl_msg_debug(__func__, "%d of %d measurements should have valid wavelengths.",
3174  nout, nin);
3175 
3176 #if 0
3177  printf("aXYPos (%"CPL_SIZE_FORMAT"x%"CPL_SIZE_FORMAT"):\n",
3178  cpl_matrix_get_ncol(aXYPos), cpl_matrix_get_nrow(aXYPos));
3179  cpl_matrix_dump(aXYPos, stdout);
3180  printf("aLambdas (%"CPL_SIZE_FORMAT"):\n", cpl_vector_get_size(aLambdas));
3181  cpl_vector_dump(aLambdas, stdout);
3182  printf("aDLambdas (%"CPL_SIZE_FORMAT"):\n", cpl_vector_get_size(aDLambdas));
3183  cpl_vector_dump(aDLambdas, stdout);
3184  fflush(stdout);
3185 #endif
3186  double rsigma = aParams->fitsigma < 0 ? 3.0 : aParams->fitsigma;
3187  int debug = getenv("MUSE_DEBUG_WAVECAL")
3188  ? atoi(getenv("MUSE_DEBUG_WAVECAL")) : 0;
3189  /* prepare the polynomial for two dimensions */
3190  *aPoly = cpl_polynomial_new(2);
3191 
3192  double rms = -1; /* start with negative (= invalid) RMS */
3193  int large_residuals = 1, /* init to force a first fit */
3194  niter = 1;
3195  while (large_residuals > 0) {
3196  /* set some typical fit error by default */
3197  cpl_error_code errfit = CPL_ERROR_SINGULAR_MATRIX;
3198 
3199  if (aDLambdas) { /* use our own polynomial fitting with weighting */
3200  /* the lvmq function wants N rows of dimension D=2 *
3201  * instead of the other way around... */
3202  cpl_matrix *xypos = cpl_matrix_transpose_create(aXYPos);
3203 
3204  int npar = 2 + (aParams->xorder + 1) * (aParams->yorder + 1);
3205  cpl_vector *pars = cpl_vector_new(npar);
3206  /* pre-fill all but the zero-order with zero as the first guess value */
3207  cpl_vector_fill(pars, 0.);
3208  /* the polynomial orders go first into the array */
3209  cpl_vector_set(pars, 0, aParams->xorder);
3210  cpl_vector_set(pars, 1, aParams->yorder);
3211  cpl_vector_set(pars, 2, 5000.); /* near lower edge of lambda range */
3212  cpl_array *aflags = cpl_array_new(npar, CPL_TYPE_INT);
3213  /* most are real (free == 1) parameters, except the first two */
3214  cpl_array_set_int(aflags, 0, 0); /* fixed xorder */
3215  cpl_array_set_int(aflags, 1, 0); /* fixed yorder */
3216  cpl_array_fill_window_int(aflags, 2, npar - 2, 1);
3217  int *pflags = cpl_array_get_data_int(aflags);
3218  errfit = cpl_fit_lvmq(xypos, /* sigma_x (unsupported) */NULL,
3219  aLambdas, aDLambdas, pars, pflags,
3220  muse_wave_poly_2d, muse_wave_dpoly_2d,
3221  CPL_FIT_LVMQ_TOLERANCE, CPL_FIT_LVMQ_COUNT,
3222  CPL_FIT_LVMQ_MAXITER, NULL, NULL, NULL);
3223  cpl_array_delete(aflags);
3224  cpl_size k = 2; /* coefficients start at the 3rd vector entry */
3225  unsigned short i;
3226  for (i = 0; i <= aParams->xorder; i++) {
3227  unsigned short j;
3228  for (j = 0; j <= aParams->yorder; j++) {
3229  cpl_size pows[2] = { i, j }; /* trick to access the polynomial */
3230  cpl_polynomial_set_coeff(*aPoly, pows,
3231  cpl_vector_get(pars, k++));
3232  } /* for j */
3233  } /* for i */
3234  cpl_matrix_delete(xypos);
3235  cpl_vector_delete(pars);
3236  } else {
3237  /* use CPL polynomial fitting for everything else */
3238  const cpl_boolean sym = CPL_FALSE;
3239  const cpl_size mindeg2d[] = { 0, 0 },
3240  maxdeg2d[] = { aParams->xorder, aParams->yorder };
3241  errfit = cpl_polynomial_fit(*aPoly, aXYPos, &sym, aLambdas, NULL,
3242  CPL_TRUE, mindeg2d, maxdeg2d);
3243  } /* CPL polynomial fitting */
3244 #if 0
3245  printf("polynomial (orders=%hu/%hu, degree=%"CPL_SIZE_FORMAT"):\n",
3246  aParams->xorder, aParams->yorder, cpl_polynomial_get_degree(*aPoly));
3247  cpl_polynomial_dump(*aPoly, stdout);
3248  fflush(stdout);
3249 #endif
3250 
3251  if (errfit) {
3252  /* the fit failed, complain and abort */
3253  cpl_msg_error(__func__, "The polynomial fit in slice %hu failed: %s",
3254  aSlice, cpl_error_get_message());
3255 
3256 #if 0
3257  /* more fancy plotting of input datapoints *
3258  * (color coded lambda over x/y-position) */
3259  FILE *gp = popen("gnuplot -persist", "w");
3260  if (gp) {
3261  /* plot title with fit details */
3262  fprintf(gp, "set title \"2D polynomial fit residuals (failed fit: "
3263  "\'%s\')\n", cpl_error_get_message());
3264  /* set nice palette */
3265  fprintf(gp, "set palette defined ( 0 \"dark-violet\","
3266  "1 \"dark-blue\", 4 \"green\", 6 \"yellow\", 8 \"orange\","
3267  "9 \"red\", 10 \"dark-red\")\n");
3268  fprintf(gp, "unset key\n");
3269  cpl_matrix *xpos = cpl_matrix_extract_row(aXYPos, 0);
3270  fprintf(gp, "set xrange [%f:%f]\n", cpl_matrix_get_min(xpos),
3271  cpl_matrix_get_max(xpos));
3272  cpl_matrix_delete(xpos);
3273  fprintf(gp, "set cbrange [%f:%f]\n", cpl_vector_get_min(aLambdas),
3274  cpl_vector_get_max(aLambdas));
3275  fprintf(gp, "plot \"-\" w p pal\n");
3276 
3277  printf("# X Y lambda\n");
3278  int n;
3279  for (n = 0; n < cpl_vector_get_size(aLambdas); n++) {
3280  printf("%4d %7.2f %8.3f\n", (int)cpl_matrix_get(aXYPos, 0, n),
3281  cpl_matrix_get(aXYPos, 1, n), cpl_vector_get(aLambdas, n));
3282  fprintf(gp, "%f %f %f\n", cpl_matrix_get(aXYPos, 0, n),
3283  cpl_matrix_get(aXYPos, 1, n), cpl_vector_get(aLambdas, n));
3284  }
3285  fflush(stdout);
3286  fprintf(gp, "EOF\n");
3287  pclose(gp);
3288  }
3289 #endif
3290 
3291  /* make sure we delete the polynomial */
3292  cpl_polynomial_delete(*aPoly);
3293  *aPoly = NULL; /* make sure we can detect the failure */
3294  return CPL_ERROR_ILLEGAL_OUTPUT;
3295  } /* if failed fit */
3296 
3297  int npoints = cpl_vector_get_size(aLambdas);
3298  cpl_vector *res = cpl_vector_new(npoints);
3299  /* the aDLambdas parameter (the errors of the datapoints) do not *
3300  * change the filled vector, just the chi**2 which we don't need */
3301  cpl_vector_fill_polynomial_fit_residual(res, aLambdas, aDLambdas, *aPoly,
3302  aXYPos, NULL);
3303  /* compute the mean-squared error, without any weighting, in any case */
3304  double mse = 0;
3305  /* compute a (weighted) mean-squared error, if we know the error bars */
3306  if (aDLambdas) { /* no errors given, equal weights for all points */
3307  double wsum = 0;
3308  int i;
3309  for (i = 0; i < npoints; i++) {
3310  double weight = 1. / cpl_vector_get(aDLambdas, i);
3311  mse += pow(cpl_vector_get(res, i), 2) * weight;
3312  wsum += weight;
3313  } /* for i */
3314  mse /= wsum;
3315  } else {
3316  mse = cpl_vector_product(res, res) / npoints;
3317  }
3318  double rlimit = rsigma * sqrt(mse); /* limit using possibly weighted RMS */
3319  if (debug) {
3320  printf("Resulting 2D polynomial of slice %hu (%d points, RMS = %f dRMS = "
3321  "%f), %.1f-sigma limit now %f (target RMS = %f):\n", aSlice,
3322  npoints, sqrt(mse), rms < 0 ? 0. : rms - sqrt(mse), rsigma, rlimit,
3323  aParams->targetrms);
3324  if (debug > 1) {
3325  cpl_polynomial_dump(*aPoly, stdout);
3326  }
3327  fflush(stdout);
3328  }
3329 #if 0
3330  /* simple plotting of intermediate residuals *
3331  * (value over arbitrary x-position in vector) */
3332  cpl_plot_vector("set title \"res\"\n", "", "", res);
3333 #endif
3334  if (aParams->rflag) { /* dump debug information into table, if requested */
3335  int nnew = cpl_vector_get_size(res),
3336  nrow = aParams->residuals ? cpl_table_get_nrow(aParams->residuals) : 0;
3337  /* create the residuals table if wanted and not yet existing */
3338  if (!aParams->residuals) {
3339  aParams->residuals = muse_cpltable_new(muse_wavedebug_def, nnew);
3340  cpl_table_set_column_savetype(aParams->residuals, "slice", CPL_TYPE_UCHAR);
3341  } else { /* resize table as needed to fill in the new entries */
3342  cpl_table_set_size(aParams->residuals, nrow + nnew);
3343  }
3344  cpl_table_fill_column_window_int(aParams->residuals, "slice",
3345  nrow, nnew, aSlice);
3346  cpl_table_fill_column_window_int(aParams->residuals, "iteration",
3347  nrow, nnew, niter);
3348  cpl_table_fill_column_window_double(aParams->residuals, "rejlimit",
3349  nrow, nnew, rlimit);
3350  int n;
3351  for (n = 0; n < cpl_vector_get_size(res); n++) {
3352  cpl_table_set_int(aParams->residuals, "x", nrow + n,
3353  (int)cpl_matrix_get(aXYPos, 0, n));
3354  cpl_table_set_float(aParams->residuals, "y", nrow + n,
3355  cpl_matrix_get(aXYPos, 1, n));
3356  cpl_table_set_float(aParams->residuals, "lambda", nrow + n,
3357  cpl_vector_get(aLambdas, n));
3358  cpl_table_set_double(aParams->residuals, "residual", nrow + n,
3359  cpl_vector_get(res, n));
3360  } /* for n (all wavelengths) */
3361  if (debug) {
3362  cpl_msg_debug(__func__, "%"CPL_SIZE_FORMAT" entries in residuals table "
3363  "after iteration %d of slice %hu",
3364  cpl_table_get_nrow(aParams->residuals), niter, aSlice);
3365  }
3366  } /* if debug table */
3367  niter++;
3368 
3369  /* if the fit is already good enough, we don't *
3370  * need to go through point rejection any more */
3371  cpl_boolean isgoodenough = (rms < 0 ? CPL_FALSE : rms - sqrt(mse) < 0.001)
3372  || (sqrt(mse) <= aParams->targetrms);
3373  large_residuals = 0;
3374  int i;
3375  for (i = 0; !isgoodenough && i < cpl_vector_get_size(res); i++) {
3376  /* compare this residual value to the RMS value */
3377  if (fabs(cpl_vector_get(res, i)) < rlimit) {
3378  /* good fit at this point */
3379  continue;
3380  }
3381  /* bad residual, remove element, from residuals vector and the data vectors */
3382 #if 0
3383  cpl_msg_debug(__func__, "residual = %f (position %fx%f, lambda=%f+/-%f)",
3384  cpl_vector_get(res, i),
3385  cpl_matrix_get(aXYPos, 0, i), cpl_matrix_get(aXYPos, 1, i),
3386  cpl_vector_get(aLambdas, i),
3387  aDLambdas ? cpl_vector_get(aDLambdas, i) : 1.);
3388 #endif
3389  if (cpl_vector_get_size(res) == 1) {
3390  cpl_msg_debug(__func__, "trying to remove the last vector/matrix "
3391  "element when checking against fit sigma (slice %hu)",
3392  aSlice);
3393  break;
3394  }
3395  /* remove bad element from the fit structures... */
3397  cpl_matrix_erase_columns(aXYPos, i, 1);
3398  muse_cplvector_erase_element(aLambdas, i);
3399  if (aDLambdas) {
3400  muse_cplvector_erase_element(aDLambdas, i);
3401  }
3402 
3403  large_residuals++;
3404  i--; /* we stay at this position to see what moved here */
3405  } /* for i */
3406 
3407  rms = sqrt(mse);
3408  if (!large_residuals) {
3409  /* no large residuals (any more), so this is the final solution */
3410 
3411  /* save the mean squared error */
3412  if (aMSE) {
3413  *aMSE = mse;
3414  }
3415  } /* if !large_residuals */
3416 
3417  cpl_vector_delete(res);
3418  } /* while large_residuals */
3419 
3420  return CPL_ERROR_NONE;
3421 } /* muse_wave_poly_fit() */
3422 
3423 /*----------------------------------------------------------------------------*/
3433 /*----------------------------------------------------------------------------*/
3434 cpl_table *
3435 muse_wave_table_create(const unsigned short aNSlices, const unsigned short aXOrder,
3436  const unsigned short aYOrder)
3437 {
3438  cpl_table *table = cpl_table_new(aNSlices);
3439  cpl_ensure(table, CPL_ERROR_UNSPECIFIED, NULL);
3440 
3441  /* create column for slice number*/
3442  cpl_table_new_column(table, MUSE_WAVECAL_TABLE_COL_SLICE_NO, CPL_TYPE_INT);
3443  cpl_table_set_column_unit(table, MUSE_WAVECAL_TABLE_COL_SLICE_NO, "No");
3444  cpl_table_set_column_format(table, MUSE_WAVECAL_TABLE_COL_SLICE_NO, "%2d");
3445 
3446  /* create columns for all coefficients, so loop over fit orders */
3447  unsigned short i;
3448  for (i = 0; i <= aXOrder; i++) {
3449  unsigned short j;
3450  for (j = 0; j <= aYOrder; j++) {
3451  /* create column name, start coefficient names at 0 */
3452  char *colname = cpl_sprintf(MUSE_WAVECAL_TABLE_COL_COEFF, i, j);
3453  cpl_table_new_column(table, colname, CPL_TYPE_DOUBLE);
3454  /* fit coeff are in wavelength space */
3455  cpl_table_set_column_unit(table, colname, "Angstrom");
3456  cpl_table_set_column_format(table, colname, "%12.5e");
3457  cpl_free(colname);
3458  }
3459  }
3460  /* create column for mean squared error of fitting parameters */
3461  cpl_table_new_column(table, MUSE_WAVECAL_TABLE_COL_MSE, CPL_TYPE_DOUBLE);
3462  cpl_table_set_column_format(table, MUSE_WAVECAL_TABLE_COL_MSE, "%f");
3463 
3464  return table;
3465 } /* muse_wave_table_create() */
3466 
3467 /*----------------------------------------------------------------------------*/
3481 /*----------------------------------------------------------------------------*/
3482 cpl_error_code
3483 muse_wave_table_add_poly(cpl_table *aTable, cpl_polynomial *aPoly,
3484  double aMSE, unsigned short aXOrder,
3485  unsigned short aYOrder, const unsigned short aRow)
3486 {
3487  cpl_ensure_code(aTable && aPoly, CPL_ERROR_NULL_INPUT);
3488  cpl_ensure_code(cpl_polynomial_get_dimension(aPoly) == 2,
3489  CPL_ERROR_ILLEGAL_INPUT);
3490 
3491  /* row numbers start at 0 but the numbers in the table should start with 1 */
3492  cpl_table_set_int(aTable, MUSE_WAVECAL_TABLE_COL_SLICE_NO, aRow, aRow + 1);
3493  cpl_table_set_double(aTable, MUSE_WAVECAL_TABLE_COL_MSE, aRow, aMSE);
3494 
3495  /* i and j loop over all orders of the polynomial */
3496  unsigned short i;
3497  for (i = 0; i <= aXOrder; i++) {
3498  unsigned short j;
3499  for (j = 0; j <= aYOrder; j++) {
3500  cpl_size pows[2] = { i, j }; /* trick to access the polynomial */
3501 
3502  char *colname = cpl_sprintf(MUSE_WAVECAL_TABLE_COL_COEFF, i, j);
3503  cpl_error_code rc = cpl_table_set_double(aTable, colname, aRow,
3504  cpl_polynomial_get_coeff(aPoly,
3505  pows));
3506  if (rc != CPL_ERROR_NONE) {
3507  cpl_msg_warning(__func__, "Problem writing %f to field %s in "
3508  "wavelength table: %s",
3509  cpl_polynomial_get_coeff(aPoly, pows), colname,
3510  cpl_error_get_message());
3511  cpl_polynomial_dump(aPoly, stdout);
3512  cpl_table_dump(aTable, aRow, 1, stdout);
3513  fflush(stdout);
3514  }
3515  cpl_free(colname);
3516  } /* for j (polynomial orders in y) */
3517  } /* for i (polynomial orders in x) */
3518 
3519  return CPL_ERROR_NONE;
3520 } /* muse_wave_table_add_poly() */
3521 
3522 /*----------------------------------------------------------------------------*/
3537 /*----------------------------------------------------------------------------*/
3538 cpl_error_code
3539 muse_wave_table_get_orders(const cpl_table *aWave, unsigned short *aXOrder,
3540  unsigned short *aYOrder)
3541 {
3542  cpl_ensure_code(aWave && aXOrder && aYOrder, CPL_ERROR_NULL_INPUT);
3543  cpl_array *cols = cpl_table_get_column_names(aWave);
3544 
3545  /* second last entry will contain highest order (last is MSE) */
3546  const char *highcol = cpl_array_get_string(cols, cpl_array_get_size(cols) - 2);
3547  char *colname = cpl_strdup(highcol); /* duplicate it so that we can manipulate it */
3548  cpl_array_delete(cols);
3549 
3550  /* we can directly get the value of the last digit */
3551  *aYOrder = atoi(colname+4);
3552  /* now null out the last digit so that we can get the first one only */
3553  colname[4] = '\0';
3554  *aXOrder = atoi(colname+3);
3555  cpl_free(colname);
3556 
3557  return CPL_ERROR_NONE;
3558 } /* muse_wave_table_get_orders() */
3559 
3560 /*----------------------------------------------------------------------------*/
3581 /*----------------------------------------------------------------------------*/
3582 cpl_polynomial *
3583 muse_wave_table_get_poly_for_slice(const cpl_table *aTable,
3584  const unsigned short aSlice)
3585 {
3586  cpl_ensure(aTable, CPL_ERROR_NULL_INPUT, NULL);
3587  cpl_ensure(aSlice >= 1 && aSlice <= kMuseSlicesPerCCD,
3588  CPL_ERROR_ILLEGAL_INPUT, NULL);
3589  /* search for row containing the requested slice, first to access it *
3590  * in a possibly incomplete table, and second to check its presence! */
3591  int irow, nrow = cpl_table_get_nrow(aTable);
3592  for (irow = 0; irow < nrow; irow++) {
3593  int err;
3594  unsigned short slice = cpl_table_get_int(aTable,
3595  MUSE_WAVECAL_TABLE_COL_SLICE_NO,
3596  irow, &err);
3597  if (slice == aSlice && !err) {
3598  break;
3599  }
3600  } /* for irow */
3601  cpl_ensure(irow < nrow, CPL_ERROR_DATA_NOT_FOUND, NULL);
3602 
3603  cpl_polynomial *pwave = cpl_polynomial_new(2);
3604  char colname[15]; /* "wlc" plus max 2x5 chars for the numbers */
3605  unsigned short wavexorder, waveyorder;
3606  muse_wave_table_get_orders(aTable, &wavexorder, &waveyorder);
3607  unsigned short l;
3608  for (l = 0; l <= wavexorder; l++) {
3609  unsigned short k;
3610  for (k = 0; k <= waveyorder; k++) {
3611  cpl_size pows[2] = { l, k }; /* trick to access the polynomial */
3612  sprintf(colname, MUSE_WAVECAL_TABLE_COL_COEFF, l, k);
3613  int err;
3614  cpl_polynomial_set_coeff(pwave, pows,
3615  cpl_table_get_double(aTable, colname, irow,
3616  &err));
3617  if (err != 0) { /* broken table entry */
3618  cpl_polynomial_delete(pwave);
3619  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_OUTPUT, "Wavelength "
3620  "calibration table broken in slice %hu (row index"
3621  " %d) column %s", aSlice, irow, colname);
3622  return NULL;
3623  } /* if */
3624  } /* for k */
3625  } /* for l */
3626  return pwave;
3627 } /* muse_wave_table_get_poly_for_slice() */
3628 
3629 /*----------------------------------------------------------------------------*/
3652 /*----------------------------------------------------------------------------*/
3653 cpl_image *
3654 muse_wave_map(muse_image *aImage, const cpl_table *aWave,
3655  const cpl_table *aTrace)
3656 {
3657  cpl_ensure(aImage && aWave && aTrace, CPL_ERROR_NULL_INPUT, NULL);
3658  int nx = cpl_image_get_size_x(aImage->data),
3659  ny = cpl_image_get_size_y(aImage->data);
3660  cpl_image *wavemap = cpl_image_new(nx, ny, CPL_TYPE_FLOAT);
3661  cpl_ensure(wavemap, cpl_error_get_code(), NULL);
3662  unsigned char ifu = muse_utils_get_ifu(aImage->header);
3663 
3664  /* get output image buffer as pointer */
3665  float *wdata = cpl_image_get_data_float(wavemap);
3666 
3667  unsigned short wavexorder, waveyorder;
3668  muse_wave_table_get_orders(aWave, &wavexorder, &waveyorder);
3669  cpl_msg_debug(__func__, "Order for trace solution is %d, for wavelength "
3670  "solution %hu/%hu, IFU %hhu", muse_trace_table_get_order(aTrace),
3671  wavexorder, waveyorder, ifu);
3672 
3673  /* loop through all slices */
3674  unsigned short islice;
3675  for (islice = 0; islice < kMuseSlicesPerCCD; islice++) {
3676 #if 0
3677  cpl_msg_debug(__func__, "Starting to process slice %d of IFU %hhu",
3678  (int)islice + 1, ifu);
3679 #endif
3680 
3681  /* fill the wavelength calibration polynomial for this slice */
3682  cpl_polynomial *pwave = muse_wave_table_get_poly_for_slice(aWave, islice + 1);
3683  /* vector for the position within the slice (for evaluation *
3684  * of the wavelength solution in two dimensions */
3685  cpl_vector *pos = cpl_vector_new(2);
3686 
3687  /* get the tracing polynomials for this slice */
3688  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
3689  islice + 1);
3690  if (!ptrace) {
3691  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: tracing polynomials "
3692  "missing!", (int)islice + 1, ifu);
3693  continue;
3694  }
3695 #if 0
3696  printf("polynomials for slice %d:\n", (int)islice + 1);
3697  cpl_polynomial_dump(ptrace[MUSE_TRACE_LEFT], stdout);
3698  cpl_polynomial_dump(ptrace[MUSE_TRACE_RIGHT], stdout);
3699  cpl_polynomial_dump(pwave, stdout);
3700  fflush(stdout);
3701 #endif
3702 
3703  /* within each slice, loop from bottom to top */
3704  int j;
3705  for (j = 1; j <= ny; j++) {
3706  /* determine the slice edges for this vertical *
3707  * position so that we can loop over those pixels */
3708  int ileft = ceil(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT],
3709  j, NULL)),
3710  iright = floor(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT],
3711  j, NULL));
3712  /* try to detect faulty polynomials */
3713  if (ileft < 1 || iright > nx || ileft > iright) {
3714  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: faulty polynomial "
3715  "detected at y=%d", (int)islice + 1, ifu, j);
3716  break; /* skip the rest of this slice */
3717  }
3718  cpl_vector_set(pos, 1, j); /* vertical pos. for wavelength evaluation */
3719 
3720  /* now loop over all pixels of this slice horizontally */
3721  int i;
3722  for (i = ileft; i <= iright; i++) {
3723  cpl_vector_set(pos, 0, i); /* horiz. pos. for wavelength evaluation */
3724 
3725  /* compute the wavelength of this pixel */
3726  wdata[(i-1) + (j-1)*nx] = cpl_polynomial_eval(pwave, pos);
3727  } /* for i (horizontal pixels) */
3728  } /* for j (vertical direction) */
3729 
3730  /* we are now done with this slice, clean up */
3731  muse_trace_polys_delete(ptrace);
3732  cpl_polynomial_delete(pwave);
3733  cpl_vector_delete(pos);
3734  } /* for islice */
3735 
3736  return wavemap;
3737 } /* muse_wave_map() */
3738 
3739 /*----------------------------------------------------------------------------*/
3764 /*----------------------------------------------------------------------------*/
3765 cpl_error_code
3766 muse_wave_plot_residuals(cpl_table *aTable, unsigned char aIFU,
3767  unsigned short aSlice, unsigned int aIter,
3768  cpl_boolean aPlotLambda, cpl_vector *aCuts)
3769 {
3770 #if HAVE_POPEN && HAVE_PCLOSE
3771  cpl_ensure_code(aTable, CPL_ERROR_NULL_INPUT);
3772  cpl_error_code rc = muse_cpltable_check(aTable, muse_wavedebug_def);
3773  cpl_ensure_code(rc == CPL_ERROR_NONE, rc);
3774 
3775  FILE *gp = popen("gnuplot", "w");
3776  if (!gp) {
3777  return CPL_ERROR_ASSIGNING_STREAM;
3778  }
3779 
3780  /* select relevant rows in the table */
3781  cpl_table_unselect_all(aTable); /* start clean */
3782 
3783  int n, nrow = cpl_table_get_nrow(aTable),
3784  error = 0;
3785  if (aSlice == 0) {
3786  printf("Selecting data of all slices");
3787  if (aIFU > 0) {
3788  printf(" of IFU %hhu", aIFU);
3789  }
3790  printf(".\n");
3791 
3792  const int *slice = cpl_table_get_data_int_const(aTable, "slice"),
3793  *iter = cpl_table_get_data_int_const(aTable, "iteration");
3794  if (!aIter) { /* last iteration */
3795  fprintf(stderr, "Selecting data of last iteration of all slices\n");
3796  /* the last row of each slice number is the one with the last iteration */
3797  int sliceno = slice[nrow - 1],
3798  iterlast = iter[nrow - 1];
3799  for (n = nrow - 2; n >= 0; n--) {
3800  if (slice[n] == sliceno && iter[n] != iterlast) {
3801  cpl_table_select_row(aTable, n);
3802  }
3803  if (slice[n] != sliceno) { /* previous slice, reset comparison */
3804  sliceno = slice[n];
3805  iterlast = iter[n];
3806  }
3807  } /* for n (table rows) */
3808  cpl_table_erase_selected(aTable);
3809  /* plot title with some details */
3810  fprintf(gp, "set title \"");
3811  if (aIFU > 0) {
3812  fprintf(gp, "IFU %hhu, ", aIFU);
3813  }
3814  fprintf(gp, "slices %d..%d, iterations %d..%d: 2D polynomial fit "
3815  "residuals (limits %f..%f)\n",
3816  (int)cpl_table_get_column_min(aTable, "slice"),
3817  (int)cpl_table_get_column_max(aTable, "slice"),
3818  (int)cpl_table_get_column_min(aTable, "iteration"),
3819  (int)cpl_table_get_column_max(aTable, "iteration"),
3820  cpl_table_get_column_min(aTable, "rejlimit"),
3821  cpl_table_get_column_max(aTable, "rejlimit"));
3822  } else {
3823  printf("Selecting data of iteration %d.\n", aIter);
3824  for (n = 0; n < nrow; n++) {
3825  if (iter[n] != (int)aIter) {
3826  cpl_table_select_row(aTable, n);
3827  }
3828  } /* for n (table rows) */
3829  cpl_table_erase_selected(aTable);
3830  /* plot title with some details */
3831  fprintf(gp, "set title \"");
3832  if (aIFU > 0) {
3833  fprintf(gp, "IFU %hhu, ", aIFU);
3834  }
3835  fprintf(gp, "slices %d..%d, iteration %d: 2D polynomial fit residuals "
3836  "(limits %f..%f)\n",
3837  (int)cpl_table_get_column_min(aTable, "slice"),
3838  (int)cpl_table_get_column_max(aTable, "slice"), aIter,
3839  cpl_table_get_column_min(aTable, "rejlimit"),
3840  cpl_table_get_column_max(aTable, "rejlimit"));
3841  }
3842  } else {
3843  printf("Selecting data of ");
3844  if (aIFU > 0) {
3845  printf("IFU %hhu ", aIFU);
3846  }
3847  printf("slice %hu.\n", aSlice);
3848  const int *slice = cpl_table_get_data_int_const(aTable, "slice");
3849  for (n = 0; n < nrow; n++) {
3850  if (slice[n] != aSlice) {
3851  cpl_table_select_row(aTable, n);
3852  }
3853  } /* for n (table rows) */
3854  cpl_table_erase_selected(aTable);
3855  nrow = cpl_table_get_nrow(aTable);
3856  cpl_table_unselect_all(aTable);
3857 
3858  const int *iter = cpl_table_get_data_int_const(aTable, "iteration");
3859  if (!aIter) { /* last iteration */
3860  /* table is sorted, so last iteration is that of the last row */
3861  aIter = iter[nrow - 1];
3862  }
3863  printf("Selecting data of iteration %d.\n", aIter);
3864  for (n = 0; n < nrow; n++) {
3865  if (iter[n] != (int)aIter) {
3866  cpl_table_select_row(aTable, n);
3867  }
3868  } /* for n (table rows) */
3869  cpl_table_erase_selected(aTable);
3870 
3871  /* plot title with some details */
3872  fprintf(gp, "set title \"");
3873  if (aIFU > 0) {
3874  fprintf(gp, "IFU %hhu, ", aIFU);
3875  }
3876  fprintf(gp, "slice %hu, iteration %d: 2D polynomial fit residuals "
3877  "(limit=%f)\n", aSlice, aIter,
3878  cpl_table_get_double(aTable, "rejlimit", 0, &error));
3879  } /* single slice */
3880 
3881  /* data access */
3882  nrow = cpl_table_get_nrow(aTable);
3883  cpl_ensure_code(nrow > 0, CPL_ERROR_DATA_NOT_FOUND);
3884  printf("Plotting %d points.\n", nrow);
3885  const int *x = cpl_table_get_data_int_const(aTable, "x");
3886  const float *y = cpl_table_get_data_float_const(aTable, "y"),
3887  *lambda = cpl_table_get_data_float_const(aTable, "lambda");
3888  const double *r = cpl_table_get_data_double_const(aTable, "residual");
3889  /* *rlimit = cpl_table_get_data_double_const(aTable, "rejlimit"); */
3890 
3891  /* determine plotting limits */
3892  int xmin = cpl_table_get_column_min(aTable, "x") - 2,
3893  xmax = cpl_table_get_column_max(aTable, "x") + 2;
3894  float ymin = cpl_table_get_column_min(aTable, "y") - 2,
3895  ymax = cpl_table_get_column_max(aTable, "y") + 2,
3896  lmin = cpl_table_get_column_min(aTable, "lambda") - 2,
3897  lmax = cpl_table_get_column_max(aTable, "lambda") + 2;
3898  double rmin = cpl_table_get_column_min(aTable, "residual"),
3899  rmax = cpl_table_get_column_max(aTable, "residual");
3900  if (aCuts && cpl_vector_get_size(aCuts) == 2) {
3901  rmin = cpl_vector_get(aCuts, 0);
3902  rmax = cpl_vector_get(aCuts, 1);
3903  }
3904 
3905  /* set nice palette */
3906  fprintf(gp, "set palette defined ( 0 \"dark-violet\","
3907  "1 \"dark-blue\", 4 \"green\", 6 \"yellow\", 8 \"orange\","
3908  "9 \"red\", 10 \"dark-red\")\n");
3909  fprintf(gp, "unset key\n");
3910  printf("Setting plotting limits: [%d:%d][%.2f:%.2f][%.4f:%.4f]\n",
3911  xmin, xmax, aPlotLambda ? lmin : ymin, aPlotLambda ? lmax : ymax,
3912  rmin, rmax);
3913  fprintf(gp, "set xrange [%d:%d]\n", xmin, xmax);
3914  if (aPlotLambda) {
3915  fprintf(gp, "set yrange [%f:%f]\n", lmin, lmax);
3916  } else {
3917  fprintf(gp, "set yrange [%f:%f]\n", ymin, ymax);
3918  }
3919  fprintf(gp, "set cbrange [%f:%f]\n", rmin, rmax);
3920  fprintf(gp, "set view map\n");
3921  fprintf(gp, "splot \"-\" w p pal\n");
3922  for (n = 0; n < nrow; n++) {
3923  if (aPlotLambda) {
3924  fprintf(gp, "%d %f %e\n", x[n], lambda[n], r[n]);
3925  } else {
3926  fprintf(gp, "%d %f %e\n", x[n], y[n], r[n]);
3927  }
3928  }
3929  fprintf(gp, "EOF\n");
3930  fflush(gp);
3931  /* request keypress, so that working with the mouse keeps *
3932  * working and gnuplot has enough time to actually draw all *
3933  * the stuff before the files get removed */
3934  printf("Press ENTER to end program and close plot\n");
3935  getchar();
3936  pclose(gp);
3937  return CPL_ERROR_NONE;
3938 #else /* no HAVE_POPEN && HAVE_PCLOSE */
3939  return CPL_ERROR_UNSUPPORTED_MODE;
3940 #endif /* HAVE_POPEN && HAVE_PCLOSE */
3941 } /* muse_wave_plot_residuals() */
3942 
3943 /*----------------------------------------------------------------------------*/
3970 /*----------------------------------------------------------------------------*/
3971 cpl_error_code
3972 muse_wave_plot_column(cpl_table *aCTable, cpl_table *aRTable,
3973  unsigned char aIFU, unsigned short aSlice,
3974  unsigned int aColumn, unsigned int aIter,
3975  cpl_boolean aPlotRes)
3976 {
3977 #if HAVE_POPEN && HAVE_PCLOSE
3978  cpl_ensure_code(aCTable && aRTable, CPL_ERROR_NULL_INPUT);
3979  cpl_error_code rc = muse_cpltable_check(aRTable, muse_wavedebug_def);
3980  cpl_ensure_code(rc == CPL_ERROR_NONE, rc);
3981  /* WAVECAL_TABLE doesn't has a _def structure to check, so test orders */
3982  unsigned short xorder, yorder;
3983  muse_wave_table_get_orders(aCTable, &xorder, &yorder);
3984  cpl_ensure_code(xorder > 0 && yorder > 0, CPL_ERROR_ILLEGAL_INPUT);
3985  cpl_ensure_code(aSlice >= 1 && aSlice <= kMuseSlicesPerCCD,
3986  CPL_ERROR_ACCESS_OUT_OF_RANGE);
3987 
3988  FILE *gp = popen("gnuplot", "w");
3989  if (!gp) {
3990  return CPL_ERROR_ASSIGNING_STREAM;
3991  }
3992 
3993  /* select relevant rows in the table */
3994  cpl_table_unselect_all(aRTable); /* start clean */
3995 
3996  printf("Selecting data of ");
3997  if (aIFU > 0) {
3998  printf("IFU %hhu ", aIFU);
3999  }
4000  printf("slice %hu.\n", aSlice);
4001  const int *slice = cpl_table_get_data_int_const(aRTable, "slice");
4002  int n, nrow = cpl_table_get_nrow(aRTable);
4003  for (n = 0; n < nrow; n++) {
4004  if (slice[n] != aSlice) {
4005  cpl_table_select_row(aRTable, n);
4006  }
4007  } /* for n (table rows) */
4008  cpl_table_erase_selected(aRTable);
4009  nrow = cpl_table_get_nrow(aRTable);
4010  cpl_ensure_code(nrow > 0, CPL_ERROR_DATA_NOT_FOUND);
4011  cpl_table_unselect_all(aRTable);
4012 
4013  const int *iter = cpl_table_get_data_int_const(aRTable, "iteration");
4014  if (!aIter) { /* last iteration */
4015  /* table is sorted, so last iteration is that of the last row */
4016  aIter = iter[nrow - 1];
4017  }
4018  printf("Selecting data of iteration %d.\n", aIter);
4019  for (n = 0; n < nrow; n++) {
4020  if (iter[n] != (int)aIter) {
4021  cpl_table_select_row(aRTable, n);
4022  }
4023  } /* for n (table rows) */
4024  cpl_table_erase_selected(aRTable);
4025  nrow = cpl_table_get_nrow(aRTable);
4026  cpl_ensure_code(nrow > 0, CPL_ERROR_DATA_NOT_FOUND);
4027  cpl_table_unselect_all(aRTable);
4028 
4029  unsigned int col1 = cpl_table_get_column_min(aRTable, "x"),
4030  col2 = cpl_table_get_column_max(aRTable, "x");
4031  if (aColumn > 0) {
4032  col1 = col2 = aColumn;
4033  } else if (aColumn > (unsigned)kMuseOutputXRight) {
4034  /* approximate central column of the slice */
4035  col1 = col2 = (col1 + col2) / 2;
4036  }
4037  printf("Plotting data of columns %u..%u.\n", col1, col2);
4038 
4039  /* determine plotting limits */
4040  float ymin = cpl_table_get_column_min(aRTable, "y") - 10,
4041  ymax = cpl_table_get_column_max(aRTable, "y") + 10,
4042  lmin = cpl_table_get_column_min(aRTable, "lambda") - 10,
4043  lmax = cpl_table_get_column_max(aRTable, "lambda") + 10;
4044  double rmin = cpl_table_get_column_min(aRTable, "residual") * 1.03,
4045  rmax = cpl_table_get_column_max(aRTable, "residual") * 1.03;
4046  /* plot title with some details */
4047  fprintf(gp, "set title \"");
4048  if (aIFU > 0) {
4049  fprintf(gp, "IFU %hhu, ", aIFU);
4050  }
4051  fprintf(gp, "slice %hu, iteration %d, column %u..%u: polynomial and ", aSlice,
4052  aIter, col1, col2);
4053  printf("Setting plotting limits: ");
4054  if (aPlotRes) {
4055  fprintf(gp, "residuals (limit=%f)\"\n",
4056  cpl_table_get_double(aRTable, "rejlimit", 0, NULL));
4057  printf("[%.2f:%.2f][%.4f:%.4f]\n", lmin, lmax, rmin, rmax);
4058  fprintf(gp, "set xrange [%f:%f]\n", lmin, lmax);
4059  fprintf(gp, "set yrange [%f:%f]\n", rmin, rmax);
4060  fprintf(gp, "set xlabel \"Wavelength [Angstrom]\"\n");
4061  fprintf(gp, "set ylabel \"Residuals [Angstrom]\"\n");
4062  } else {
4063  fprintf(gp, "arc line positions\"\n");
4064  printf("[%.2f:%.2f][%.2f:%.2f]\n", ymin, ymax, lmin, lmax);
4065  fprintf(gp, "set xrange [%g:%g]\n", ymin, ymax);
4066  fprintf(gp, "set yrange [%f:%f]\n", lmin, lmax);
4067  fprintf(gp, "set xlabel \"y-position [pix]\"\n");
4068  fprintf(gp, "set ylabel \"Wavelength [Angstrom]\"\n");
4069  }
4070  fprintf(gp, "set key outside below\n");
4071  fprintf(gp, "set samples 1000\n"); /* more samples so that one can zoom in */
4072 
4073  /* create gnuplot polynomial from table coefficients */
4074  fprintf(gp, "p(x,y) = 0 ");
4075  if (!aPlotRes) { /* create full polynomial */
4076  unsigned short i;
4077  for (i = 0; i <= xorder; i++) {
4078  unsigned short j;
4079  for (j = 0; j <= yorder; j++) {
4080  char *coeff = cpl_sprintf(MUSE_WAVECAL_TABLE_COL_COEFF, i, j);
4081  double cvalue = cpl_table_get(aCTable, coeff, aSlice - 1, NULL);
4082  cpl_free(coeff);
4083  fprintf(gp, " + (%g) * x**(%hu) * y**(%hu)", cvalue, i, j);
4084  } /* for j (y orders) */
4085  } /* for i (x orders) */
4086  }
4087  fprintf(gp, "\n");
4088 
4089  const int *x = cpl_table_get_data_int_const(aRTable, "x");
4090  const float *y = cpl_table_get_data_float_const(aRTable, "y"),
4091  *lambda = cpl_table_get_data_float_const(aRTable, "lambda");
4092  const double *r = cpl_table_get_data_double_const(aRTable, "residual");
4093 
4094  /* distribute columns over 256 color values */
4095  double dcol = (col2 - col1) / 255.;
4096  if (dcol == 0.) { /* take care not to produce NANs below when dividing */
4097  dcol = 1.;
4098  }
4099  /* construct the plot command as a single long line */
4100  fprintf(gp, "plot ");
4101  if (aPlotRes) {
4102  fprintf(gp, "0 t \"\", "); /* plot line to represent solution */
4103  }
4104  unsigned int ncol, npoints = 0;
4105  for (ncol = col1; ncol <= col2; ncol++) {
4106  /* plot polynomial and values for the specified CCD column */
4107  int red = (ncol - col1) / dcol, /* red */
4108  grn = (col2 - ncol) / dcol, /* green */
4109  blu = 0; /* blue */
4110  if (aPlotRes) {
4111  fprintf(gp, "\"-\" u 2:3 t \"col %u\" w p ps 0.8 lt rgb \"#%02x%02x%02x\"",
4112  ncol, red, grn, blu);
4113  } else {
4114  /* create data values from polynomial and residuals for given CCD column */
4115  fprintf(gp, "p(%u, x) t \"\" w l lw 0.7 lt rgb \"#%02x%02x%02x\", "
4116  "\"-\" u 1:(p(%u,$1)+$3) t \"col %u\" w p ps 0.8 lt rgb \"#%02x%02x%02x\"",
4117  ncol, red, grn, blu, ncol, ncol, red, grn, blu);
4118  }
4119  if (ncol == col2) {
4120  fprintf(gp, "\n"); /* end the plot command */
4121  } else {
4122  fprintf(gp, ", "); /* plot command of next column to come */
4123  }
4124  } /* for ncol (relevant columns) */
4125  for (ncol = col1; ncol <= col2; ncol++) {
4126  for (n = 0; n < nrow; n++) {
4127  if (x[n] == (int)ncol) {
4128  fprintf(gp, "%f %f %g\n", y[n], lambda[n], r[n]);
4129  npoints++;
4130  }
4131  }
4132  fprintf(gp, "EOF\n");
4133  } /* for ncol (relevant columns) */
4134  printf("Plotted %u points.\n", npoints);
4135  fflush(gp);
4136  /* request keypress, so that working with the mouse keeps *
4137  * working and gnuplot has enough time to actually draw all *
4138  * the stuff before the files get removed */
4139  printf("Press ENTER to end program and close plot\n");
4140  getchar();
4141  pclose(gp);
4142  return CPL_ERROR_NONE;
4143 #else /* no HAVE_POPEN && HAVE_PCLOSE */
4144  return CPL_ERROR_UNSUPPORTED_MODE;
4145 #endif /* HAVE_POPEN && HAVE_PCLOSE */
4146 } /* muse_wave_plot_column() */
4147 
cpl_table * muse_wave_lines_search(muse_image *aColumnImage, double aSigma, const unsigned short aSlice, const unsigned char aIFU)
Search and store emission lines in a column of an arc frame.
cpl_table * muse_wave_calib_lampwise(muse_imagelist *aImages, cpl_table *aTrace, cpl_table *aLinelist, muse_wave_params *aParams)
Find wavelength calibration solution using a list of arc images with different lamps.
cpl_table * muse_wave_calib(muse_image *aImage, cpl_table *aTrace, cpl_table *aLinelist, muse_wave_params *aParams)
Find wavelength calibration solution on an arc frame.
cpl_polynomial ** muse_trace_table_get_polys_for_slice(const cpl_table *aTable, const unsigned short aSlice)
construct polynomial from the trace table entry for the given slice
cpl_error_code muse_wave_table_get_orders(const cpl_table *aWave, unsigned short *aXOrder, unsigned short *aYOrder)
Determine the x- and y-order of the polynomial stored in a wavelength calibration table...
muse_wave_params * muse_wave_params_new(cpl_propertylist *aHeader)
Allocate a wavelength parameters structure and fill it with defaults.
Structure definition for a collection of muse_images.
int muse_trace_table_get_order(const cpl_table *aTable)
determine order of tracing polynomial from table
cpl_error_code muse_wave_plot_residuals(cpl_table *aTable, unsigned char aIFU, unsigned short aSlice, unsigned int aIter, cpl_boolean aPlotLambda, cpl_vector *aCuts)
Fancy plotting of wavelength calibration residuals (color coded over x/y-position) using gnuplot...
void muse_image_delete(muse_image *aImage)
Deallocate memory associated to a muse_image object.
Definition: muse_image.c:85
muse_wave_weighting_type fitweighting
cpl_polynomial * muse_wave_table_get_poly_for_slice(const cpl_table *aTable, const unsigned short aSlice)
Construct polynomial from the wavelength calibration table entry for the given slice.
const char * muse_pfits_get_insmode(const cpl_propertylist *aHeaders)
find out the observation mode
Definition: muse_pfits.c:1383
cpl_vector * muse_wave_lines_get_for_lamp(cpl_table *aTable, const char *aLamp, int aGoodnessLimit, double aFluxLimit)
Load wavelengths for a given lamp from a linelist table into a vector.
cpl_error_code muse_wave_lines_identify(cpl_table *aLines, cpl_vector *aLambdas, const muse_wave_params *aParams)
Identify the wavelength of arc detected lines using pattern matching.
void muse_wave_params_delete(muse_wave_params *aParams)
Deallocate memory associated to a wavelength parameters structure.
unsigned char muse_utils_get_ifu(const cpl_propertylist *aHeaders)
Find out the IFU/channel from which this header originated.
Definition: muse_utils.c:98
cpl_image * data
the data extension
Definition: muse_image.h:46
muse_ins_mode mode
cpl_boolean rflag
cpl_error_code muse_wave_poly_fit(cpl_matrix *aXYPos, cpl_vector *aLambdas, cpl_vector *aDLambdas, cpl_polynomial **aPoly, double *aMSE, muse_wave_params *aParams, const unsigned short aSlice)
Compute the wavelength solution from the sample positions and the respective wavelengths.
cpl_table * muse_wave_table_create(const unsigned short aNSlices, const unsigned short aXOrder, const unsigned short aYOrder)
Create the table to save te wave wavelength calibration coefficients.
muse_image * muse_combine_median_create(muse_imagelist *aImages)
Median combine a list of input images.
Definition: muse_combine.c:316
cpl_image * stat
the statistics extension
Definition: muse_image.h:64
void muse_imagelist_delete(muse_imagelist *aList)
Free the memory of the MUSE image list.
cpl_vector * muse_wave_lines_get(cpl_table *aTable, int aGoodnessLimit, double aFluxLimit)
Load usable wavelengths from a linelist table into a vector.
cpl_table * residuals
const char * muse_pfits_get_bunit(const cpl_propertylist *aHeaders)
find out the unit string
Definition: muse_pfits.c:223
cpl_boolean muse_wave_lines_check(muse_table *aTable)
Check that a LINE_CATALOG has the expected format.
Structure definition of MUSE three extension FITS file.
Definition: muse_image.h:40
int muse_pfits_get_lampnum(const cpl_propertylist *aHeaders)
query the number of lamps installed
Definition: muse_pfits.c:1483
cpl_table * muse_wave_line_handle_multiplet(muse_image *aImage, cpl_table *aLinelist, unsigned int aIdx, cpl_polynomial *aPoly, cpl_polynomial **aTrace, const muse_wave_params *aParams, const unsigned short aSlice, int aDebug)
Handle fitting of all multiplets across the columns a given slice.
cpl_propertylist * header
the FITS header
Definition: muse_image.h:72
cpl_error_code muse_cpltable_check(const cpl_table *aTable, const muse_cpltable_def *aDef)
Check whether the table contains the fields of the definition.
unsigned int muse_imagelist_get_size(muse_imagelist *aList)
Return the number of stored images.
void muse_trace_polys_delete(cpl_polynomial *aPolys[])
Delete the multi-polynomial array created in relation to tracing.
cpl_image * dq
the data quality extension
Definition: muse_image.h:56
cpl_table * muse_cpltable_new(const muse_cpltable_def *aDef, cpl_size aLength)
Create an empty table according to the specified definition.
const char * muse_wave_lines_get_lampname(cpl_table *aTable, const int aIdx)
Associate the ion listed in a linelist table row to a lamp name.
cpl_boolean muse_wave_lines_covered_by_data(double aLambda, muse_ins_mode aMode)
Check, if a given wavelength is covered by a given instrument mode.
muse_image * muse_imagelist_get(muse_imagelist *aList, unsigned int aIdx)
Get the muse_image of given list index.
cpl_error_code muse_cplvector_erase_element(cpl_vector *aVector, int aElement)
delete the given element from the input vector
Structure containing wavelength calibration parameters.
cpl_polynomial * muse_utils_iterate_fit_polynomial(cpl_matrix *aPos, cpl_vector *aVal, cpl_vector *aErr, cpl_table *aExtra, const unsigned int aOrder, const double aRSigma, double *aMSE, double *aChiSq)
Iterate a polynomial fit.
Definition: muse_utils.c:2234
unsigned short xorder
cpl_table * table
The table.
Definition: muse_table.h:49
cpl_error_code muse_wave_table_add_poly(cpl_table *aTable, cpl_polynomial *aPoly, double aMSE, unsigned short aXOrder, unsigned short aYOrder, const unsigned short aRow)
Save the given polynomials to the wavelength calibration table.
cpl_image * muse_wave_map(muse_image *aImage, const cpl_table *aWave, const cpl_table *aTrace)
Write out a wavelength map for visual checks.
const muse_cpltable_def muse_line_catalog_def[]
Structure to store a table together with a property list.
Definition: muse_table.h:43
const muse_cpltable_def muse_wavedebug_def[]
MUSE wavelength calibration residuals table definition.
cpl_error_code muse_utils_fit_multigauss_1d(const cpl_vector *aX, const cpl_bivector *aY, cpl_vector *aCenter, double *aSigma, cpl_vector *aFlux, cpl_vector *aPoly, double *aMSE, double *aRedChisq, cpl_matrix **aCovariance)
Carry out a multi-Gaussian fit of data in a vector.
Definition: muse_utils.c:1558
char * muse_utils_header_get_lamp_names(cpl_propertylist *aHeader, char aSep)
Concatenate names of all active calibration lamps.
Definition: muse_utils.c:989
cpl_propertylist * header
the header
Definition: muse_table.h:56
cpl_array * muse_utils_header_get_lamp_numbers(cpl_propertylist *aHeader)
List numbers of all active calibration lamps.
Definition: muse_utils.c:1056
cpl_error_code muse_wave_plot_column(cpl_table *aCTable, cpl_table *aRTable, unsigned char aIFU, unsigned short aSlice, unsigned int aColumn, unsigned int aIter, cpl_boolean aPlotRes)
Plot wavelength calibration polynomial and data or residuals using gnuplot.
cpl_error_code muse_image_save(muse_image *aImage, const char *aFilename)
Save the three image extensions and the FITS headers of a MUSE image to a file.
Definition: muse_image.c:405
cpl_error_code muse_wave_line_fit_iterate(cpl_table *aFitTable, double aLambda, const muse_wave_params *aParams)
Use a low-order polynomial to find and discard bad values for line centroid fits of single arc line a...
muse_imagelist * muse_imagelist_new(void)
Create a new (empty) MUSE image list.
cpl_error_code muse_wave_line_fit_single(muse_image *aImage, int aX, double aY, int aHalfWidth, double aSigma, cpl_table *aFitTable, int aRowsNeeded)
Fit a Gaussian to a single emission line in an arc frame and do simple error handling.
unsigned short yorder
cpl_size muse_cplvector_count_unique(const cpl_vector *aVector)
Count the number of unique entries in a given vector.
Definition of a cpl table structure.
muse_image * muse_image_new(void)
Allocate memory for a new muse_image object.
Definition: muse_image.c:66
cpl_error_code muse_wave_line_fit_multiple(muse_image *aImage, int aX, cpl_bivector *aPeaks, cpl_vector *aLambdas, int aHalfWidth, double aSigma, cpl_table *aFitTable, int aRowsNeeded)
Fit a multi-Gaussian to a multiplet of arc emission lines and do simple error handling.
cpl_error_code muse_imagelist_set(muse_imagelist *aList, muse_image *aImage, unsigned int aIdx)
Set the muse_image of given list index.
const muse_cpltable_def muse_wavelines_def[]
MUSE wavelength calibration arc line fit properties table definition.
cpl_table * muse_wave_line_handle_singlet(muse_image *aImage, cpl_table *aLinelist, unsigned int aIdx, cpl_polynomial *aPoly, cpl_polynomial **aTrace, const muse_wave_params *aParams, const unsigned short aSlice, int aDebug)
Handle fitting of all single lines across the columns a given slice.
muse_ins_mode muse_pfits_get_mode(const cpl_propertylist *aHeaders)
find out the observation mode
Definition: muse_pfits.c:1352