MUSE Pipeline Reference Manual  2.1.1
muse_sky_master.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  * This file is part of the MUSE Instrument Pipeline
5  * Copyright (C) 2008-2014 European Southern Observatory
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  */
21 
22 #ifdef HAVE_CONFIG_H
23 #include <config.h>
24 #endif
25 
26 #include <math.h>
27 #include <string.h>
28 
29 #include "muse_sky.h"
30 #include "muse_instrument.h"
31 #include "muse_lsf.h"
32 #include "muse_optimize.h"
33 #include "muse_utils.h"
34 #include "muse_data_format_z.h"
35 
39 /*----------------------------------------------------------------------------*/
49 /*----------------------------------------------------------------------------*/
50 static cpl_array *
51 muse_sky_lines_firstguess(int ngroups) {
52  cpl_array *pars = cpl_array_new(ngroups+2, CPL_TYPE_DOUBLE);
53 
54  // Line strengths
55  cpl_size i;
56  for (i = 0; i < ngroups; i++) {
57  cpl_array_set(pars, i, 1e-1);
58  }
59  cpl_array_set(pars, ngroups, 0.0);
60  cpl_array_set(pars, ngroups+1, 0.0);
61  return pars;
62 }
63 
64 typedef struct {
65  const cpl_array *lambda; // wavelength
66  const cpl_array *values; // data values
67  const cpl_array *weights; // data weights
68  const cpl_table *lines; // constant sky physics data
69  const cpl_size ngroups; // number of groups in the data
70  const cpl_image *lsfImage;
71  const muse_wcs *wcs;
72 } muse_master_fit_struct;
73 
74 /*----------------------------------------------------------------------------*/
82 /*----------------------------------------------------------------------------*/
83 static cpl_array *
84 simulate_master_sky_parameters(muse_master_fit_struct *aFitData,
85  const cpl_array *aPar)
86 {
87  cpl_table *lines = cpl_table_duplicate(aFitData->lines);
88  int res = 0;
89  double fac = cpl_array_get(aPar, cpl_array_get_size(aPar)-2, &res);
90  double offset = cpl_array_get(aPar, cpl_array_get_size(aPar)-1, &res);
91  cpl_table_multiply_scalar(lines, "lambda", 1+fac);
92  cpl_table_add_scalar(lines, "lambda", offset-7000 * fac);
93  cpl_array *line_strengths = cpl_array_duplicate(aPar);
94  cpl_array_multiply(line_strengths, aPar);
95  muse_sky_lines_apply_strength(lines, line_strengths);
96  cpl_array_delete(line_strengths);
97 
98  double maxflux = cpl_table_get_column_max(lines, "flux");
99  muse_sky_lines_cut(lines, 1e-4 * maxflux);
100 
101  cpl_array *simulated
102  = muse_sky_lines_spectrum(aFitData->lambda, lines,
103  aFitData->lsfImage, aFitData->wcs);
104  cpl_table_delete(lines);
105 
106  return simulated;
107 }
108 
109 /*----------------------------------------------------------------------------*/
118 /*----------------------------------------------------------------------------*/
119 static cpl_error_code
120 muse_sky_master_eval(void *aData, cpl_array *aPar, cpl_array *aRetval)
121 {
122  cpl_size size = cpl_array_get_size(aRetval);
123  muse_master_fit_struct *fitData = aData;
124  cpl_array *simulated = simulate_master_sky_parameters(fitData, aPar);
125  cpl_array_subtract(simulated, fitData->values);
126  cpl_array *dsimulated = muse_cplarray_diff(simulated, 1);
127 
128  cpl_array_multiply(dsimulated, fitData->weights);
129 
130  cpl_array_fill_window_double(aRetval, 0, size, 0.0);
131  memcpy(cpl_array_get_data_double(aRetval),
132  cpl_array_get_data_double_const(dsimulated),
133  size * sizeof(double));
134  cpl_array_delete(simulated);
135  cpl_array_delete(dsimulated);
136 
137  return CPL_ERROR_NONE;
138 }
139 
140 /*----------------------------------------------------------------------------*/
153 /*----------------------------------------------------------------------------*/
154 static cpl_error_code
155 muse_sky_lines_correct_firstguess(muse_master_fit_struct *aFitData,
156  cpl_array *aPar)
157 {
158  // evaluate the first guess
159  cpl_array *simulated = simulate_master_sky_parameters(aFitData, aPar);
160 
161  cpl_table *lines = cpl_table_duplicate(aFitData->lines);
162  cpl_array *line_strengths = cpl_array_duplicate(aPar);
163  cpl_array_multiply(line_strengths, aPar);
164  muse_sky_lines_apply_strength(lines, line_strengths);
165  cpl_array_delete(line_strengths);
166  cpl_size i_group;
167  for (i_group = 0; i_group < aFitData->ngroups; i_group++) {
168  // take the strongest line of each group
169  cpl_table_unselect_all(lines);
170  cpl_table_or_selected_int(lines, "group", CPL_EQUAL_TO, i_group);
171  cpl_table *gtable = cpl_table_extract_selected(lines);
172  if (cpl_table_get_nrow(gtable) == 0) {
173  cpl_table_delete(gtable);
174  continue;
175  }
176  cpl_size row;
177  cpl_table_get_column_maxpos(gtable, "flux", &row);
178  double wavelength = cpl_table_get_double(gtable, "lambda", row, NULL);
179 
180  // divide measured data and first guess result
181  cpl_size i_lbda1 = muse_cplarray_find_sorted(aFitData->lambda,
182  wavelength - 3.0);
183  cpl_size i_lbda2 = muse_cplarray_find_sorted(aFitData->lambda,
184  wavelength + 3.0);
185  double y_data = 0;
186  double y_sim = 0;
187  double offset = cpl_array_get(aFitData->values, i_lbda1, NULL);
188  offset += cpl_array_get(aFitData->values, i_lbda2, NULL);
189  offset /= 2;
190  cpl_size i_lbda;
191  for (i_lbda = i_lbda1; i_lbda <= i_lbda2; i_lbda++) {
192  double wy_data = cpl_array_get(aFitData->values, i_lbda, NULL) - offset;
193  if (wy_data < 0) wy_data = 0;
194  double wy_sim = cpl_array_get(simulated, i_lbda, NULL);
195  y_data += wy_data;
196  y_sim += wy_sim;
197  }
198 
199  // take this as correction factor
200  if (y_sim > 0) {
201  double y = cpl_array_get(aPar, i_group, NULL);
202  cpl_array_set(aPar, i_group, y*sqrt(y_data/y_sim));
203  }
204  cpl_table_delete(gtable);
205  }
206  cpl_table_delete(lines);
207  cpl_array_delete(simulated);
208 
209  return CPL_ERROR_NONE;
210 }
211 
212 /*----------------------------------------------------------------------------*/
225 /*----------------------------------------------------------------------------*/
226 cpl_error_code
227 muse_sky_lines_fit(cpl_table *aSpectrum, cpl_table *aLines,
228  cpl_image *aLsfImage, muse_wcs *aWCS)
229 {
230  cpl_ensure_code(aSpectrum, CPL_ERROR_NULL_INPUT);
231  cpl_ensure_code(aLines, CPL_ERROR_NULL_INPUT);
232  cpl_size nRows = cpl_table_get_nrow(aSpectrum);
233  cpl_ensure_code(nRows > 0, CPL_ERROR_DATA_NOT_FOUND);
234 
235  cpl_array *lambda = muse_cpltable_extract_column(aSpectrum, "lambda");
236  cpl_array *data = muse_cpltable_extract_column(aSpectrum, "data");
237  cpl_array *stat = muse_cpltable_extract_column(aSpectrum, "stat");
238 
239  cpl_array *weights = cpl_array_extract(stat, 0, nRows - 1);
240  cpl_array *w2 = cpl_array_extract(stat, 1, nRows);
241  cpl_array_add(weights, w2);
242  cpl_array_delete(w2);
243  cpl_array_power(weights, -0.5);
244  // cpl_array_fill_window(weights, 0, cpl_array_get_size(weights)-1, 1.0);
245 
246  muse_master_fit_struct fit_data = {
247  lambda,
248  data,
249  weights,
250  aLines,
251  cpl_table_get_column_max(aLines, "group") + 1,
252  aLsfImage,
253  aWCS
254  };
255 
256  cpl_array *pars = muse_sky_lines_firstguess(fit_data.ngroups);
257  muse_sky_lines_correct_firstguess(&fit_data, pars);
258 
259  // do the fit, ignoring possible errors
260  int debug = getenv("MUSE_DEBUG_LSF_FIT")
261  && atoi(getenv("MUSE_DEBUG_LSF_FIT")) > 0;
263  -1, -1, -1, -1, // default ftol, xtol, gtol, maxiter
264  debug
265  };
266  /* this potentially takes a long time, better output something */
267  cpl_msg_info(__func__, "Starting sky line fit");
268  cpl_error_code r = muse_cpl_optimize_lvmq(&fit_data, pars, nRows-1,
269  muse_sky_master_eval, &ctrl);
270  if (r != CPL_ERROR_NONE) {
271  cpl_msg_error(__func__, "Sky line fit failed with error code %i: %s",
272  r, cpl_error_get_message());
273  } else {
274  int res = 0;
275  double fac = cpl_array_get(pars, cpl_array_get_size(pars)-2, &res);
276  double offset = cpl_array_get(pars, cpl_array_get_size(pars)-1, &res);
277  cpl_table_multiply_scalar(aLines, "lambda", 1+fac);
278  cpl_table_add_scalar(aLines, "lambda", offset-7000 * fac);
279  double l_min = cpl_table_get_column_min(aLines, "lambda");
280  double l_max = cpl_table_get_column_max(aLines, "lambda");
281  cpl_msg_info(__func__, "Sky line fit finished successfully. "
282  "Offset %.3f A (at %.0f A) ... %.3f A (at %.0f A)",
283  offset + (l_min - 7000) *fac, l_min,
284  offset + (l_max - 7000) * fac, l_max);
285  }
286 
287  cpl_array_multiply(pars, pars);
288  muse_sky_lines_apply_strength(aLines, pars);
289  cpl_array_delete(pars);
290  // sort by flux
291  cpl_propertylist *order = cpl_propertylist_new();
292  cpl_propertylist_append_bool(order, "flux", TRUE);
293  cpl_table_sort(aLines, order);
294  cpl_propertylist_delete(order);
295  cpl_array_unwrap(lambda);
296  cpl_array_unwrap(data);
297  cpl_array_unwrap(stat);
298  cpl_array_delete(weights);
299 
300  return CPL_ERROR_NONE;
301 }
302 
303 /*----------------------------------------------------------------------------*/
317 /*----------------------------------------------------------------------------*/
318 cpl_table *
319 muse_sky_continuum_create(cpl_table *aSpectrum, cpl_table *aLines,
320  cpl_image *aLsfImage, muse_wcs *aLsfWCS,
321  double aBinWidth)
322 {
323  cpl_ensure(aSpectrum, CPL_ERROR_NULL_INPUT, NULL);
324  cpl_ensure(aLines, CPL_ERROR_NULL_INPUT, NULL);
325  cpl_ensure(aLsfImage, CPL_ERROR_NULL_INPUT, NULL);
326  cpl_ensure(aLsfImage, CPL_ERROR_NULL_INPUT, NULL);
327 
328  cpl_array *lambda = muse_cpltable_extract_column(aSpectrum, "lambda");
329  cpl_array *flux = muse_cpltable_extract_column(aSpectrum, "data");
330  cpl_array *simulated = muse_sky_lines_spectrum(lambda, aLines, aLsfImage, aLsfWCS);
331  cpl_array_subtract(simulated, flux);
332  cpl_array_multiply_scalar(simulated, -1.);
333 
334  double l_min = cpl_array_get_min(lambda);
335  double l_max = cpl_array_get_max(lambda);
336  cpl_size n_new = (l_max - l_min) / aBinWidth;
337  cpl_table *continuum = muse_cpltable_new(muse_fluxspectrum_def, n_new);
338  cpl_table_fill_column_window(continuum, "flux", 0, n_new, 0.0);
339  cpl_array *lambda_new = muse_cpltable_extract_column(continuum, "lambda");
340 
341  cpl_size i;
342  for (i = 0; i < n_new; i++) {
343  cpl_table_set(continuum, "lambda", i, l_min + i * aBinWidth);
344  }
345  cpl_array *flux_new = muse_cplarray_interpolate_linear(lambda_new,
346  lambda, simulated);
347  // Copy the values to the table column
348  memcpy(cpl_table_get_data_double(continuum, "flux"),
349  cpl_array_get_data_double(flux_new),
350  n_new * sizeof(double));
351  cpl_array_delete(simulated);
352  cpl_array_unwrap(lambda);
353  cpl_array_unwrap(flux);
354  cpl_array_unwrap(lambda_new);
355  cpl_array_delete(flux_new);
356  return continuum;
357 }
358 
359 /*----------------------------------------------------------------------------*/
365 /*----------------------------------------------------------------------------*/
366 cpl_table *
368 {
369  cpl_ensure(aProcessing, CPL_ERROR_NULL_INPUT, NULL);
370 
371  /* search for frames with the tag for sky continuum set */
372  cpl_frameset *frames_c = muse_frameset_find(aProcessing->inframes,
373  MUSE_TAG_SKY_CONT, 0, CPL_FALSE);
374  if (frames_c == NULL || cpl_frameset_get_size(frames_c) < 1) {
375  cpl_frameset_delete(frames_c);
376  cpl_msg_debug(__func__, "No sky continuum found in input frameset!");
377  return NULL;
378  }
379  cpl_frame *frame_c = cpl_frameset_get_position(frames_c, 0);
380  const char *fn = cpl_frame_get_filename(frame_c);
381  cpl_table *continuum = muse_cpltable_load(fn, "CONTINUUM",
383 
384  if (continuum == NULL) {
385  cpl_msg_warning(__func__, "Could not load sky continuum from \"%s\"", fn);
386  cpl_frameset_delete(frames_c);
387  return NULL;
388  }
389 
390  cpl_msg_info(__func__, "Loaded sky continuum from \"%s\"", fn);
391  muse_processing_append_used(aProcessing, frame_c, CPL_FRAME_GROUP_CALIB, 1);
392  cpl_frameset_delete(frames_c);
393  return continuum;
394 } /* muse_sky_continuum_load() */
395 
cpl_error_code muse_sky_lines_fit(cpl_table *aSpectrum, cpl_table *aLines, cpl_image *aLsfImage, muse_wcs *aWCS)
Fit all entries of the pixel table to the master sky.
cpl_table * muse_sky_continuum_create(cpl_table *aSpectrum, cpl_table *aLines, cpl_image *aLsfImage, muse_wcs *aLsfWCS, double aBinWidth)
Create a continuum spectrum.
cpl_error_code muse_sky_lines_apply_strength(cpl_table *, const cpl_array *)
Apply the line strengths to the lines.
A structure containing a spatial two-axis WCS.
Definition: muse_wcs.h:105
cpl_error_code muse_cpl_optimize_lvmq(void *aData, cpl_array *aPar, int aSize, muse_cpl_evaluate_func *aFunction, muse_cpl_optimize_control_t *aCtrl)
Minimize a function with the Levenberg-Marquardt algorithm.
cpl_table * muse_cpltable_load(const char *aFile, const char *aExtension, const muse_cpltable_def aDefinition[])
Load a table from disk (and check against definition).
cpl_table * muse_cpltable_new(const muse_cpltable_def *aDef, cpl_size aLength)
Create an empty table according to the specified definition.
cpl_table * muse_sky_continuum_load(muse_processing *aProcessing)
Load the sky continuum.
cpl_array * muse_cpltable_extract_column(cpl_table *aTable, const char *aColumn)
Create an array from a section of a column.
cpl_array * muse_cplarray_interpolate_linear(const cpl_array *aTargetAbscissa, const cpl_array *aSourceAbscissa, const cpl_array *aSourceOrdinate)
Linear interpolation of a 1d array.
Optimization control parameters.
Definition: muse_optimize.h:32
const muse_cpltable_def muse_fluxspectrum_def[]
Definition of the flux spectrum table structure.
void muse_processing_append_used(muse_processing *aProcessing, cpl_frame *aFrame, cpl_frame_group aGroup, int aDuplicate)
Add a frame to the set of used frames.
cpl_array * muse_cplarray_diff(const cpl_array *aArray, int aOffset)
Build the difference of any element and one of the next elements.
cpl_array * muse_sky_lines_spectrum(const cpl_array *, cpl_table *, const cpl_image *, const muse_wcs *)
Create spectrum for a single slice.
cpl_error_code muse_sky_lines_cut(cpl_table *, double)
Remove all lines below a certain flux limit.
cpl_frameset * inframes
cpl_frameset * muse_frameset_find(const cpl_frameset *aFrames, const char *aTag, unsigned char aIFU, cpl_boolean aInvert)
return frameset containing data from an IFU/channel with a certain tag
Definition: muse_utils.c:160
cpl_size muse_cplarray_find_sorted(const cpl_array *aArray, double aValue)
Find a row in an array.