MUSE Pipeline Reference Manual  2.1.1
muse_create_sky.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  * This file is part of the MUSE Instrument Pipeline
5  * Copyright (C) 2005-2014 European Southern Observatory
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  */
21 
22 #ifdef HAVE_CONFIG_H
23 #include <config.h>
24 #endif
25 
26 /*---------------------------------------------------------------------------*
27  * Includes *
28  *---------------------------------------------------------------------------*/
29 #include <string.h>
30 #include <math.h>
31 
32 #include <muse.h>
33 #include "muse_create_sky_z.h"
34 
35 /*---------------------------------------------------------------------------*
36  * Functions code *
37  *---------------------------------------------------------------------------*/
38 static muse_pixtable *
39 muse_create_sky_load_pixtable(muse_processing *aProcessing,
40  muse_create_sky_params_t *aParams)
41 {
42  muse_pixtable *pixtable = NULL;
43 
44  /* sort input pixel tables into different exposures */
45  cpl_table *exposures = muse_processing_sort_exposures(aProcessing);
46  if (!exposures) {
47  cpl_msg_error(__func__, "no science exposures found in input");
48  return NULL;
49  }
50  int nexposures = cpl_table_get_nrow(exposures);
51  if (nexposures != 1) {
52  cpl_msg_error(__func__, "More than one exposure (%i) in sky creation",
53  nexposures);
54  }
55 
56  /* now process all the pixel tables, do it separately for each exposure */
57  int i;
58  for (i = 0; i < nexposures; i++) {
59  cpl_table *thisexp = cpl_table_extract(exposures, i, 1);
61  aParams->lambdamin,
62  aParams->lambdamax);
63  cpl_table_delete(thisexp);
64  if (p == NULL) {
65  muse_pixtable_delete(pixtable);
66  pixtable = NULL;
67  break;
68  }
69  /* erase pre-existing QC parameters */
70  cpl_propertylist_erase_regexp(p->header, "ESO QC ", 0);
71  if (pixtable == NULL) {
72  pixtable = p;
73  } else {
74  cpl_table_insert(pixtable->table, p->table,
75  cpl_table_get_nrow(pixtable->table));
77  }
78  }
79  cpl_table_delete(exposures);
80 
81  if ((pixtable != NULL) && (muse_pixtable_is_skysub(pixtable) == CPL_TRUE)) {
82  cpl_msg_error(__func__, "Pixel table already sky subtracted");
83  muse_pixtable_delete(pixtable);
84  pixtable = NULL;
85  }
86 
87  muse_table *response = muse_processing_load_table(aProcessing,
88  MUSE_TAG_STD_RESPONSE, 0),
89  *telluric = muse_processing_load_table(aProcessing,
90  MUSE_TAG_STD_TELLURIC, 0);
91  cpl_table *extinction = muse_processing_load_ctable(aProcessing,
92  MUSE_TAG_EXTINCT_TABLE, 0);
93 
94  if ((pixtable != NULL) && (response != NULL)) {
95  if (muse_pixtable_is_fluxcal(pixtable) == CPL_TRUE) {
96  cpl_msg_error(__func__,
97  "Pixel table already flux calibrated. Dont specify %s, %s, %s",
98  MUSE_TAG_STD_RESPONSE, MUSE_TAG_EXTINCT_TABLE,
99  MUSE_TAG_STD_TELLURIC);
100  muse_pixtable_delete(pixtable);
101  pixtable = NULL;
102  } else {
103  /* See if we need to revert the flat-field spectrum correction, *
104  * to match pixel table and response curve, before applying it. */
105  muse_postproc_revert_ffspec_maybe(pixtable, response);
106  cpl_error_code rc = muse_flux_calibrate(pixtable, response, extinction,
107  telluric);
108  if (rc != CPL_ERROR_NONE) {
109  cpl_msg_error(__func__, "while muse_flux_calibrate");
110  muse_pixtable_delete(pixtable);
111  pixtable = NULL;
112  }
113  }
114  }
115 
116  muse_table_delete(response);
117  muse_table_delete(telluric);
118  cpl_table_delete(extinction);
119 
120  if ((pixtable != NULL) && (muse_pixtable_is_fluxcal(pixtable) != CPL_TRUE)) {
121  cpl_msg_error(__func__, "Pixel table not flux calibrated");
122  muse_pixtable_delete(pixtable);
123  pixtable = NULL;
124  }
125 
126  if (pixtable != NULL) {
127  cpl_table_and_selected_int(pixtable->table, MUSE_PIXTABLE_DQ,
128  CPL_NOT_EQUAL_TO, EURO3D_GOODPIXEL);
129  cpl_table_erase_selected(pixtable->table);
130 
131  /* do DAR correction for WFM data */
132  if (muse_pfits_get_mode(pixtable->header) < MUSE_MODE_NFM_AO_N) {
133  cpl_msg_debug(__func__, "WFM detected: starting DAR correction");
134  cpl_error_code rc = muse_dar_correct(pixtable, aParams->lambdaref);
135  cpl_msg_debug(__func__, "DAR correction returned rc=%d: %s", rc,
136  rc != CPL_ERROR_NONE ? cpl_error_get_message() : "");
137  } /* if WFM */
138  }
139 
140  return pixtable;
141 }
142 
143 /*----------------------------------------------------------------------------*/
153 /*----------------------------------------------------------------------------*/
154 static muse_image *
155 muse_create_sky_whitelight_image(muse_pixtable *aPixtable, double aCRSigma)
156 {
157  cpl_boolean usegrid = getenv("MUSE_COLLAPSE_PIXTABLE")
158  && atoi(getenv("MUSE_COLLAPSE_PIXTABLE")) > 0;
162  if (aCRSigma > 0.) {
164  params->crsigma = aCRSigma;
165  }
166  muse_pixgrid *grid = NULL;
167  muse_datacube *cube = muse_resampling_cube(aPixtable, params,
168  usegrid ? &grid : NULL);
169  if (cube == NULL) {
170  cpl_msg_error(__func__, "Could not create cube for whitelight image");
172  muse_pixgrid_delete(grid);
173  return NULL;
174  }
175  muse_image *image = NULL;
176  muse_table *fwhite = muse_table_load_filter(NULL, "white");
177  if (usegrid) {
179  image = muse_resampling_collapse_pixgrid(aPixtable, grid,
180  cube, fwhite, params);
181  } else {
182  image = muse_datacube_collapse(cube, fwhite);
183  }
185  muse_pixgrid_delete(grid);
186  muse_datacube_delete(cube);
187  muse_table_delete(fwhite);
188  return image;
189 } /* muse_create_sky_whitelight_image() */
190 
191 /*----------------------------------------------------------------------------*/
198 /*----------------------------------------------------------------------------*/
199 int
200 muse_create_sky_compute(muse_processing *aProcessing,
201  muse_create_sky_params_t *aParams)
202 {
203  cpl_array *crarray = muse_cplarray_new_from_delimited_string(aParams->crsigma, ",");
204  double crsigmac = 15., /* for cube-based CR rejection */
205  crsigmas = 0.; /* spectrum-based CR rejection */
206  if (cpl_array_get_size(crarray) < 2) {
207  cpl_msg_warning(__func__, "Less than two values given by crsigma "
208  "parameter, using defaults (%.3f.,%.3f)!", crsigmac,
209  crsigmas);
210  } else {
211  crsigmac = cpl_array_get_string(crarray, 0)
212  ? atof(cpl_array_get_string(crarray, 0)) : 15.;
213  crsigmas = cpl_array_get_string(crarray, 1)
214  ? atof(cpl_array_get_string(crarray, 1)) : 0.;
215  }
216  cpl_array_delete(crarray);
217 
218  muse_pixtable *pixtable = muse_create_sky_load_pixtable(aProcessing, aParams);
219  if (pixtable == NULL) {
220  cpl_msg_error(__func__, "Could not load pixel table");
221  return -1;
222  }
223 
224  /*
225  First step: find the regions containing sky:
226  - create a whitelight image
227  - create the mask from it
228  - select pixtable rows containing sky
229  - create spectrum from selected rows
230  */
231  muse_mask *smask = muse_processing_load_mask(aProcessing, MUSE_TAG_SKY_MASK);
232  if (smask) { /* apply existing mask on input pixel table */
233  cpl_table_select_all(pixtable->table);
234  muse_pixtable_and_selected_mask(pixtable, smask);
235  muse_mask_delete(smask);
236  }
237 
238  muse_image *whitelight = muse_create_sky_whitelight_image(pixtable, crsigmac);
239  if (whitelight == NULL) {
240  cpl_msg_error(__func__, "Could not create whitelight image");
241  return -1;
242  }
243  muse_processing_save_image(aProcessing, -1, whitelight, MUSE_TAG_IMAGE_FOV);
244 
245  muse_mask *sky_mask = muse_sky_create_skymask(whitelight, aParams->ignore,
246  aParams->fraction,
247  "ESO QC SKY");
248  muse_processing_save_mask(aProcessing, -1, sky_mask, MUSE_TAG_SKY_MASK);
249 
250  cpl_table_select_all(pixtable->table);
251  muse_pixtable_and_selected_mask(pixtable, sky_mask);
252  cpl_table_not_selected(pixtable->table);
253  cpl_table_erase_selected(pixtable->table);
254  muse_image_delete(whitelight);
255  muse_mask_delete(sky_mask);
256 
257  /* Apply mask and create spectrum, possibly with one CR-reject iteration */
258  cpl_table *spectrum = muse_resampling_spectrum_iterate(pixtable,
259  aParams->sampling,
260  0., crsigmas, 1);
261  if (spectrum == NULL) {
262  muse_pixtable_delete(pixtable);
263  return -1;
264  }
265  muse_processing_save_table(aProcessing, -1, spectrum, pixtable->header,
266  MUSE_TAG_SKY_SPECTRUM, MUSE_TABLE_TYPE_CPL);
267 
268  cpl_table *lines = muse_sky_lines_load(aProcessing);
269  if (lines != NULL) {
270  double lambda_low = cpl_table_get_column_min(spectrum, "lambda");
271  double lambda_high = cpl_table_get_column_max(spectrum, "lambda");
272  muse_sky_lines_set_range(lines, lambda_low-5, lambda_high+5);
273  }
274 
275  cpl_table *continuum = muse_sky_continuum_load(aProcessing);
276 
277  /*
278  Second step: create master sky, containing line list and
279  continuum. If a continuum was given as calibration frame, we
280  ignore the computed one and replace it by the external one.
281  */
282 
283  cpl_errorstate prestate = cpl_errorstate_get();
284  /*
285  Create the average LSF image for the spectrum of the pixels
286  selected as sky. This is done in the following steps:
287 
288  1. Average all cubes with the pixel count as weights into a single
289  image.
290 
291  2. Fold an additional rectangle to model the sky spectrum binning
292  */
293  muse_lsf_cube **lsfCube = muse_lsf_cube_load_all(aProcessing);
294  if (lsfCube != NULL) {
295  cpl_image *lsfImage = muse_lsf_average_cube_all(lsfCube, pixtable);
296  muse_wcs *lsfWCS = muse_lsf_cube_get_wcs_all(lsfCube);
297  muse_lsf_fold_rectangle(lsfImage, lsfWCS, aParams->sampling);
298 
299  cpl_msg_info(__func__, "Creating master sky spectrum using fits to lines "
300  "(fluxes) and residual continuum");
301 
302  // do the fit, ignoring possible errors
303  muse_sky_lines_fit(spectrum, lines, lsfImage, lsfWCS);
304  if (!cpl_errorstate_is_equal(prestate)) {
305  cpl_errorstate_dump(prestate, CPL_FALSE, NULL);
306  cpl_errorstate_set(prestate);
307  }
308 
309  if (continuum == NULL) {
310  continuum = muse_sky_continuum_create(spectrum, lines, lsfImage, lsfWCS,
311  aParams->csampling);
312  }
313 
314  muse_lsf_cube_delete_all(lsfCube);
315  cpl_image_delete(lsfImage);
316 #ifdef USE_LSF_PARAMS
317  } else {
318  cpl_errorstate_set(prestate);
319  muse_lsf_params **lsfParams = muse_processing_lsf_params_load(aProcessing, 0);
320  if (lsfParams != NULL) { // Old LSF params code
321 
322  // do the fit, ignoring possible errors
323  muse_sky_lines_fit_old(spectrum, lines);
324  if (!cpl_errorstate_is_equal(prestate)) {
325  cpl_errorstate_dump(prestate, CPL_FALSE, NULL);
326  cpl_errorstate_set(prestate);
327  }
328 
329  if (continuum == NULL) {
330  muse_sky_subtract_lines_old(pixtable, lines, lsfParams);
331  continuum = muse_resampling_spectrum(pixtable, aParams->csampling);
332  cpl_table_erase_column(continuum, "stat");
333  cpl_table_erase_column(continuum, "dq");
334  cpl_table_name_column(continuum, "data", "flux");
335  }
336 
337  muse_lsf_params_delete_all(lsfParams);
338  } else {
339  cpl_msg_error(__func__, "Could not load LSF. Continuum is not created.");
340  }
341 #endif
342  }
343  /* add QC parameters for both emission lines and continuum and save them */
344  cpl_propertylist *header = cpl_propertylist_new();
345  muse_sky_qc_lines(header, lines, "ESO QC SKY");
346  muse_sky_lines_save(aProcessing, lines, header);
347  cpl_propertylist_delete(header);
348 
349  header = cpl_propertylist_new();
350  muse_sky_qc_continuum(header, continuum, "ESO QC SKY");
351  muse_sky_save_continuum(aProcessing, continuum, header);
352  cpl_propertylist_delete(header);
353 
354  /* Clean up the local objects. */
355  cpl_table_delete(spectrum);
356  cpl_table_delete(lines);
357  cpl_table_delete(continuum);
358  muse_pixtable_delete(pixtable);
359  return cpl_errorstate_is_equal(prestate) ? 0 : -1;
360 }
361 
cpl_error_code muse_sky_lines_fit(cpl_table *, cpl_table *, cpl_image *, muse_wcs *)
Fit all entries of the pixel table to the master sky.
cpl_table * muse_sky_continuum_create(cpl_table *, cpl_table *, cpl_image *, muse_wcs *, double)
Create a continuum spectrum.
#define MUSE_PIXTABLE_DQ
Definition: muse_pixtable.h:49
muse_image * muse_resampling_collapse_pixgrid(muse_pixtable *aPixtable, muse_pixgrid *aGrid, muse_datacube *aCube, const muse_table *aFilter, muse_resampling_params *aParams)
Integrate a pixel table / pixel grid along the wavelength direction.
Structure definition of a MUSE datacube.
Definition: muse_datacube.h:48
void muse_image_delete(muse_image *aImage)
Deallocate memory associated to a muse_image object.
Definition: muse_image.c:85
int muse_processing_save_mask(muse_processing *aProcessing, int aIFU, muse_mask *aMask, const char *aTag)
Save a computed MUSE mask to disk.
A structure containing a spatial two-axis WCS.
Definition: muse_wcs.h:105
cpl_table * muse_resampling_spectrum_iterate(muse_pixtable *aPixtable, double aBinwidth, float aLo, float aHi, unsigned char aIter)
Iteratively resample selected pixels of a pixel table into spectrum.
cpl_error_code muse_sky_lines_save(muse_processing *, const cpl_table *, cpl_propertylist *)
Save sky lines table to file.
void muse_mask_delete(muse_mask *aMask)
Deallocate memory associated to a muse_mask object.
Definition: muse_mask.c:78
cpl_error_code muse_dar_correct(muse_pixtable *aPixtable, double aLambdaRef)
Correct the pixel coordinates of all pixels of a given pixel table for differential atmospheric refra...
Definition: muse_dar.c:139
cpl_error_code muse_postproc_revert_ffspec_maybe(muse_pixtable *aPt, const muse_table *aResponse)
Revert correction of on-sky data with the flat-field spectrum.
cpl_image * muse_lsf_average_cube_all(muse_lsf_cube **aLsfCube, muse_pixtable *aPixtable)
Create an average image from all LSF cubes.
cpl_error_code muse_pixtable_and_selected_mask(muse_pixtable *aPixtable, muse_mask *aMask)
Select all pixels where the (x,y) positions are enabled in the given mask.
cpl_table * muse_sky_lines_load(muse_processing *)
Load the sky data files.
double csampling
Spectral sampling of the continuum spectrum [Angstrom].
muse_datacube * muse_resampling_cube(muse_pixtable *aPixtable, muse_resampling_params *aParams, muse_pixgrid **aGrid)
Resample a pixel table onto a regular grid structure representing a FITS NAXIS=3 datacube.
void muse_datacube_delete(muse_datacube *aCube)
Deallocate memory associated to a muse_datacube object.
Data cube/stacked image list containing the LSF for one IFU.
Definition: muse_lsf.h:49
void muse_sky_qc_lines(cpl_propertylist *, cpl_table *, const char *)
Fill a header with the QC parameters for the sky lines.
Definition: muse_sky_qc.c:46
The pixel grid.
Definition: muse_pixgrid.h:65
Structure definition of MUSE three extension FITS file.
Definition: muse_image.h:40
cpl_error_code muse_flux_calibrate(muse_pixtable *aPixtable, const muse_table *aResponse, const cpl_table *aExtinction, const muse_table *aTelluric)
Convert the input pixel table from counts to fluxes.
Definition: muse_flux.c:3083
cpl_table * table
The pixel table.
void muse_lsf_params_delete_all(muse_lsf_params **aParams)
Delete an allocated array of muse_lsf_params structure.
const char * crsigma
Sigma level clipping for cube-based and spectrum-based CR rejection. This has to be a string of two c...
muse_resampling_crstats_type crtype
void muse_sky_qc_continuum(cpl_propertylist *, cpl_table *, const char *)
Fill a header with the QC parameters for the sky continuum.
Definition: muse_sky_qc.c:102
cpl_boolean muse_pixtable_is_skysub(muse_pixtable *aPixtable)
Determine whether the pixel table is sky subtracted.
void muse_pixgrid_delete(muse_pixgrid *aGrid)
Delete a pixel grid and remove its memory.
Definition: muse_pixgrid.c:575
cpl_table * muse_sky_continuum_load(muse_processing *)
Load the sky continuum.
cpl_array * muse_cplarray_new_from_delimited_string(const char *aString, const char *aDelim)
Convert a delimited string into an array of strings.
Structure definition of MUSE pixel table.
cpl_error_code muse_sky_subtract_lines_old(muse_pixtable *aPixtable, cpl_table *aLines, muse_lsf_params **aLsfParams)
Subtract sky lines from a pixtable.
Definition: muse_sky_old.c:532
double ignore
Fraction of the image to be ignored. If an input sky mask is provided, the fraction is applied to the...
muse_resampling_params * muse_resampling_params_new(muse_resampling_type aMethod)
Create the resampling parameters structure.
cpl_boolean muse_pixtable_is_fluxcal(muse_pixtable *aPixtable)
Determine whether the pixel table is flux calibrated.
muse_lsf_cube ** muse_lsf_cube_load_all(muse_processing *aProcessing)
Load all LSF cubes for all IFUs into an array.
muse_table * muse_processing_load_table(muse_processing *aProcessing, const char *aTag, unsigned char aIFU)
Load a MUSE table according to its tag and IFU/channel number.
muse_table * muse_table_load_filter(muse_processing *aProcessing, const char *aFilterName)
Load a table for a given filter name.
Definition: muse_utils.c:799
cpl_error_code muse_sky_lines_set_range(cpl_table *, double, double)
Limit the lines in the table to a wavelength range.
Structure to store a table together with a property list.
Definition: muse_table.h:43
muse_mask * muse_processing_load_mask(muse_processing *aProcessing, const char *aTag)
Load a mask file and its FITS header.
void muse_table_delete(muse_table *aTable)
Deallocate memory associated to a muse_table object.
Definition: muse_table.c:80
int muse_processing_save_image(muse_processing *aProcessing, int aIFU, muse_image *aImage, const char *aTag)
Save a computed MUSE image to disk.
cpl_table * muse_processing_load_ctable(muse_processing *aProcessing, const char *aTag, unsigned char aIFU)
Load a CPL table according to its tag and IFU/channel number.
muse_pixtable * muse_pixtable_load_merge_channels(cpl_table *aExposureList, double aLambdaMin, double aLambdaMax)
Load and merge the pixel tables of the 24 MUSE sub-fields.
Handling of "mask" files.
Definition: muse_mask.h:43
void muse_lsf_cube_delete_all(muse_lsf_cube **aLsfCube)
Delete all LSF cubes.
double lambdamin
Cut off the data below this wavelength after loading the pixel table(s).
muse_mask * muse_sky_create_skymask(muse_image *, double, double, const char *)
Select spaxels to be considered as sky.
double sampling
Spectral sampling of the sky spectrum [Angstrom].
cpl_error_code muse_processing_save_table(muse_processing *aProcessing, int aIFU, void *aTable, cpl_propertylist *aHeader, const char *aTag, muse_table_type aType)
Save a computed table to disk.
double fraction
Fraction of the image (without the ignored part) to be considered as sky. If an input sky mask is pro...
muse_resampling_type
Resampling types.
double lambdamax
Cut off the data above this wavelength after loading the pixel table(s).
cpl_error_code muse_lsf_fold_rectangle(cpl_image *aLsfImage, const muse_wcs *aWCS, double aBinWidth)
Filter an LSF image with a rectangle to model spectrum binning.
double lambdaref
Reference wavelength used for correction of differential atmospheric refraction. The R-band (peak wav...
Resampling parameters.
void muse_resampling_params_delete(muse_resampling_params *aParams)
Delete a resampling parameters structure.
muse_resampling_type method
muse_lsf_params ** muse_processing_lsf_params_load(muse_processing *aProcessing, int aIFU)
Load slice LSF parameters.
muse_image * muse_datacube_collapse(muse_datacube *aCube, const muse_table *aFilter)
Integrate a FITS NAXIS=3 datacube along the wavelength direction.
cpl_table * muse_resampling_spectrum(muse_pixtable *aPixtable, double aBinwidth)
Resample the selected pixels of a pixel table into a spectrum.
cpl_error_code muse_sky_lines_fit_old(cpl_table *aSpectrum, cpl_table *aLines)
Fit all entries of the pixel table to the master sky.
Definition: muse_sky_old.c:436
cpl_table * muse_processing_sort_exposures(muse_processing *aProcessing)
Sort input frames (containing lists of pixel table filenames) into different exposures.
Structure definition of detector (slice) parameters.
void muse_pixtable_delete(muse_pixtable *aPixtable)
Deallocate memory associated to a pixel table object.
Structure to hold the parameters of the muse_create_sky recipe.
muse_ins_mode muse_pfits_get_mode(const cpl_propertylist *aHeaders)
find out the observation mode
Definition: muse_pfits.c:1352
cpl_propertylist * header
The FITS header.
cpl_error_code muse_sky_save_continuum(muse_processing *, const cpl_table *, cpl_propertylist *)
Save sky continuum table to file.