MUSE Pipeline Reference Manual  2.1.1
muse_flat.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  * This file is part of the MUSE Instrument Pipeline
5  * Copyright (C) 2005-2014 European Southern Observatory
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  */
21 
22 #ifdef HAVE_CONFIG_H
23 #include <config.h>
24 #endif
25 
26 /*----------------------------------------------------------------------------*
27  * Includes *
28  *----------------------------------------------------------------------------*/
29 #include <stdio.h>
30 #include <float.h>
31 #include <math.h>
32 #include <string.h>
33 #include <cpl.h>
34 #include <muse.h>
35 #include "muse_flat_z.h"
36 
37 /*---------------------------------------------------------------------------*
38  * Functions code *
39  *---------------------------------------------------------------------------*/
40 
41 /*---------------------------------------------------------------------------*/
47 /*---------------------------------------------------------------------------*/
48 static void
49 muse_flat_qc_header(muse_image *aImage, muse_imagelist *aList)
50 {
51  cpl_msg_debug(__func__, "Adding QC keywords");
52 
53  unsigned stats = CPL_STATS_MEDIAN | CPL_STATS_MEAN | CPL_STATS_STDEV
54  | CPL_STATS_MIN | CPL_STATS_MAX;
55 
56  /* write the image statistics for input flat field exposures */
57  unsigned int k;
58  for (k = 0; k < muse_imagelist_get_size(aList); k++) {
59  char *keyword = cpl_sprintf(QC_FLAT_PREFIXi, k+1);
61  aImage->header, keyword, stats);
62  cpl_free(keyword);
63  keyword = cpl_sprintf(QC_FLAT_PREFIXi" "QC_BASIC_NSATURATED, k+1);
64  int nsaturated = cpl_propertylist_get_int(muse_imagelist_get(aList, k)->header,
65  MUSE_HDR_TMP_NSAT);
66  cpl_propertylist_update_int(aImage->header, keyword, nsaturated);
67  cpl_free(keyword);
68  } /* for k (all images in list) */
69 
70  /* properties of the resulting master frame */
71  stats |= CPL_STATS_FLUX;
73  QC_FLAT_MASTER_PREFIX, stats);
74 } /* muse_flat_qc_header() */
75 
76 /*---------------------------------------------------------------------------*/
82 /*---------------------------------------------------------------------------*/
83 static void
84 muse_flat_qc_trace_header(cpl_propertylist *aHeader, const cpl_table *aTrace)
85 {
86  if (!aHeader || !aTrace) {
87  return;
88  }
89  unsigned char ifu = muse_utils_get_ifu(aHeader);
90  cpl_msg_debug(__func__, "Adding tracing QC keywords for IFU %hhu", ifu);
91 
92  /* create vector for gap statistics */
93  cpl_array *gaps = cpl_array_new(kMuseSlicesPerCCD - 1, CPL_TYPE_DOUBLE);
94  double redge = -1; /* keep right edge of slice */
95  int nslice;
96  for (nslice = 1; nslice <= kMuseSlicesPerCCD; nslice++) {
97  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
98  nslice);
99  if (!ptrace) {
100  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: tracing polynomials "
101  "missing!", nslice, ifu);
102  continue;
103  }
104 
105  /* ask for the center of the slice at the vertical center of the CCD */
106  float xpos = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER],
107  kMuseOutputYTop/2, NULL);
108  if (xpos < 1 || xpos > kMuseOutputXRight || !isnormal(xpos)) {
109  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: faulty polynomial "
110  "detected at y=%d (center: %f)", nslice, ifu,
111  kMuseOutputYTop/2, xpos);
112  muse_trace_polys_delete(ptrace);
113  continue; /* next slice */
114  }
115  /* also may need the centers of the trace *
116  * at bottom and top for tilt computation: */
117  double x1 = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER], 1, NULL),
118  x2 = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_CENTER],
119  kMuseOutputYTop, NULL);
120  float tilt = atan((x2 - x1) / (kMuseOutputYTop - 1)) * CPL_MATH_DEG_RAD;
121  if (nslice == 1) {
122  cpl_propertylist_append_float(aHeader, QC_TRACE_L_XPOS, xpos);
123  cpl_propertylist_append_float(aHeader, QC_TRACE_L_TILT, tilt);
124  } else if (nslice == kMuseSlicesPerCCD) {
125  cpl_propertylist_append_float(aHeader, QC_TRACE_R_XPOS, xpos);
126  cpl_propertylist_append_float(aHeader, QC_TRACE_R_TILT, tilt);
127  }
128  if (redge > 0) {
129  /* the gap is the difference between the left edge of the *
130  * current slice and the right edge of the previous slice */
131  double gap = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT],
132  kMuseOutputYTop/2, NULL)
133  - redge;
134  cpl_array_set_double(gaps, nslice - 2, gap);
135  }
136  /* compute right edge for gap computation */
137  redge = cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT], kMuseOutputYTop/2,
138  NULL);
139 
140  /* Compute derivatives of all three tracing polynominals to *
141  * derive the maximum slope of the solutions within the CCD. */
142  double maxslope = 0.;
143  int ipoly, j, jmax = -1, ipolymax = -1;
144  for (ipoly = MUSE_TRACE_CENTER; ipoly <= MUSE_TRACE_RIGHT; ipoly++) {
145  cpl_polynomial_derivative(ptrace[ipoly], 0);
146  /* have to evaluate the polynomial at all positions to *
147  * get the maximum within the typical CCD boundaries... */
148  for (j = 1; j <= kMuseOutputYTop; j++) {
149  double slope = cpl_polynomial_eval_1d(ptrace[ipoly], j, NULL);
150  if (fabs(slope) > fabs(maxslope)) {
151  maxslope = slope;
152  jmax = j;
153  ipolymax = ipoly;
154  } /* if */
155  } /* for j (all positions in vertical direction) */
156  } /* for ipoly (all tracing polynomials) */
157 #define MAX_NORMAL_SLOPE 0.015
158  if (fabs(maxslope) > MAX_NORMAL_SLOPE) {
159  cpl_msg_warning(__func__, "Slope in slice %2d of IFU %hhu is unusually "
160  "high (maximum in polynomial %d at y=%d is |%f| > %f)",
161  nslice, ifu, ipolymax, jmax, maxslope, MAX_NORMAL_SLOPE);
162  }
163  char *kw = cpl_sprintf(QC_TRACE_SLICEj_MXSLOP, nslice);
164  cpl_propertylist_append_float(aHeader, kw, maxslope);
165  cpl_free(kw);
166  muse_trace_polys_delete(ptrace);
167 
168  /* add widths of corner slices as QC parameter to detect clipped edges */
169  if (nslice == 10 || nslice == 46 || nslice == 3 || nslice == 39) {
170  /* get entry for the slice (table should be ordered, but to be sure) */
171  int irow, err, nrow = cpl_table_get_nrow(aTrace);
172  for (irow = 0; irow < nrow; irow++) {
173  int slice = cpl_table_get_int(aTrace, MUSE_TRACE_TABLE_COL_SLICE_NO,
174  irow, &err);
175  if (slice == nslice && !err) {
176  break;
177  }
178  } /* for irow */
179  if (irow >= nrow) { /* slice not found! */
180  cpl_msg_warning(__func__, "Slice %d not found in trace table!", nslice);
181  continue;
182  }
183  float width = cpl_table_get(aTrace, MUSE_TRACE_TABLE_COL_WIDTH, irow, &err);
184  kw = cpl_sprintf(QC_TRACE_SLICEj_WIDTH, nslice);
185  cpl_propertylist_append_float(aHeader, kw, width);
186  cpl_free(kw);
187  } /* if corner slice */
188  } /* for nslice */
189 
190  float median, mean, stdev, min, max;
191  /* now add the gap statistics as QC parameters */
192  median = cpl_array_get_median(gaps);
193  mean = cpl_array_get_mean(gaps);
194  stdev = cpl_array_get_stdev(gaps);
195  min = cpl_array_get_min(gaps);
196  max = cpl_array_get_max(gaps);
197  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_MEDIAN, median);
198  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_MEAN, mean);
199  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_STDEV, stdev);
200  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_MIN, min);
201  cpl_propertylist_append_float(aHeader, QC_TRACE_GAPS_MAX, max);
202  cpl_array_delete(gaps);
203 
204  /* add the slice widths also as QC paramters */
205  median = cpl_table_get_column_median(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
206  mean = cpl_table_get_column_mean(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
207  stdev = cpl_table_get_column_stdev(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
208  min = cpl_table_get_column_min(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
209  max = cpl_table_get_column_max(aTrace, MUSE_TRACE_TABLE_COL_WIDTH);
210  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_MEDIAN, median);
211  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_MEAN, mean);
212  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_STDEV, stdev);
213  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_MIN, min);
214  cpl_propertylist_append_float(aHeader, QC_TRACE_WIDTHS_MAX, max);
215 } /* muse_flat_qc_trace_header() */
216 
217 /*---------------------------------------------------------------------------*/
223 /*---------------------------------------------------------------------------*/
224 static void
225 muse_flat_qc_per_slice(muse_image *aImage, const cpl_table *aTrace)
226 {
227  if (!aImage->header || !aTrace) {
228  return;
229  }
230  unsigned char ifu = muse_utils_get_ifu(aImage->header);
231  cpl_msg_debug(__func__, "Adding per-slice QC statistics for IFU %hhu", ifu);
232 
233  /* common vertical region in each slice, around the slice center */
234  const int y1 = kMuseOutputYTop/2 - 100,
235  y2 = kMuseOutputYTop/2 + 100;
236  int nslice;
237  for (nslice = 1; nslice <= kMuseSlicesPerCCD; nslice++) {
238  cpl_polynomial **ptrace = muse_trace_table_get_polys_for_slice(aTrace,
239  nslice);
240  if (!ptrace) {
241  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: tracing polynomials "
242  "missing!", nslice, ifu);
243  continue;
244  }
245  char *kwmean = cpl_sprintf(QC_FLAT_MASTER_SLICEj_MEAN, nslice),
246  *kwstdev = cpl_sprintf(QC_FLAT_MASTER_SLICEj_STDEV, nslice);
247  /* horizontal region for this slice */
248  int x1 = lround(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_LEFT],
249  kMuseOutputYTop/2, NULL)) + 1,
250  x2 = lround(cpl_polynomial_eval_1d(ptrace[MUSE_TRACE_RIGHT],
251  kMuseOutputYTop/2, NULL)) - 1;
252  if (x1 < 1 || x2 > kMuseOutputXRight || x1 > x2) {
253  cpl_msg_warning(__func__, "slice %2d of IFU %hhu: faulty polynomial "
254  "detected at y=%d (borders: %d ... %d)", nslice, ifu,
255  kMuseOutputYTop/2, x1, x2);
256  muse_trace_polys_delete(ptrace);
257  continue; /* next slice */
258  }
259  double mean = cpl_image_get_mean_window(aImage->data, x1, y1, x2, y2),
260  stdev = cpl_image_get_stdev_window(aImage->data, x1, y1, x2, y2);
261  cpl_propertylist_update_float(aImage->header, kwmean, mean);
262  cpl_propertylist_update_float(aImage->header, kwstdev, stdev);
263 
264  muse_trace_polys_delete(ptrace);
265  cpl_free(kwmean);
266  cpl_free(kwstdev);
267  } /* for nslice */
268 } /* muse_flat_qc_per_slice() */
269 
270 /*---------------------------------------------------------------------------*/
278 /*---------------------------------------------------------------------------*/
279 static cpl_error_code
280 muse_flat_trace_badpix(muse_processing *aProcessing,
281  muse_flat_params_t *aParams, muse_image *aImage)
282 {
283  cpl_table *samples = NULL;
284  cpl_table *tracetable = muse_trace(aImage, aParams->nsum, aParams->edgefrac,
285  aParams->order,
286  aParams->samples ? &samples : NULL);
287  if (!tracetable) {
288  cpl_table_delete(samples); /* just in case... */
289  return cpl_error_get_code();
290  }
291 
292  /* create table header by copying most of the image header */
293  cpl_propertylist *header = cpl_propertylist_duplicate(aImage->header);
294  cpl_propertylist_erase_regexp(header,
295  "^SIMPLE$|^BITPIX$|^NAXIS|^EXTEND$|^XTENSION$|"
296  "^DATASUM$|^DATAMIN$|^DATAMAX$|^DATAMD5$|"
297  "^PCOUNT$|^GCOUNT$|^HDUVERS$|^BLANK$|"
298  "^BZERO$|^BSCALE$|^BUNIT$|^CHECKSUM$|^INHERIT$|"
299  "^PIPEFILE$|^ESO QC |^ESO PRO ", 0);
300 
301  muse_flat_qc_trace_header(header, tracetable);
302  cpl_error_code rc = muse_processing_save_table(aProcessing, aParams->nifu,
303  tracetable, header,
304  MUSE_TAG_TRACE_TABLE,
306  if (samples) {
307  /* don't want to save QC headers with the samples table: */
308  cpl_propertylist_erase_regexp(header, QC_TRACE_PREFIX, 0);
309  muse_processing_save_table(aProcessing, aParams->nifu, samples, header,
310  MUSE_TAG_TRACE_SAMPLES, MUSE_TABLE_TYPE_CPL);
311  cpl_table_delete(samples);
312  }
313  cpl_propertylist_delete(header);
314  if (rc != CPL_ERROR_NONE) {
315  cpl_table_delete(tracetable);
316  return rc;
317  }
318 
319  /* if we have successfully traced, we can also search for dark pixels */
320  int nbad = muse_quality_flat_badpix(aImage, tracetable,
321  aParams->losigmabadpix,
322  aParams->hisigmabadpix);
323  cpl_msg_info(__func__, "Found %d bad pixels in the flat-field image of IFU %d",
324  nbad, aParams->nifu);
325 
326  /* add image statistics as QC now that we know the slice locations */
327  muse_flat_qc_per_slice(aImage, tracetable);
328 
329  cpl_table_delete(tracetable);
330  return rc;
331 } /* muse_flat_trace_badpix() */
332 
333 /*---------------------------------------------------------------------------*/
340 /*---------------------------------------------------------------------------*/
341 int
342 muse_flat_compute(muse_processing *aProcessing, muse_flat_params_t *aParams)
343 {
345  "muse.muse_flat");
346  muse_imagelist *images = muse_basicproc_load(aProcessing, aParams->nifu, bpars);
348  cpl_ensure(images, cpl_error_get_code(), -1);
349 
350  muse_combinepar *cpars = muse_combinepar_new(aProcessing->parameters,
351  "muse.muse_flat");
352  muse_image *masterimage = muse_combine_images(cpars, images);
353  muse_combinepar_delete(cpars);
354  if (!masterimage) {
355  cpl_msg_error(__func__, "Combining input frames failed for IFU %d!",
356  aParams->nifu);
357  muse_imagelist_delete(images);
358  return -1;
359  }
360  cpl_propertylist_erase_regexp(masterimage->header, MUSE_WCS_KEYS, 0);
361 
362  muse_flat_qc_header(masterimage, images);
363  muse_imagelist_delete(images);
364 
365  cpl_error_code rc1 = CPL_ERROR_NONE;
366  if (aParams->trace) {
367  /* trace and search for bad pixels; do this before saving the *
368  * masterimage, so that we can add bad pixels to its DQ extension */
369  rc1 = muse_flat_trace_badpix(aProcessing, aParams, masterimage);
370  if (rc1 != CPL_ERROR_NONE) {
371  /* warn but don't return yet; tracing is important but the user should *
372  * still be able to get the masterimage saved to take a look at it */
373  cpl_msg_error(__func__, "Tracing/bad pixel search failed in IFU %d",
374  aParams->nifu);
375  }
376  }
377 
378  /* if requested, normalize the output flat-field to 1 using the average value */
379  if (aParams->normalize) {
380  double mean = cpl_propertylist_get_float(masterimage->header,
381  QC_FLAT_MASTER_PREFIX" MEAN");
382  muse_image_scale(masterimage, 1. / mean);
383  }
384 
385  muse_basicproc_qc_saturated(masterimage, QC_FLAT_MASTER_PREFIX);
386  cpl_error_code rc2 = muse_processing_save_image(aProcessing, aParams->nifu,
387  masterimage,
388  MUSE_TAG_MASTER_FLAT);
389  muse_image_delete(masterimage);
390  return rc1 == CPL_ERROR_NONE && rc2 == CPL_ERROR_NONE ? 0 : -1;
391 } /* muse_flat_compute() */
muse_imagelist * muse_basicproc_load(muse_processing *aProcessing, unsigned char aIFU, muse_basicproc_params *aBPars)
Load the raw input files from disk and do basic processing.
cpl_polynomial ** muse_trace_table_get_polys_for_slice(const cpl_table *aTable, const unsigned short aSlice)
construct polynomial from the trace table entry for the given slice
int nifu
IFU to handle. If set to 0, all IFUs are processed serially. If set to -1, all IFUs are processed in ...
Definition: muse_flat_z.h:50
int order
Order of polynomial fit to the trace.
Definition: muse_flat_z.h:97
Structure definition for a collection of muse_images.
void muse_image_delete(muse_image *aImage)
Deallocate memory associated to a muse_image object.
Definition: muse_image.c:85
int muse_quality_flat_badpix(muse_image *aFlat, cpl_table *aTrace, double aSigmaLo, double aSigmaHi)
Find bad (especially dark) pixels (in a master flat).
Definition: muse_quality.c:315
int muse_image_scale(muse_image *aImage, double aScale)
Scale a muse_image with correct treatment of variance.
Definition: muse_image.c:703
cpl_table * muse_trace(const muse_image *aImage, int aNSum, double aEdgeFrac, int aFitorder, cpl_table **aSamples)
carry out the tracing of the slices on CCD, save parameters in table
Definition: muse_tracing.c:967
unsigned char muse_utils_get_ifu(const cpl_propertylist *aHeaders)
Find out the IFU/channel from which this header originated.
Definition: muse_utils.c:98
cpl_image * data
the data extension
Definition: muse_image.h:46
int normalize
Normalize the master flat to the average flux.
Definition: muse_flat_z.h:88
Structure to hold the parameters of the muse_flat recipe.
Definition: muse_flat_z.h:48
void muse_imagelist_delete(muse_imagelist *aList)
Free the memory of the MUSE image list.
muse_basicproc_params * muse_basicproc_params_new(cpl_parameterlist *aParameters, const char *aPrefix)
Create a new structure of basic processing parameters.
muse_image * muse_combine_images(muse_combinepar *aCPars, muse_imagelist *aImages)
Combine several images into one.
Definition: muse_combine.c:741
Structure definition of MUSE three extension FITS file.
Definition: muse_image.h:40
void muse_basicproc_params_delete(muse_basicproc_params *aBPars)
Free a structure of basic processing parameters.
cpl_propertylist * header
the FITS header
Definition: muse_image.h:72
unsigned int muse_imagelist_get_size(muse_imagelist *aList)
Return the number of stored images.
void muse_trace_polys_delete(cpl_polynomial *aPolys[])
Delete the multi-polynomial array created in relation to tracing.
void muse_combinepar_delete(muse_combinepar *aCPars)
Clear the combination parameters.
Definition: muse_combine.c:715
#define MUSE_WCS_KEYS
regular expression for WCS properties
Definition: muse_wcs.h:48
muse_image * muse_imagelist_get(muse_imagelist *aList, unsigned int aIdx)
Get the muse_image of given list index.
int nsum
Number of lines over which to average when tracing.
Definition: muse_flat_z.h:94
int samples
Create a table containing all tracing sample points.
Definition: muse_flat_z.h:109
muse_combinepar * muse_combinepar_new(cpl_parameterlist *aParameters, const char *aPrefix)
Create a new set of combination parameters.
Definition: muse_combine.c:672
cpl_error_code muse_basicproc_stats_append_header(cpl_image *aImage, cpl_propertylist *aHeader, const char *aPrefix, unsigned aStats)
Compute image statistics of an image and add them to a header.
int muse_processing_save_image(muse_processing *aProcessing, int aIFU, muse_image *aImage, const char *aTag)
Save a computed MUSE image to disk.
double edgefrac
Fractional change required to identify edge when tracing.
Definition: muse_flat_z.h:100
double losigmabadpix
Low sigma to find dark pixels in the master flat.
Definition: muse_flat_z.h:103
cpl_error_code muse_processing_save_table(muse_processing *aProcessing, int aIFU, void *aTable, cpl_propertylist *aHeader, const char *aTag, muse_table_type aType)
Save a computed table to disk.
int trace
Trace the position of the slices on the master flat.
Definition: muse_flat_z.h:91
Structure of basic processing parameters.
cpl_parameterlist * parameters
double hisigmabadpix
High sigma to find bright pixels in the master flat.
Definition: muse_flat_z.h:106
cpl_error_code muse_basicproc_qc_saturated(muse_image *aImage, const char *aPrefix)
Add QC parameter about saturated pixels to a muse_image.