MUSE Pipeline Reference Manual  2.1.1
muse_basicproc.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  * This file is part of the MUSE Instrument Pipeline
5  * Copyright (C) 2005-2015 European Southern Observatory
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  */
21 
22 #ifdef HAVE_CONFIG_H
23 #include <config.h>
24 #endif
25 
26 /*----------------------------------------------------------------------------*
27  * Includes *
28  *----------------------------------------------------------------------------*/
29 #include <stdio.h>
30 #include <float.h>
31 #include <math.h>
32 #include <string.h>
33 #include <cpl.h>
34 
35 #include "muse_basicproc.h"
36 
37 #include "muse_combine.h"
38 #include "muse_pfits.h"
39 #include "muse_quadrants.h"
40 #include "muse_quality.h"
41 #include "muse_utils.h"
42 #include "muse_data_format_z.h"
43 
44 /*----------------------------------------------------------------------------*
45  * Debugging Macros *
46  * Set these to 1 or higher for (lots of) debugging output *
47  *----------------------------------------------------------------------------*/
48 #define GENERATE_TEST_IMAGES 0 /* generate FITS file(s) to be used for testing */
49 
50 /*---------------------------------------------------------------------------*/
57 /*---------------------------------------------------------------------------*/
58 
61 /*---------------------------------------------------------------------------*/
83 /*---------------------------------------------------------------------------*/
85 muse_basicproc_params_new(cpl_parameterlist *aParameters, const char *aPrefix)
86 {
87  muse_basicproc_params *bpars = cpl_calloc(1, sizeof(muse_basicproc_params));
88  cpl_parameter *param;
89  param = muse_cplparamerterlist_find_prefix(aParameters, aPrefix, "overscan");
90  bpars->overscan = cpl_strdup(cpl_parameter_get_string(param));
91  param = muse_cplparamerterlist_find_prefix(aParameters, aPrefix, "ovscreject");
92  bpars->rejection = cpl_strdup(cpl_parameter_get_string(param));
93  param = muse_cplparamerterlist_find_prefix(aParameters, aPrefix, "ovscsigma");
94  cpl_errorstate state = cpl_errorstate_get();
95  bpars->ovscsigma = cpl_parameter_get_double(param);
96  if (!cpl_errorstate_is_equal(state)) { /* try again as int, may be misidentified */
97  cpl_errorstate_set(state);
98  bpars->ovscsigma = cpl_parameter_get_int(param);
99  }
100  param = muse_cplparamerterlist_find_prefix(aParameters, aPrefix, "ovscignore");
101  bpars->ovscignore = cpl_parameter_get_int(param);
102 
103  if (strstr(aPrefix, "muse_scibasic")) {
104  param = muse_cplparamerterlist_find_prefix(aParameters, aPrefix, "cr");
105  bpars->crmethod = cpl_strdup(cpl_parameter_get_string(param));
106  param = muse_cplparamerterlist_find_prefix(aParameters, aPrefix, "xbox");
107  bpars->dcrxbox = cpl_parameter_get_int(param);
108  param = muse_cplparamerterlist_find_prefix(aParameters, aPrefix, "ybox");
109  bpars->dcrybox = cpl_parameter_get_int(param);
110  param = muse_cplparamerterlist_find_prefix(aParameters, aPrefix, "passes");
111  bpars->dcrpasses = cpl_parameter_get_int(param);
112  param = muse_cplparamerterlist_find_prefix(aParameters, aPrefix, "thres");
113  state = cpl_errorstate_get();
114  bpars->dcrthres = cpl_parameter_get_double(param);
115  if (!cpl_errorstate_is_equal(state)) { /* try again as int */
116  cpl_errorstate_set(state);
117  bpars->dcrthres = cpl_parameter_get_int(param);
118  }
119  } /* if scibasic prefix */
120 
121  /* components keepflat and flatimage are automatically CPL_FALSE and NULL */
122  return bpars;
123 } /* muse_basicproc_params_new() */
124 
125 /*---------------------------------------------------------------------------*/
141 /*---------------------------------------------------------------------------*/
143 muse_basicproc_params_new_from_propertylist(const cpl_propertylist *aHeader)
144 {
145  cpl_ensure(aHeader, CPL_ERROR_NULL_INPUT, NULL);
146 
147  /* create a parameter list from the input header, so *
148  * that we can give it to muse_basicproc_params_new() */
149  cpl_parameterlist *parlist = muse_cplparameterlist_from_propertylist(aHeader, 1);
150  cpl_ensure(parlist, CPL_ERROR_ILLEGAL_INPUT, NULL);
151  const char *recipe = cpl_propertylist_get_string(aHeader, "ESO PRO REC1 ID");
152  char *prefix = cpl_sprintf("muse.%s", recipe);
153  muse_basicproc_params *bpars = muse_basicproc_params_new(parlist, prefix);
154  cpl_parameterlist_delete(parlist);
155  cpl_free(prefix);
156  return bpars;
157 } /* muse_basicproc_params_new_from_propertylist() */
158 
159 /*---------------------------------------------------------------------------*/
167 /*---------------------------------------------------------------------------*/
168 void
170 {
171  if (!aBPars) {
172  return;
173  }
174  cpl_free(aBPars->overscan);
175  cpl_free(aBPars->rejection);
176  cpl_free(aBPars->crmethod);
177  muse_image_delete(aBPars->flatimage);
178  aBPars->flatimage = NULL;
179  cpl_free(aBPars);
180 }
181 
182 /*---------------------------------------------------------------------------*/
201 /*---------------------------------------------------------------------------*/
202 static cpl_error_code
203 muse_basicproc_verify_setup(const muse_image *aImage, const muse_image *aRef)
204 {
205  cpl_ensure_code(aImage && aRef, CPL_ERROR_NULL_INPUT);
206  cpl_ensure_code(aImage->header && aRef->header, CPL_ERROR_NULL_INPUT);
207  /* shortcuts to the headers */
208  cpl_propertylist *him = aImage->header,
209  *href = aRef->header;
210  /* reference image need to have a processed category (PRO.CATG) */
211  const char *fn1 = cpl_propertylist_get_string(him, MUSE_HDR_TMP_FN),
212  *fn2 = cpl_propertylist_get_string(href, MUSE_HDR_TMP_FN),
213  *catg = muse_pfits_get_pro_catg(href);
214  if (!catg) {
215  cpl_msg_error(__func__, "\"%s\" does not contain a category (ESO.PRO.CATG)!",
216  fn2);
217  return CPL_ERROR_ILLEGAL_INPUT;
218  }
219 
220  muse_ins_mode mode1 = muse_pfits_get_mode(him),
221  mode2 = muse_pfits_get_mode(href);
222  const char *modestr1 = muse_pfits_get_insmode(him),
223  *modestr2 = muse_pfits_get_insmode(href),
224  *rawtag = cpl_propertylist_get_string(him, MUSE_HDR_TMP_INTAG);
225  int binx1 = muse_pfits_get_binx(him),
226  biny1 = muse_pfits_get_biny(him),
227  readid1 = muse_pfits_get_read_id(him),
228  binx2 = muse_pfits_get_binx(href),
229  biny2 = muse_pfits_get_biny(href),
230  readid2 = muse_pfits_get_read_id(href);
231  const char *readname1 = muse_pfits_get_read_name(him),
232  *readname2 = muse_pfits_get_read_name(href),
233  *chipname1 = muse_pfits_get_chip_name(him),
234  *chipid1 = muse_pfits_get_chip_id(him),
235  *chipname2 = muse_pfits_get_chip_name(href),
236  *chipid2 = muse_pfits_get_chip_id(href);
237  cpl_boolean chipinfo = chipname1 && chipid1
238  && chipname2 && chipid2;
239  if (!chipinfo) {
240  cpl_msg_warning(__func__, "CHIP information is missing (ESO.DET.CHIP."
241  "{NAME,ID}) from \"%s\" (%s, %s) or \"%s\" (%s, %s)",
242  fn1, chipname1, chipid1, fn2, chipname2, chipid2);
243  }
244  /* Everything should fail for non-matching binning. */
245  if (binx1 != binx2 || biny1 != biny2) {
246  cpl_msg_error(__func__, "Binning of \"%s\" (%dx%d) and \"%s\" (%dx%d) does "
247  "not match", fn1, binx1, biny1, fn2, binx2, biny2);
248  return CPL_ERROR_TYPE_MISMATCH;
249  }
250 
251  /* The pipeline should refuse to work if the bias is not taken in the *
252  * same read-out as the image that is being bias-subtracted. Hence it *
253  * should give an ERROR message when searching for bias files and stop. */
254  if (!strncmp(catg, "MASTER_BIAS", 12)) {
255  if (readid1 != readid2) {
256  cpl_msg_error(__func__, "Read-out mode of \"%s\" (%d: %s) and \"%s\" (%d:"
257  " %s) does not match", fn1, readid1, readname1, fn2,
258  readid2, readname2);
259  return CPL_ERROR_TYPE_MISMATCH;
260  }
261  if (chipinfo && (strcmp(chipname1, chipname2) || strcmp(chipid1, chipid2))) {
262  cpl_msg_error(__func__, "CHIP information (ESO.DET.CHIP.{NAME,ID}) "
263  "does not match for \"%s\" (%s, %s) and \"%s\" (%s, %s)",
264  fn1, chipname1, chipid1, fn2, chipname2, chipid2);
265  return CPL_ERROR_TYPE_MISMATCH;
266  }
267  } /* if ref is bias */
268 
269  /* We probably need similar guards for the instrument mode, so that one *
270  * cannot use a flat-field in WFM-NOAO-N for data taken with WFM-NOAO-E. */
271  if (!strncmp(catg, "MASTER_FLAT", 12)) {
272  if (mode1 != mode2) {
273  if (rawtag && !strncmp(rawtag, MUSE_TAG_ILLUM, strlen(MUSE_TAG_ILLUM) + 1)) {
274  /* ignore mode differences between ILLUM and other calibrations */
275  cpl_msg_debug(__func__, "Instrument modes for \"%s\" (%s, is %s) and \"%s\""
276  " (%s) do not match", fn1, modestr1, rawtag, fn2, modestr2);
277  } else {
278  cpl_msg_error(__func__, "Instrument modes for \"%s\" (%s) and \"%s\" (%s)"
279  " do not match", fn1, modestr1, fn2, modestr2);
280  return CPL_ERROR_TYPE_MISMATCH;
281  } /* else */
282  } /* if modes different */
283  } /* if ref is flat */
284 
285  /* XXX add check to not mix WFM and NFM for illuminated exposures */
286 
287  /* It should output WARNINGs (but continue processing), when other *
288  * calibrations are not in the same read-out mode or if the images *
289  * originate from different chips or chip installation dates. */
290  if (readid1 != readid2) {
291  cpl_msg_warning(__func__, "Read-out mode of \"%s\" (%d: %s) and \"%s\" (%d:"
292  " %s) does not match", fn1, readid1, readname1, fn2,
293  readid2, readname2);
294  }
295  if (chipinfo && (strcmp(chipname1, chipname2) || strcmp(chipid1, chipid2))) {
296  cpl_msg_warning(__func__, "CHIP information (ESO.DET.CHIP.{NAME,ID,DATE}) "
297  "does not match for \"%s\" (%s, %s) and \"%s\" (%s, %s)",
298  fn1, chipname1, chipid1, fn2, chipname2, chipid2);
299  }
300 
301  return CPL_ERROR_NONE;
302 } /* muse_basicproc_verify_setup() */
303 
304 /*---------------------------------------------------------------------------*/
319 /*---------------------------------------------------------------------------*/
320 static cpl_error_code
321 muse_basicproc_overscans_compute_stats(muse_imagelist *aList,
322  muse_basicproc_params *aBPars)
323 {
324  cpl_ensure_code(aList, CPL_ERROR_NULL_INPUT);
325  unsigned int k;
326  for (k = 0; k < aList->size; k++) {
327  muse_image *image = muse_imagelist_get(aList, k);
328  if (muse_quadrants_overscan_stats(image, aBPars ? aBPars->rejection : NULL,
329  aBPars ? aBPars->ovscignore : 0)
330  != CPL_ERROR_NONE) {
331  cpl_msg_warning(__func__, "Could not compute overscan statistics in IFU "
332  "%hhu of exposure %u: %s", muse_utils_get_ifu(image->header),
333  k+1, cpl_error_get_message());
334  } /* if */
335  } /* for k (all images) */
336  return CPL_ERROR_NONE;
337 } /* muse_basicproc_overscans_compute_stats() */
338 
339 /*---------------------------------------------------------------------------*/
352 /*---------------------------------------------------------------------------*/
353 static cpl_error_code
354 muse_basicproc_correct_overscans_vpoly(muse_imagelist *aList,
355  muse_basicproc_params *aBPars)
356 {
357  cpl_ensure_code(aList, CPL_ERROR_NULL_INPUT);
358  cpl_boolean ovscvpoly = aBPars && aBPars->overscan
359  && !strncmp(aBPars->overscan, "vpoly", 5);
360  if (!ovscvpoly) {
361  cpl_msg_debug(__func__, "not vpoly: %s!", aBPars ? aBPars->overscan : "");
362  return CPL_ERROR_NONE;
363  }
364  /* vertical polyfit requested, see if there are more parameters */
365  unsigned char ovscvorder = 5;
366  double frms = 1.01,
367  fchisq = 1.04;
368  char *rest = strchr(aBPars->overscan, ':');
369  if (strlen(aBPars->overscan) > 6 && rest++) { /* try to access info after "vpoly:" */
370  ovscvorder = strtol(rest, &rest, 10);
371  if (strlen(rest++) > 0) { /* ++ to skip over the comma */
372  frms = strtod(rest, &rest);
373  if (strlen(rest++) > 0) {
374  fchisq = strtod(rest, &rest);
375  }
376  }
377  } /* if */
378  cpl_msg_debug(__func__, "vpoly: %s (vorder=%hhu, frms=%f, fchisq=%f, ignore=%u,"
379  " sigma=%.3f)", aBPars->overscan, ovscvorder, frms, fchisq,
380  aBPars->ovscignore, aBPars->ovscsigma);
381 
382  cpl_error_code rc = CPL_ERROR_NONE;
383  unsigned int k;
384  for (k = 0; k < aList->size; k++) {
385  muse_image *image = muse_imagelist_get(aList, k);
387  ovscvorder, aBPars->ovscsigma,
388  frms, fchisq);
389  if (rc != CPL_ERROR_NONE) {
390  unsigned char ifu = muse_utils_get_ifu(image->header);
391  cpl_msg_error(__func__, "Could not correct quadrants levels using vertical"
392  " overscan fit in IFU %hhu: %s", ifu, cpl_error_get_message());
393  } /* if */
394  } /* for k (all images) */
395  return rc;
396 } /* muse_basicproc_correct_overscans_vpoly() */
397 
398 /*---------------------------------------------------------------------------*/
412 /*---------------------------------------------------------------------------*/
413 static cpl_error_code
414 muse_basicproc_trim_images(muse_imagelist *aList)
415 {
416  cpl_ensure_code(aList, CPL_ERROR_NULL_INPUT);
417 
418  unsigned int k;
419  for (k = 0; k < aList->size; k++) {
420  muse_image *image = muse_imagelist_get(aList, k);
421  muse_image *trimmed = muse_quadrants_trim_image(image);
422  cpl_ensure_code(trimmed, cpl_error_get_code());
423 
424  /* setting a new one deletes the old, so no need to free |image| */
425  muse_imagelist_set(aList, trimmed, k);
426  } /* for k (all images) */
427  return muse_imagelist_is_uniform(aList) == 0 ? CPL_ERROR_NONE
428  : CPL_ERROR_ILLEGAL_OUTPUT;
429 } /* muse_basicproc_trim_images() */
430 
431 /*---------------------------------------------------------------------------*/
449 /*---------------------------------------------------------------------------*/
450 static cpl_error_code
451 muse_basicproc_correct_overscans_offset(muse_imagelist *aList,
452  muse_processing *aProcessing,
453  muse_basicproc_params *aBPars)
454 {
455  cpl_ensure_code(aList && aProcessing, CPL_ERROR_NULL_INPUT);
456  /* this only makes sense to do for inputs that are bias */
457  if (!muse_processing_check_intags(aProcessing, MUSE_TAG_BIAS, 5)) {
458  return CPL_ERROR_NONE;
459  }
460  cpl_boolean ovscoffset = aBPars && aBPars->overscan
461  && !strncmp(aBPars->overscan, "offset", 7);
462  if (!ovscoffset) {
463  return CPL_ERROR_NONE; /* no correction necessary */
464  }
465  unsigned char ifu = muse_utils_get_ifu(muse_imagelist_get(aList, 0)->header);
466  cpl_msg_info(__func__, "Running overscan correction using %u %s images in IFU"
467  " %hhu", aList->size, MUSE_TAG_BIAS, ifu);
468  muse_image *ref = muse_imagelist_get(aList, 0);
469  unsigned int k;
470  for (k = 1; k < aList->size; k++) {
472  } /* for k (all images except first) */
473  return CPL_ERROR_NONE;
474 } /* muse_basicproc_correct_overscans_offset() */
475 
476 /*---------------------------------------------------------------------------*/
501 /*---------------------------------------------------------------------------*/
502 static cpl_error_code
503 muse_basicproc_check_overscans(muse_imagelist *aList,
504  muse_processing *aProcessing,
505  muse_basicproc_params *aBPars)
506 {
507  cpl_ensure_code(aList && aProcessing, CPL_ERROR_NULL_INPUT);
508  /* files other than bias are already checked when *
509  * subtracting the bias, so we can skip this check */
510  if (!muse_processing_check_intags(aProcessing, MUSE_TAG_BIAS, 5)) {
511  return CPL_ERROR_NONE;
512  }
513  if (aList->size < 2) { /* doesn't make sense for single image lists */
514  return CPL_ERROR_NONE;
515  }
516  cpl_boolean ovscnone = aBPars && aBPars->overscan
517  && !strncmp(aBPars->overscan, "none", 5);
518  if (!ovscnone) { /* check not needed */
519  return CPL_ERROR_NONE;
520  }
521  double sigma = aBPars ? aBPars->ovscsigma : 1.;
522 
523  /* header of the first image */
524  muse_image *refimage = muse_imagelist_get(aList, 0);
525  cpl_propertylist *refheader = refimage->header;
526  cpl_error_code rc = CPL_ERROR_NONE;
527  unsigned char n, ifu = muse_utils_get_ifu(refheader);
528  for (n = 1; n <= 4; n++) {
529  /* create correct header keyword names */
530  char *keywordmean = cpl_sprintf(MUSE_HDR_OVSC_MEAN, n),
531  *keywordstdev = cpl_sprintf(MUSE_HDR_OVSC_STDEV, n);
532 
533  /* overscan stats for first image in list */
534  const char *reffn = cpl_propertylist_get_string(refheader, MUSE_HDR_TMP_FN);
535  float refmean = cpl_propertylist_get_float(refheader, keywordmean),
536  refstdev = cpl_propertylist_get_float(refheader, keywordstdev),
537  hilimit = refmean + sigma * refstdev,
538  lolimit = refmean - sigma * refstdev;
539  /* variables to create average output values */
540  double summean = refmean,
541  sumstdev = pow(refstdev, 2.);
542 
543  /* compare with the other images */
544  unsigned int k;
545  for (k = 1; k < aList->size; k++) {
546  cpl_propertylist *h = muse_imagelist_get(aList, k)->header;
547  float mean = cpl_propertylist_get_float(h, keywordmean),
548  stdev = cpl_propertylist_get_float(h, keywordstdev);
549  summean += mean;
550  sumstdev += pow(stdev, 2.) + pow(mean - refmean, 2.);
551  const char *fn = cpl_propertylist_get_string(h, MUSE_HDR_TMP_FN);
552  if (mean > hilimit || mean < lolimit) {
553  cpl_msg_error(__func__, "Overscan of IFU %hhu, quadrant %1hhu of image %u [%s] "
554  "(%.3f+/-%.3f) differs from first image [%s] (%.3f+/-%.3f)!",
555  ifu, n, k+1, fn, mean, stdev, reffn, refmean, refstdev);
556  rc = cpl_error_set(__func__, CPL_ERROR_INCOMPATIBLE_INPUT);
557  continue; /* debug output only if there was no error */
558  }
559  cpl_msg_debug(__func__, "Overscan of IFU %hhu, quadrant %1hhu of image %u [%s] "
560  "%.3f+/-%.3f (first image [%s] %.3f+/-%.3f)",
561  ifu, n, k+1, fn, mean, stdev, reffn, refmean, refstdev);
562  } /* for k (all images except first) */
563 
564  /* update values in the 1st header to be the averaged values *
565  * which should be propagated to the final combined frame */
566  summean /= aList->size;
567  sumstdev = sqrt(sumstdev) / aList->size;
568  cpl_msg_debug(__func__, "Averaged overscan values in IFU %hhu, quadrant %1hhu: "
569  "%.3f +/- %.3f (%d images)", ifu, n, summean, sumstdev, aList->size);
570  cpl_propertylist_update_float(refheader, keywordmean, summean);
571  cpl_propertylist_update_float(refheader, keywordstdev, sumstdev);
572 
573  cpl_free(keywordmean);
574  cpl_free(keywordstdev);
575  } /* for n (quadrants) */
576 
577  return rc;
578 } /* muse_basicproc_check_overscans() */
579 
580 /*---------------------------------------------------------------------------*/
599 /*---------------------------------------------------------------------------*/
600 static cpl_error_code
601 muse_basicproc_apply_badpix(muse_imagelist *aList, muse_processing *aProcessing,
602  unsigned char aIFU)
603 {
604  cpl_ensure_code(aList && aProcessing, CPL_ERROR_NULL_INPUT);
605  cpl_frameset *frames = muse_frameset_find(aProcessing->inframes,
606  MUSE_TAG_BADPIX_TABLE, aIFU,
607  CPL_FALSE);
608  if (!frames) { /* should not happen... */
609  return CPL_ERROR_NONE;
610  }
611 
612  /* loop through all frames that were found to have bad pixels for this IFU */
613  cpl_errorstate prestate = cpl_errorstate_get();
614  cpl_size iframe, nframes = cpl_frameset_get_size(frames);
615  for (iframe = 0; iframe < nframes; iframe++) {
616  cpl_frame *frame = cpl_frameset_get_position(frames, iframe);
617  const char *fn = cpl_frame_get_filename(frame);
618  char *extname = cpl_sprintf("CHAN%02hhu", aIFU);
619  cpl_table *table = muse_cpltable_load(fn, extname, muse_badpix_table_def);
620  cpl_free(extname);
621  if (!table) {
622  continue; /* file couldn't be loaded or is faulty */
623  }
624  muse_processing_append_used(aProcessing, frame, CPL_FRAME_GROUP_CALIB, 1);
625 
626  /* valid table found, loop through all entries and mark them in all images */
627  int nrow = cpl_table_get_nrow(table);
628  unsigned int k, nbadpix = 0, /* could transferred bad pixels */
629  nbadpos = 0; /* and entries with bad positions */
630  for (k = 0; k < aList->size; k++) {
631  muse_image *image = muse_imagelist_get(aList, k);
632  /* XXX need to verify the detector/chip properties here, too! */
633  int i;
634  for (i = 0; i < nrow; i++) {
635  int x = cpl_table_get(table, MUSE_BADPIX_X, i, NULL),
636  y = cpl_table_get(table, MUSE_BADPIX_Y, i, NULL);
637 
638  /* get value from table and check that getting *
639  * the value from the DQ image succeeds */
640  uint32_t dq = cpl_table_get(table, MUSE_BADPIX_DQ, i, NULL);
641  cpl_errorstate state = cpl_errorstate_get();
642  int err;
643  uint32_t value = cpl_image_get(image->dq, x, y, &err);
644  if (err != 0 || !cpl_errorstate_is_equal(state)) {
645  cpl_errorstate_set(state); /* swallow the error, nothing was changed */
646  if (k == 0) {
647  nbadpos++;
648  } /* if first image */
649  continue;
650  } /* if error occurred */
651 
652  /* OR the data in the DQ extension with what's in the badpix table */
653  cpl_image_set(image->dq, x, y, dq | value);
654  if (k == 0) {
655  nbadpix++; /* count bad pixels transferred to 1st image */
656  } /* if first image */
657  } /* for i (all table rows) */
658  } /* for k (all images) */
659  cpl_table_delete(table);
660 
661  cpl_msg_debug(__func__, "Applied %u bad pixels from %s \"%s\" in IFU %hhu.",
662  nbadpix, MUSE_TAG_BADPIX_TABLE, fn, aIFU);
663  /* warn, if there were bad entries */
664  if (nbadpos > 0) {
665  cpl_msg_warning(__func__, "\"%s\" contained %u entries outside the CCD in"
666  " IFU %hhu!", fn, nbadpos, aIFU);
667  } /* if bad entries */
668  } /* for iframe (all bad pixel tables found) */
669  cpl_frameset_delete(frames);
670  return cpl_errorstate_is_equal(prestate) ? CPL_ERROR_NONE
671  : cpl_error_get_code();
672 } /* muse_basicproc_apply_badpix() */
673 
674 /*---------------------------------------------------------------------------*/
683 /*---------------------------------------------------------------------------*/
684 static cpl_error_code
685 muse_basicproc_check_saturation(muse_imagelist *aList)
686 {
687  cpl_ensure_code(aList, CPL_ERROR_NULL_INPUT);
688  unsigned int k;
689  for (k = 0; k < aList->size; k++) {
690  muse_image *image = muse_imagelist_get(aList, k);
691  unsigned char ifu = muse_utils_get_ifu(image->header);
692  int nsaturated = muse_quality_set_saturated(image);
693  /* if we have more than 10% of saturated pixels then something is wrong */
694  int npix = cpl_image_get_size_x(image->data)
695  * cpl_image_get_size_y(image->data);
696  if (nsaturated > (0.01 * npix)) {
697  const char *fn = cpl_propertylist_get_string(image->header,
698  MUSE_HDR_TMP_FN);
699  cpl_msg_error(__func__, "Raw exposure %u [%s] is strongly saturated in "
700  "IFU %hhu (%d of %d pixels)!", k+1, fn, ifu, nsaturated,
701  npix);
702  } else if (nsaturated > (0.001 * npix)) {
703  const char *fn = cpl_propertylist_get_string(image->header,
704  MUSE_HDR_TMP_FN);
705  cpl_msg_warning(__func__, "Raw exposure %u [%s] is probably saturated in "
706  "IFU %hhu (%d of %d pixels)!", k+1, fn, ifu, nsaturated,
707  npix);
708  } else {
709  cpl_msg_debug(__func__, "Raw exposure %u in IFU %hhu (%d of %d pixels "
710  "saturated)!", k+1, ifu, nsaturated, npix);
711  }
712  cpl_propertylist_update_int(image->header, MUSE_HDR_TMP_NSAT, nsaturated);
713  } /* for k (all images) */
714  return CPL_ERROR_NONE;
715 } /* muse_basicproc_check_saturation() */
716 
717 /*---------------------------------------------------------------------------*/
730 /*---------------------------------------------------------------------------*/
731 static cpl_error_code
732 muse_basicproc_quadrant_statistics(muse_imagelist *aList,
733  muse_processing *aProcessing)
734 {
735  cpl_ensure_code(aList, CPL_ERROR_NULL_INPUT);
736  /* this only makes sense to do for inputs that are bias */
737  if (!muse_processing_check_intags(aProcessing, MUSE_TAG_BIAS, 5)) {
738  return CPL_ERROR_NONE;
739  }
740  unsigned char ifu = muse_utils_get_ifu(muse_imagelist_get(aList, 0)->header);
741  cpl_msg_info(__func__, "Computing per-quadrant medians for %u exposures in "
742  "IFU %hhu", aList->size, ifu);
743  unsigned int k;
744  for (k = 0; k < aList->size; k++) {
745  muse_image *image = muse_imagelist_get(aList, k);
746  unsigned char n;
747  for (n = 1; n <= 4; n++) {
748  cpl_size *w = muse_quadrants_get_window(muse_imagelist_get(aList, k), n);
749  float median = cpl_image_get_median_window(image->data, w[0], w[2],
750  w[1], w[3]);
751  cpl_free(w);
752  char *kw = cpl_sprintf(MUSE_HDR_TMP_QUADnMED, n);
753  cpl_propertylist_append_float(image->header, kw, median);
754  cpl_free(kw);
755  } /* for n (quadrants) */
756  } /* for k (all images) */
757  return CPL_ERROR_NONE;
758 } /* muse_basicproc_quadrant_statistics() */
759 
760 /*---------------------------------------------------------------------------*/
782 /*---------------------------------------------------------------------------*/
783 static cpl_error_code
784 muse_basicproc_apply_bias(muse_imagelist *aList, muse_processing *aProcessing,
785  unsigned char aIFU, muse_basicproc_params *aBPars)
786 {
787  cpl_ensure_code(aList && aProcessing, CPL_ERROR_NULL_INPUT);
788  cpl_frame *biasframe = muse_frameset_find_master(aProcessing->inframes,
789  MUSE_TAG_MASTER_BIAS, aIFU);
790  cpl_errorstate prestate = cpl_errorstate_get();
791  if (!biasframe) return CPL_ERROR_NONE;
792  cpl_errorstate_set(prestate);
793  const char *biasname = cpl_frame_get_filename(biasframe);
794  muse_image *biasimage = muse_image_load(biasname);
795  if (!biasimage) {
796  /* remember error message for a while, but reset the state, *
797  * before trying to load from channel extensions */
798  char *errmsg = cpl_strdup(cpl_error_get_message());
799  cpl_errorstate_set(prestate);
800  biasimage = muse_image_load_from_extensions(biasname, aIFU);
801  if (!biasimage) {
802  /* now display both the older and the new error messages, *
803  * so that they cannot be swallowed by parallelization */
804  cpl_msg_error(__func__, "%s", errmsg);
805  cpl_msg_error(__func__, "%s", cpl_error_get_message());
806  cpl_free(errmsg);
807  cpl_frame_delete(biasframe);
808  return cpl_error_get_code();
809  } /* if 2nd load failure */
810  cpl_free(errmsg);
811  cpl_msg_info(__func__, "Bias correction in IFU %hhu using \"%s[CHAN%02hhu."
812  "DATA]\"", aIFU, biasname, aIFU);
813  } else {
814  cpl_msg_info(__func__, "Bias correction in IFU %hhu using \"%s[DATA]\"",
815  aIFU, biasname);
816  }
817  /* add temporary input filename for diagnostics */
818  cpl_propertylist_append_string(biasimage->header, MUSE_HDR_TMP_FN, biasname);
819 
820  /* cross-check with the parameters of the master bias */
821  muse_basicproc_params *mbpars
823  cpl_boolean parmatch = mbpars && aBPars; /* both sets exist */
824  if (parmatch) {
825  parmatch = parmatch && mbpars->overscan && aBPars->overscan
826  && !strncmp(aBPars->overscan, mbpars->overscan,
827  CPL_MIN(strlen(aBPars->overscan), strlen(mbpars->overscan) + 1));
828  parmatch = parmatch && mbpars->rejection && aBPars->rejection
829  && !strncmp(aBPars->rejection, mbpars->rejection,
830  CPL_MIN(strlen(aBPars->rejection), strlen(mbpars->rejection) + 1));
831  parmatch = parmatch && fabs(aBPars->ovscsigma - mbpars->ovscsigma)
832  < 10 * DBL_EPSILON;
833  parmatch = parmatch && aBPars->ovscignore == mbpars->ovscignore;
834  }
835  if (!parmatch) {
836  cpl_msg_warning(__func__, "overscan parameters differ between %s and recipe"
837  " parameters!", MUSE_TAG_MASTER_BIAS);
838  } else {
839  cpl_msg_debug(__func__, "overscan parameters between %s and given "
840  "parameters nicely match", MUSE_TAG_MASTER_BIAS);
841  }
843 
844  cpl_boolean ovscoffset = aBPars && aBPars->overscan
845  && !strncmp(aBPars->overscan, "offset", 7),
846  ovscvpoly = aBPars && aBPars->overscan
847  && !strncmp(aBPars->overscan, "vpoly", 5); /* up to the : */
848  muse_processing_append_used(aProcessing, biasframe, CPL_FRAME_GROUP_CALIB, 0);
849  cpl_error_code rc = CPL_ERROR_NONE;
850  unsigned int k;
851  for (k = 0; k < aList->size && rc == CPL_ERROR_NONE; k++) {
852  muse_image *image = muse_imagelist_get(aList, k);
853  rc = muse_basicproc_verify_setup(image, biasimage);
854  if (rc != CPL_ERROR_NONE) {
855  break;
856  }
857  rc = muse_image_variance_create(image, biasimage);
858  if (ovscoffset) {
859  muse_quadrants_overscan_correct(image, biasimage);
860  } else if (ovscvpoly) {
861  /* create error message and code, if the bias was not *
862  * vpoly-handled, i.e. still has the original bias level *
863  * of ~1000 adu or > 100xsigma above ~zero */
864  cpl_boolean good = muse_quadrants_overscan_check(image, biasimage, 100.);
865  if (!good) {
866  cpl_msg_error(__func__, "Very different overscan levels found between "
867  "%s and raw exposure %u in IFU %hhu: incompatible overscan"
868  " parameters?", MUSE_TAG_MASTER_BIAS, k + 1, aIFU);
869  rc = cpl_error_set(__func__, CPL_ERROR_INCOMPATIBLE_INPUT);
870  break;
871  } /* if not good */
872  } else {
873  /* just warn above indicated sigma level, ignore failure */
874  muse_quadrants_overscan_check(image, biasimage,
875  aBPars ? aBPars->ovscsigma : 1.);
876  }
877  rc = muse_image_subtract(image, biasimage);
878 #if GENERATE_TEST_IMAGES
879  /* if we need to generate images for automated tests, here is a good *
880  * place to write them out, as they are trimmed and bias corrected */
881  if (k == 0) {
882  muse_image_save(image, "trimmed_bias_sub.fits");
883  }
884 #endif
885  } /* for k (all images) */
886  muse_image_delete(biasimage);
887  return rc;
888 } /* muse_basicproc_apply_bias() */
889 
890 /*---------------------------------------------------------------------------*/
909 /*---------------------------------------------------------------------------*/
910 static cpl_error_code
911 muse_basicproc_check_gain(muse_imagelist *aList, muse_processing *aProcessing,
912  unsigned char aIFU)
913 {
914  cpl_ensure_code(aList && aProcessing, CPL_ERROR_NULL_INPUT);
915  /* this only makes sense to do for input flat-fields */
916  if (!muse_processing_check_intags(aProcessing, MUSE_TAG_FLAT, 5)) {
917  return CPL_ERROR_NONE;
918  }
919  cpl_ensure_code(muse_imagelist_get_size(aList) >= 2,
920  CPL_ERROR_INCOMPATIBLE_INPUT);
921 
922  cpl_frame *fbias = muse_frameset_find_master(aProcessing->inframes,
923  MUSE_TAG_MASTER_BIAS, aIFU);
924  if (!fbias) {
925  /* it's OK if there is no bias frame, probably we are not in muse_bias */
926  return CPL_ERROR_NONE;
927  }
928  const char *biasname = cpl_frame_get_filename(fbias);
929  cpl_propertylist *hbias = cpl_propertylist_load(biasname, 0);
930  if (!cpl_propertylist_has(hbias, QC_BIAS_MASTER_RON)) {
931  cpl_propertylist_delete(hbias);
932  /* try to load from channel extension */
933  char *extname = cpl_sprintf("CHAN%02hhu.%s", aIFU, EXTNAME_DATA);
934  int extension = cpl_fits_find_extension(biasname, extname);
935  hbias = cpl_propertylist_load(biasname, extension);
936  cpl_free(extname);
937  }
938  cpl_frame_delete(fbias);
939  cpl_image *f1 = muse_imagelist_get(aList, 0)->data,
940  *f2 = muse_imagelist_get(aList, 1)->data;
941  cpl_propertylist *header = muse_imagelist_get(aList, 0)->header;
942  cpl_image *diff = cpl_image_subtract_create(f1, f2);
943 
944  unsigned char n;
945  for (n = 1; n <= 4; n++) {
946  cpl_size *w = muse_quadrants_get_window(muse_imagelist_get(aList, 0), n);
947  double m1 = cpl_image_get_mean_window(f1, w[0], w[2], w[1], w[3]),
948  m2 = cpl_image_get_mean_window(f2, w[0], w[2], w[1], w[3]),
949  sf = cpl_image_get_stdev_window(diff, w[0], w[2], w[1], w[3]);
950  char *keyword = cpl_sprintf(QC_BIAS_MASTERn_PREFIX" MEAN", n);
951  float mb = cpl_propertylist_get_float(hbias, keyword);
952  cpl_free(keyword);
953  keyword = cpl_sprintf(QC_BIAS_MASTER_RON, n);
954  /* the RON formula taken from Howell inverted to give sigma(b1-b2) */
955  double gainheader = muse_pfits_get_gain(header, n),
956  sb = cpl_propertylist_get_float(hbias, keyword) * sqrt(2.)
957  / gainheader, /* gain in count/adu */
958  /* the gain formula taken from Howell: */
959  gain = (m1 + m2 - 2*mb) / (sf*sf - sb*sb),
960  dgain = 1. - gain / gainheader;
961  /* output warning for difference larger than 20%, info message otherwise */
962  if (dgain > 0.2) {
963  cpl_msg_warning(__func__, "IFU %hhu, quadrant %1hhu: estimated gain %.3f "
964  "count/adu but header gives %.3f!", aIFU, n, gain,
965  gainheader);
966  } else {
967  cpl_msg_info(__func__, "IFU %hhu, quadrant %1hhu: estimated gain %.3f "
968  "count/adu (header %.3f, delta %.3f)", aIFU, n, gain,
969  gainheader, dgain);
970  }
971  cpl_free(keyword);
972  cpl_free(w);
973  } /* for n (quadrants) */
974 
975  cpl_image_delete(diff);
976  cpl_propertylist_delete(hbias);
977 
978  return CPL_ERROR_NONE;
979 } /* muse_basicproc_check_gain() */
980 
981 /*---------------------------------------------------------------------------*/
1000 /*---------------------------------------------------------------------------*/
1001 static cpl_error_code
1002 muse_basicproc_gain_override(muse_imagelist *aList,
1003  muse_processing *aProcessing, unsigned char aIFU)
1004 {
1005  cpl_ensure_code(aList && aProcessing, CPL_ERROR_NULL_INPUT);
1006 
1007  if (muse_processing_check_intags(aProcessing, MUSE_TAG_LINEARITY_BIAS, 16) ||
1008  muse_processing_check_intags(aProcessing, MUSE_TAG_LINEARITY_FLAT, 15))
1009  {
1010  return CPL_ERROR_NONE;
1011  }
1012 
1013  cpl_frame *fnonlingain = muse_frameset_find_master(aProcessing->inframes,
1014  MUSE_TAG_NONLINGAIN, aIFU);
1015  if (!fnonlingain) {
1016  /* it's OK if there is no nonlinearity frame, it's optional... */
1017  return CPL_ERROR_NONE;
1018  }
1019  if (getenv("MUSE_BASICPROC_SKIP_GAIN_OVERRIDE") &&
1020  atoi(getenv("MUSE_BASICPROC_SKIP_GAIN_OVERRIDE")) > 0) {
1021  cpl_msg_info(__func__, "Skipping gain override, although %s is given",
1022  MUSE_TAG_NONLINGAIN);
1023  return CPL_ERROR_NONE;
1024  }
1025  cpl_errorstate state = cpl_errorstate_get();
1026  const char *fn = cpl_frame_get_filename(fnonlingain);
1027  /* immediately try to load from channel extension */
1028  char *extname = cpl_sprintf("CHAN%02hhu", aIFU);
1029  int extension = cpl_fits_find_extension(fn, extname);
1030  cpl_propertylist *header = cpl_propertylist_load(fn, extension);
1031  cpl_msg_info(__func__, "Overriding gain in IFU %hhu using \"%s[%s]\"",
1032  aIFU, fn, extname);
1033  cpl_free(extname);
1034  muse_processing_append_used(aProcessing, fnonlingain, CPL_FRAME_GROUP_CALIB, 0);
1035 
1036  unsigned int k;
1037  for (k = 0; k < aList->size; k++) {
1038  muse_image *image = muse_imagelist_get(aList, k);
1039  /* XXX need to verify the detector/chip properties here, too! */
1040  unsigned char n;
1041  for (n = 1; n <= 4; n++) {
1042  /* transfer the GAIN of this quadrant into the image */
1043  double gain = muse_pfits_get_gain(header, n);
1044  char *kw = cpl_sprintf("ESO DET OUT%d GAIN", n);
1045  cpl_propertylist_update_double(image->header, kw, gain);
1046  cpl_free(kw);
1047  } /* for n (quadrants) */
1048  } /* for k (all images) */
1049  cpl_propertylist_delete(header);
1050  return cpl_errorstate_is_equal(state) ? CPL_ERROR_NONE : cpl_error_get_code();
1051 } /* muse_basicproc_gain_override() */
1052 
1053 /*---------------------------------------------------------------------------*/
1067 /*---------------------------------------------------------------------------*/
1068 static cpl_error_code
1069 muse_basicproc_adu_to_count(muse_imagelist *aList, muse_processing *aProcessing)
1070 {
1071  cpl_ensure_code(aList && aProcessing, CPL_ERROR_NULL_INPUT);
1072  /* this only makes sense to do for inputs that are not bias, and not *
1073  * inputs used to measure gain and linearity */
1074  if (muse_processing_check_intags(aProcessing, MUSE_TAG_BIAS, 5) ||
1075  muse_processing_check_intags(aProcessing, MUSE_TAG_LINEARITY_BIAS, 16) ||
1076  muse_processing_check_intags(aProcessing, MUSE_TAG_LINEARITY_FLAT, 15)) {
1077  return CPL_ERROR_NONE;
1078  }
1079 
1080  unsigned char ifu = muse_utils_get_ifu(muse_imagelist_get(aList, 0)->header);
1081  cpl_msg_info(__func__, "Converting %u exposures from adu to count (= electron)"
1082  " units in IFU %hhu", aList->size, ifu);
1083  cpl_error_code rc = CPL_ERROR_NONE;
1084  unsigned int k;
1085  for (k = 0; k < aList->size; k++) {
1087  } /* for k (all images) */
1088  return rc;
1089 } /* muse_basicproc_adu_to_count() */
1090 
1091 /*---------------------------------------------------------------------------*/
1113 /*---------------------------------------------------------------------------*/
1114 static cpl_error_code
1115 muse_basicproc_corr_nonlinearity(muse_imagelist *aList,
1116  muse_processing *aProcessing,
1117  unsigned char aIFU)
1118 {
1119  cpl_ensure_code(aList && aProcessing, CPL_ERROR_NULL_INPUT);
1120  /* this only makes sense to do for inputs that are illuminated and *
1121  * inputs which are not used to measure gain and linearity */
1122  if (muse_processing_check_intags(aProcessing, MUSE_TAG_BIAS, 5) ||
1123  muse_processing_check_intags(aProcessing, MUSE_TAG_DARK, 5) ||
1124  muse_processing_check_intags(aProcessing, MUSE_TAG_LINEARITY_BIAS, 16) ||
1125  muse_processing_check_intags(aProcessing, MUSE_TAG_LINEARITY_FLAT, 15)) {
1126  return CPL_ERROR_NONE;
1127  }
1128 
1129  cpl_frame *fnonlingain = muse_frameset_find_master(aProcessing->inframes,
1130  MUSE_TAG_NONLINGAIN, aIFU);
1131  if (!fnonlingain) {
1132  /* it's OK if there is no nonlinearity frame, it's optional... */
1133  return CPL_ERROR_NONE;
1134  }
1135  if (getenv("MUSE_BASICPROC_SKIP_NONLIN_CORR") &&
1136  atoi(getenv("MUSE_BASICPROC_SKIP_NONLIN_CORR")) > 0) {
1137  cpl_msg_debug(__func__, "Skipping nonlinearity correction, although %s is "
1138  "given", MUSE_TAG_NONLINGAIN);
1139  return CPL_ERROR_NONE;
1140  }
1141  const char *fn = cpl_frame_get_filename(fnonlingain);
1142  /* immediately try to load from channel extension */
1143  char *extname = cpl_sprintf("CHAN%02hhu", aIFU);
1144  int extension = cpl_fits_find_extension(fn, extname);
1145  cpl_propertylist *header = cpl_propertylist_load(fn, extension);
1146  cpl_msg_info(__func__, "Correcting nonlinearity in IFU %hhu using \"%s[%s]\"",
1147  aIFU, fn, extname);
1148  cpl_free(extname);
1149  muse_processing_append_used(aProcessing, fnonlingain, CPL_FRAME_GROUP_CALIB, 0);
1150 
1151  cpl_error_code rc = CPL_ERROR_NONE;
1152  unsigned int k;
1153  for (k = 0; k < aList->size; k++) {
1154  muse_image *image = muse_imagelist_get(aList, k);
1155  int nx = cpl_image_get_size_x(image->data);
1156  float *data = cpl_image_get_data_float(image->data),
1157  *stat = cpl_image_get_data_float(image->stat);
1158 
1159  /* XXX need to verify the detector/chip properties here, too! */
1160 
1161  unsigned char n;
1162  for (n = 1; n <= 4; n++) {
1163  /* create the 1D nonlinearity polynomial for this quadrant, then read *
1164  * all the parameters from the header set the polynomial up accordingly */
1165  cpl_polynomial *poly = cpl_polynomial_new(1);
1166  char *kw = cpl_sprintf(MUSE_HDR_NONLINn_ORDER, n);
1167  unsigned char o, order = cpl_propertylist_get_int(header, kw);
1168  cpl_free(kw);
1169  for (o = 0; o <= order; o++) {
1170  kw = cpl_sprintf(MUSE_HDR_NONLINn_COEFFo, n, o);
1171  cpl_size pows = o;
1172  cpl_polynomial_set_coeff(poly, &pows,
1173  cpl_propertylist_get_double(header, kw));
1174  cpl_free(kw);
1175  } /* for i (polynomial orders) */
1176  /* compute linear extrapolations */
1177  kw = cpl_sprintf(MUSE_HDR_NONLINn_LLO, n);
1178  double lolim = cpl_propertylist_get_double(header, kw);
1179  cpl_free(kw);
1180  kw = cpl_sprintf(MUSE_HDR_NONLINn_LHI, n);
1181  double hilim = cpl_propertylist_get_double(header, kw);
1182  cpl_free(kw);
1183  /* coefficients for linear extrapolation beyond low and high limits */
1184  double p1lo, p0lo = cpl_polynomial_eval_1d(poly, lolim, &p1lo),
1185  p1hi, p0hi = cpl_polynomial_eval_1d(poly, hilim, &p1hi);
1186  /* convert limits from log10(adu) to adu */
1187  lolim = pow(10, lolim);
1188  hilim = pow(10, hilim);
1189 #if 0
1190  double values[] = { 1., 20., 200., 2000., 20000., 65000.,
1191  lolim, hilim, -1. };
1192  int idx = 0;
1193  while (values[idx] > 0) {
1194  double v = values[idx],
1195  logv = log10(v);
1196  cpl_msg_debug(__func__, "%f adu -> %f log10(adu) ==> poly = %f ==> x %f",
1197  v, logv, cpl_polynomial_eval_1d(poly, logv, NULL),
1198  1. / (1. + cpl_polynomial_eval_1d(poly, logv, NULL)));
1199  idx++;
1200  } /* while */
1201  cpl_polynomial_dump(poly, stdout);
1202  fflush(stdout);
1203  cpl_msg_debug(__func__, "beyond low limit (%f adu): %f + (%f) * (log10(adu) - %f)",
1204  lolim, p0lo, p1lo, log10(lolim));
1205  cpl_msg_debug(__func__, "beyond high limit (%f adu): %f + (%f) * (log10(adu) - %f)",
1206  hilim, p0hi, p1hi, log10(hilim));
1207 #endif
1208 
1209  /* now loop through the full quadrant and correct the scaling */
1210  cpl_size *w = muse_quadrants_get_window(image, n);
1211  int i;
1212  for (i = w[0] - 1; i < w[1]; i++) {
1213  int j;
1214  for (j = w[2] - 1; j < w[3]; j++) {
1215  if (data[i + j*nx] <= 0) {
1216  continue; /* don't do anything for non-positive datapoints */
1217  }
1218  /* compute the percent-level deviation (applying to log10(adu)) */
1219  double pcor;
1220  if (data[i + j*nx] < lolim) {
1221  pcor = p0lo + p1lo * (log10(data[i + j*nx]) - log10(lolim));
1222  } else if (data[i + j*nx] > hilim) {
1223  pcor = p0hi + p1hi * (log10(data[i + j*nx]) - log10(hilim));
1224  } else {
1225  pcor = cpl_polynomial_eval_1d(poly, log10(data[i + j*nx]), NULL);
1226  }
1227  /* then derive the multiplicative correction factor */
1228  double fcor = 1. / (1. + pcor);
1229  data[i + j*nx] *= fcor;
1230  stat[i + j*nx] *= fcor*fcor;
1231  } /* for j (vertical pixels) */
1232  } /* for i (horizontal pixels) */
1233  cpl_free(w);
1234  cpl_polynomial_delete(poly);
1235  } /* for n (quadrants) */
1236  } /* for k (all images) */
1237  cpl_propertylist_delete(header);
1238 
1239  return rc;
1240 } /* muse_basicproc_corr_nonlinearity() */
1241 
1242 /*---------------------------------------------------------------------------*/
1258 /*---------------------------------------------------------------------------*/
1259 static cpl_error_code
1260 muse_basicproc_apply_dark(muse_imagelist *aList, muse_processing *aProcessing,
1261  unsigned char aIFU)
1262 {
1263  cpl_ensure_code(aList && aProcessing, CPL_ERROR_NULL_INPUT);
1264 
1265  cpl_frame *darkframe = muse_frameset_find_master(aProcessing->inframes,
1266  MUSE_TAG_MASTER_DARK, aIFU);
1267  cpl_errorstate prestate = cpl_errorstate_get();
1268  if (!darkframe) return CPL_ERROR_NONE;
1269  cpl_errorstate_set(prestate);
1270  const char *darkname = cpl_frame_get_filename(darkframe);
1271  muse_image *darkimage = muse_image_load(darkname);
1272  if (!darkimage) {
1273  char *errmsg = cpl_strdup(cpl_error_get_message());
1274  cpl_errorstate_set(prestate);
1275  darkimage = muse_image_load_from_extensions(darkname, aIFU);
1276  if (!darkimage) {
1277  cpl_msg_error(__func__, "%s", errmsg);
1278  cpl_msg_error(__func__, "%s", cpl_error_get_message());
1279  cpl_free(errmsg);
1280  cpl_frame_delete(darkframe);
1281  return cpl_error_get_code();
1282  } /* if 2nd load failure */
1283  cpl_free(errmsg);
1284  cpl_msg_info(__func__, "Dark correction in IFU %hhu using \"%s[CHAN%02hhu."
1285  "DATA]\"", aIFU, darkname, aIFU);
1286  } else {
1287  cpl_msg_info(__func__, "Dark correction in IFU %hhu using \"%s[DATA]\"",
1288  aIFU, darkname);
1289  }
1290  cpl_propertylist_append_string(darkimage->header, MUSE_HDR_TMP_FN, darkname);
1291 
1292  muse_processing_append_used(aProcessing, darkframe, CPL_FRAME_GROUP_CALIB, 0);
1293  cpl_error_code rc = CPL_ERROR_NONE;
1294  unsigned int k;
1295  for (k = 0; k < aList->size; k++) {
1296  muse_image *image = muse_imagelist_get(aList, k);
1297  rc = muse_basicproc_verify_setup(image, darkimage);
1298  if (rc != CPL_ERROR_NONE) {
1299  break;
1300  }
1301 
1302  /* duplicate the dark, because the scaling will destroy its normalization */
1303  muse_image *dark = muse_image_duplicate(darkimage);
1304  /* scale and subtract the dark by comparing exposure time *
1305  * of the dark and the other image */
1306  double scale = muse_pfits_get_exptime(image->header);
1307  if (muse_pfits_get_exptime(dark->header) > 0) {
1308  scale /= muse_pfits_get_exptime(dark->header);
1309  }
1310  rc = muse_image_scale(dark, scale);
1311  rc = muse_image_subtract(image, dark);
1312 
1313  muse_image_delete(dark);
1314  } /* for k (all images) */
1315  muse_image_delete(darkimage);
1316 
1317  return rc;
1318 } /* muse_basicproc_apply_dark() */
1319 
1320 /*---------------------------------------------------------------------------*/
1335 /*---------------------------------------------------------------------------*/
1336 static cpl_error_code
1337 muse_basicproc_apply_cr(muse_imagelist *aList,
1338  muse_basicproc_params *aBPars)
1339 {
1340  cpl_ensure_code(aList, CPL_ERROR_NULL_INPUT);
1341 
1342  /* if we didn't get cr parameters we were not supposed *
1343  * to find cosmic rays, so just return without error */
1344  cpl_boolean isdcr = aBPars && aBPars->crmethod
1345  && !strncmp(aBPars->crmethod, "dcr", 4);
1346  if (!isdcr) {
1347  return CPL_ERROR_NONE;
1348  }
1349 
1350  cpl_error_code rc = CPL_ERROR_NONE;
1351  unsigned int k;
1352  for (k = 0; k < aList->size; k++) {
1353  muse_image *image = muse_imagelist_get(aList, k);
1354  unsigned char ifu = muse_utils_get_ifu(image->header);
1355  int ncr = 0;
1356  ncr = muse_cosmics_dcr(image, aBPars->dcrxbox, aBPars->dcrybox,
1357  aBPars->dcrpasses, aBPars->dcrthres);
1358  if (ncr <= 0) {
1359  cpl_msg_error(__func__, "Cosmic ray rejection using DCR in IFU %hhu failed"
1360  " for image %u (ncr = %d): %s", ifu, k+1, ncr,
1361  cpl_error_get_message());
1362  rc = cpl_error_get_code();
1363  } else {
1364  cpl_msg_info(__func__, "Cosmic ray rejection using DCR in IFU %hhu found "
1365  "%d affected pixels in image %u", ifu, ncr, k+1);
1366  }
1367  } /* for k (all images) */
1368 
1369  return rc;
1370 } /* muse_basicproc_apply_cr() */
1371 
1372 /*---------------------------------------------------------------------------*/
1387 /*---------------------------------------------------------------------------*/
1388 static cpl_error_code
1389 muse_basicproc_apply_flat(muse_imagelist *aList, muse_processing *aProcessing,
1390  muse_basicproc_params *aBPars, unsigned char aIFU)
1391 {
1392  cpl_ensure_code(aList && aProcessing, CPL_ERROR_NULL_INPUT);
1393 
1394  cpl_frame *flatframe = muse_frameset_find_master(aProcessing->inframes,
1395  MUSE_TAG_MASTER_FLAT, aIFU);
1396  cpl_errorstate prestate = cpl_errorstate_get();
1397  if (!flatframe) return CPL_ERROR_NONE;
1398  cpl_errorstate_set(prestate);
1399  const char *flatname = cpl_frame_get_filename(flatframe);
1400  prestate = cpl_errorstate_get();
1401  muse_image *flatimage = muse_image_load(flatname);
1402  if (!flatimage) {
1403  char *errmsg = cpl_strdup(cpl_error_get_message());
1404  cpl_errorstate_set(prestate);
1405  flatimage = muse_image_load_from_extensions(flatname, aIFU);
1406  if (!flatimage) {
1407  cpl_msg_error(__func__, "%s", errmsg);
1408  cpl_msg_error(__func__, "%s", cpl_error_get_message());
1409  cpl_free(errmsg);
1410  cpl_frame_delete(flatframe);
1411  return cpl_error_get_code();
1412  } /* if 2nd load failure */
1413  cpl_free(errmsg);
1414  cpl_msg_info(__func__, "Flat-field correction in IFU %hhu using \"%s[CHAN%02hhu."
1415  "DATA]\"", aIFU, flatname, aIFU);
1416  } else {
1417  cpl_msg_info(__func__, "Flat-field correction in IFU %hhu using \"%s[DATA]\"",
1418  aIFU, flatname);
1419  }
1420  /* copy QC parameter containing integrated flux */
1421  double fflux = cpl_propertylist_get_double(flatimage->header,
1422  QC_FLAT_MASTER_INTFLUX);
1423  cpl_propertylist_append_string(flatimage->header, MUSE_HDR_TMP_FN, flatname);
1424  muse_processing_append_used(aProcessing, flatframe, CPL_FRAME_GROUP_CALIB, 0);
1425 
1426  cpl_error_code rc = CPL_ERROR_NONE;
1427  unsigned int k;
1428  for (k = 0; k < aList->size; k++) {
1429  cpl_msg_debug(__func__, "Flat-field division in IFU %hhu of image %u of %u",
1430  aIFU, k+1, aList->size);
1431  muse_image *image = muse_imagelist_get(aList, k);
1432  rc = muse_basicproc_verify_setup(image, flatimage);
1433  if (rc != CPL_ERROR_NONE) {
1434  break;
1435  }
1436  rc = muse_image_divide(image, flatimage);
1437  cpl_propertylist_update_double(image->header, MUSE_HDR_FLAT_FLUX_LAMP, fflux);
1438  } /* for k (all images) */
1439 
1440  if (aBPars && aBPars->keepflat) {
1441  aBPars->flatimage = flatimage;
1442  } else {
1443  muse_image_delete(flatimage);
1444  }
1445  return rc;
1446 } /* muse_basicproc_apply_flat() */
1447 
1448 /*---------------------------------------------------------------------------*/
1469 /*---------------------------------------------------------------------------*/
1470 static muse_imagelist *
1471 muse_basicproc_load_raw(muse_processing *aProcessing, unsigned char aIFU)
1472 {
1473  cpl_ensure(aProcessing, CPL_ERROR_NULL_INPUT, NULL);
1474 
1475  /* rawframes contains the list of input files that we want to load */
1476  cpl_frameset *rawframes = muse_frameset_check_raw(aProcessing->inframes,
1477  aProcessing->intags, aIFU);
1478  cpl_ensure(rawframes, CPL_ERROR_NULL_INPUT, NULL);
1479 
1480  muse_imagelist *images = muse_imagelist_new();
1481  unsigned int k = 0;
1482  cpl_size iframe, nframes = cpl_frameset_get_size(rawframes);
1483  for (iframe = 0; iframe < nframes; iframe++) {
1484  cpl_frame *frame = cpl_frameset_get_position(rawframes, iframe);
1485  const char *fileName = cpl_frame_get_filename(frame);
1486  int extension = muse_utils_get_extension_for_ifu(fileName, aIFU);
1487  if (extension == -1) {
1488  cpl_msg_error(__func__, "\"%s\" does not contain data from IFU %hhu",
1489  fileName, aIFU);
1490  break;
1491  }
1492  muse_image *raw = muse_image_load_from_raw(fileName, extension);
1493  if (!raw) {
1494  continue;
1495  }
1496  /* add temporary input filename for diagnostics */
1497  cpl_propertylist_append_string(raw->header, MUSE_HDR_TMP_FN, fileName);
1498  /* add temporary input tag, for use when deriving the output tag */
1499  cpl_propertylist_append_string(raw->header, MUSE_HDR_TMP_INTAG,
1500  cpl_frame_get_tag(frame));
1501 
1502  muse_imagelist_set(images, raw, k);
1503  muse_processing_append_used(aProcessing, frame, CPL_FRAME_GROUP_RAW, 1);
1504  k++;
1505  } /* for frame (all rawframes) */
1506  cpl_frameset_delete(rawframes);
1507 
1508  /* some checks if we ended up with a valid imagelist */
1509  if (!images->size) {
1510  if ((int)cpl_error_get_code() != MUSE_ERROR_CHIP_NOT_LIVE) {
1511  cpl_error_set(__func__, CPL_ERROR_ILLEGAL_OUTPUT);
1512  cpl_msg_error(__func__, "No raw images loaded for IFU %hhu", aIFU);
1513  }
1514  muse_imagelist_delete(images); /* still free the structure */
1515  return NULL;
1516  }
1517  if (muse_imagelist_is_uniform(images) != 0) {
1518  cpl_error_set(__func__, CPL_ERROR_ILLEGAL_OUTPUT);
1519  cpl_msg_error(__func__, "Non-uniform imagelist for IFU %hhu", aIFU);
1520  muse_imagelist_delete(images);
1521  return NULL;
1522  }
1523 
1524  return images;
1525 } /* muse_basicproc_load_raw() */
1526 
1527 /*---------------------------------------------------------------------------*/
1570 /*---------------------------------------------------------------------------*/
1572 muse_basicproc_load(muse_processing *aProcessing, unsigned char aIFU,
1573  muse_basicproc_params *aBPars)
1574 {
1575  if (muse_processing_check_input(aProcessing, aIFU) != CPL_ERROR_NONE) {
1576  return NULL;
1577  }
1578  muse_imagelist *images = muse_basicproc_load_raw(aProcessing, aIFU);
1579  if (!images) {
1580  return NULL;
1581  }
1582  cpl_errorstate prestate = cpl_errorstate_get();
1583  if (muse_basicproc_apply_badpix(images, aProcessing, aIFU) != CPL_ERROR_NONE) {
1584  cpl_msg_warning(__func__, "Applying bad pixel table to IFU %hhu failed: %s",
1585  aIFU, cpl_error_get_message());
1586  cpl_errorstate_set(prestate); /* this is not critical, continue */
1587  }
1588  if (muse_basicproc_check_saturation(images) != CPL_ERROR_NONE) {
1589  muse_imagelist_delete(images);
1590  return NULL;
1591  }
1592  muse_basicproc_quadrant_statistics(images, aProcessing); /* non-fatal */
1593  if (muse_basicproc_overscans_compute_stats(images, aBPars) /* incl. CR rejection */
1594  != CPL_ERROR_NONE) {
1595  muse_imagelist_delete(images); /* this failure should be fatal */
1596  return NULL;
1597  }
1598  if (muse_basicproc_correct_overscans_vpoly(images, aBPars)
1599  != CPL_ERROR_NONE) {
1600  muse_imagelist_delete(images); /* this failures should be fatal! */
1601  return NULL;
1602  }
1603  if (muse_basicproc_trim_images(images) != CPL_ERROR_NONE) {
1604  muse_imagelist_delete(images);
1605  return NULL;
1606  }
1607  if (muse_basicproc_correct_overscans_offset(images, aProcessing, aBPars)
1608  != CPL_ERROR_NONE) { /* for BIAS files only */
1609  cpl_msg_warning(__func__, "Bias-level correction using overscans failed in "
1610  "IFU %hhu: %s", aIFU, cpl_error_get_message());
1611  }
1612  if (muse_basicproc_check_overscans(images, aProcessing, aBPars)
1613  != CPL_ERROR_NONE) { /* only for BIAS files and when overscan==none */
1614  muse_imagelist_delete(images);
1615  return NULL;
1616  }
1617  muse_basicproc_gain_override(images, aProcessing, aIFU);
1618  muse_basicproc_check_gain(images, aProcessing, aIFU); /* for flats */
1619  if (muse_basicproc_apply_bias(images, aProcessing, aIFU, aBPars)
1620  != CPL_ERROR_NONE) {
1621  muse_imagelist_delete(images);
1622  return NULL;
1623  }
1624  if (muse_basicproc_corr_nonlinearity(images, aProcessing, aIFU) != CPL_ERROR_NONE) {
1625  muse_imagelist_delete(images);
1626  return NULL;
1627  }
1628  if (muse_basicproc_adu_to_count(images, aProcessing) != CPL_ERROR_NONE) {
1629  muse_imagelist_delete(images);
1630  return NULL;
1631  }
1632  if (muse_basicproc_apply_dark(images, aProcessing, aIFU) != CPL_ERROR_NONE) {
1633  muse_imagelist_delete(images);
1634  return NULL;
1635  }
1636  if (muse_basicproc_apply_cr(images, aBPars) != CPL_ERROR_NONE) {
1637  muse_imagelist_delete(images);
1638  return NULL;
1639  }
1640  if (muse_basicproc_apply_flat(images, aProcessing, aBPars, aIFU) != CPL_ERROR_NONE) {
1641  muse_imagelist_delete(images);
1642  return NULL;
1643  }
1644  return images;
1645 } /* muse_basicproc_load() */
1646 
1647 /*---------------------------------------------------------------------------*/
1660 /*---------------------------------------------------------------------------*/
1662 muse_basicproc_load_reduced(muse_processing *aProcessing, unsigned char aIFU)
1663 {
1664  muse_imagelist *images = muse_imagelist_new();
1665  cpl_frameset *redframes = muse_frameset_find_tags(aProcessing->inframes,
1666  aProcessing->intags,
1667  aIFU, CPL_FALSE);
1668  cpl_size iframe, nframes = cpl_frameset_get_size(redframes);
1669  for (iframe = 0; iframe < nframes; iframe++) {
1670  cpl_frame *frame = cpl_frameset_get_position(redframes, iframe);
1671  cpl_errorstate prestate = cpl_errorstate_get();
1672  const char *imagename = cpl_frame_get_filename(frame);
1673  muse_image *image = muse_image_load(imagename);
1674  if (!image) {
1675  cpl_errorstate_set(prestate);
1676  image = muse_image_load_from_extensions(imagename, aIFU);
1677  }
1678  muse_imagelist_set(images, image, iframe);
1679  muse_processing_append_used(aProcessing, frame, CPL_FRAME_GROUP_RAW, 1);
1680  } /* for iframe (all used frames) */
1681  cpl_frameset_delete(redframes);
1682  return images;
1683 } /* muse_basicproc_load_reduced() */
1684 
1685 /*---------------------------------------------------------------------------*/
1700 /*---------------------------------------------------------------------------*/
1701 static cpl_table *
1702 muse_basicproc_prepare_illum(muse_pixtable *aPT)
1703 {
1704  cpl_ensure(aPT && aPT->header && aPT->table,
1705  CPL_ERROR_NULL_INPUT, NULL);
1706 
1707  /* crop pixel table to small wavelength range *
1708  * and separate into per-slice tables */
1709  muse_pixtable_restrict_wavelength(aPT, 6500., 7500.);
1711  int ipt, npt = muse_pixtable_extracted_get_size(pts);
1712 
1713  unsigned char ifu = muse_utils_get_ifu(aPT->header);
1714  cpl_msg_info(__func__, "Preparing %s flat: %d slices in the data of IFU %hhu "
1715  "found.", MUSE_TAG_ILLUM, npt, ifu);
1716  cpl_table *tattached = cpl_table_new(npt);
1717  cpl_table_new_column(tattached, "slice", CPL_TYPE_INT);
1718  cpl_table_new_column(tattached, "fflat", CPL_TYPE_DOUBLE);
1719  for (ipt = 0; ipt < npt; ipt++) {
1720  uint32_t origin = cpl_table_get_int(pts[ipt]->table, MUSE_PIXTABLE_ORIGIN, 0, NULL);
1721  unsigned short slice = muse_pixtable_origin_get_slice(origin);
1722  double median = cpl_table_get_column_median(pts[ipt]->table,
1724  cpl_msg_debug(__func__, "Found median of %f in slice %d of IFU %hhu "
1725  "of illum flat.", median, slice, ifu);
1726  cpl_table_set_int(tattached, "slice", ipt, slice);
1727  cpl_table_set_double(tattached, "fflat", ipt, 1. / median);
1728  } /* for ipt */
1730  /* normalize the flat-field scales */
1731  double mean = cpl_table_get_column_mean(tattached, "fflat");
1732  cpl_msg_debug(__func__, "Normalizing whole illum-flat table if IFU %hhu to "
1733  "%e.", ifu, mean);
1734  cpl_table_multiply_scalar(tattached, "fflat", 1. / mean);
1735  cpl_table_set_column_format(tattached, "fflat", "%.6f");
1736  return tattached;
1737 } /* muse_basicproc_prepare_illum() */
1738 
1739 /*---------------------------------------------------------------------------*/
1757 /*---------------------------------------------------------------------------*/
1758 cpl_table *
1759 muse_basicproc_get_illum(muse_imagelist *aImages, cpl_table *aTrace,
1760  cpl_table *aWave, cpl_table *aGeo)
1761 {
1762  cpl_ensure(aImages && aTrace && aWave && aGeo, CPL_ERROR_NULL_INPUT, NULL);
1763 
1764  cpl_table *tattached = NULL;
1765  unsigned int k, nimages = muse_imagelist_get_size(aImages);
1766  /* also create list of illum exposures to be able to erase them at the end */
1767  cpl_boolean *isillum = cpl_calloc(nimages, sizeof(cpl_boolean));
1768  for (k = 0; k < nimages; k++) {
1769  isillum[k] = CPL_FALSE;
1770  muse_image *image = muse_imagelist_get(aImages, k);
1771  const char *tag = cpl_propertylist_get_string(image->header,
1772  MUSE_HDR_TMP_INTAG);
1773  if (tag && !strncmp(tag, MUSE_TAG_ILLUM, strlen(MUSE_TAG_ILLUM) + 1)) {
1774  isillum[k] = CPL_TRUE; /* is an attached FLAT,ILLUM */
1775  /* also verify the template ID */
1776  if (cpl_propertylist_has(image->header, "ESO TPL ID")) {
1777  const char *tplid = cpl_propertylist_get_string(image->header,
1778  "ESO TPL ID"),
1779  *fn = cpl_propertylist_get_string(image->header,
1780  MUSE_HDR_TMP_FN),
1781  *tplatt = "MUSE_wfm_cal_specflatatt",
1782  *tplill = "MUSE_wfm_cal_illum";
1783  if (strncmp(tplid, tplatt, strlen(tplatt) + 1) &&
1784  strncmp(tplid, tplill, strlen(tplill) + 1)) {
1785  cpl_msg_warning(__func__, "%s input (\"%s\") was taken with neither"
1786  " %s nor %s template, but %s!", MUSE_TAG_ILLUM, fn,
1787  tplatt, tplill, tplid);
1788  } else {
1789  cpl_msg_debug(__func__, "%s input (\"%s\") was taken with template "
1790  "%s", MUSE_TAG_ILLUM, fn, tplid);
1791  } /* else */
1792  } /* if TPL.ID present */
1793  } /* if */
1794  unsigned char ifu = muse_utils_get_ifu(image->header);
1795  if (isillum[k]) {
1796  if (tattached) {
1797  cpl_msg_warning(__func__, "Image %u of %u of IFU %hhu is illum flat, "
1798  "but not the first; not using it!", k + 1, nimages, ifu);
1799  continue;
1800  }
1801  cpl_msg_debug(__func__, "Image %u of %u of IFU %hhu is illum flat.",
1802  k + 1, nimages, ifu);
1803  muse_pixtable *pt = muse_pixtable_create(image, aTrace, aWave, aGeo);
1804  tattached = muse_basicproc_prepare_illum(pt);
1806  } else {
1807  cpl_msg_debug(__func__, "Image %u of %u of IFU %hhu is not an illum flat.",
1808  k + 1, nimages, ifu);
1809  }
1810  } /* for k */
1811 
1812  /* remove the ILLUM image(s) from the image list */
1813  unsigned int k2;
1814  for (k = 0, k2 = 0; k < nimages; k++) {
1815  if (isillum[k]) {
1816  muse_image *image = muse_imagelist_unset(aImages, k2);
1817  muse_image_delete(image);
1818  } else {
1819  k2++; /* only step, if the image was not removed */
1820  }
1821  } /* for k, k2 */
1822  cpl_free(isillum);
1823 
1824  return tattached;
1825 } /* muse_basicproc_get_illum() */
1826 
1827 /*---------------------------------------------------------------------------*/
1840 /*---------------------------------------------------------------------------*/
1841 cpl_error_code
1842 muse_basicproc_apply_illum(muse_pixtable *aPT, cpl_table *aAttached)
1843 {
1844  cpl_ensure_code(aPT && aPT->header && aPT->table && aAttached,
1845  CPL_ERROR_NULL_INPUT);
1846 
1847  unsigned char ifu = muse_utils_get_ifu(aPT->header);
1848 
1850  int ipt, npt = muse_pixtable_extracted_get_size(pts);
1851  cpl_msg_info(__func__, "Applying %s flat-field in IFU %hhu (%d slices)",
1852  MUSE_TAG_ILLUM, ifu, npt);
1853  cpl_array *afactors = cpl_array_new(npt, CPL_TYPE_DOUBLE);
1854  for (ipt = 0; ipt < npt; ipt++) {
1855  uint32_t origin = cpl_table_get_int(pts[ipt]->table, MUSE_PIXTABLE_ORIGIN, 0, NULL);
1856  unsigned short slice = muse_pixtable_origin_get_slice(origin),
1857  fslice = cpl_table_get_int(aAttached, "slice", ipt, NULL);
1858  int err;
1859  double fflat = cpl_table_get_double(aAttached, "fflat", ipt, &err);
1860  if (!err && slice == fslice) {
1861  cpl_table_multiply_scalar(pts[ipt]->table, MUSE_PIXTABLE_DATA, fflat);
1862  cpl_table_multiply_scalar(pts[ipt]->table, MUSE_PIXTABLE_STAT, fflat*fflat);
1863  cpl_array_set_double(afactors, ipt, fflat);
1864  char *kw = cpl_sprintf(MUSE_HDR_PT_ILLUMi, slice);
1865  cpl_propertylist_update_double(aPT->header, kw, fflat);
1866  cpl_free(kw);
1867  } else {
1868  cpl_msg_warning(__func__, "some error (%d) occurred when applying illum-"
1869  "flat correction to slice %hu/%hu of IFU %hhu: %s", err,
1870  slice, fslice, ifu, cpl_error_get_message());
1871  } /* else */
1872  } /* for ipt */
1874  cpl_propertylist_update_double(aPT->header, MUSE_HDR_PT_ILLUM_MEAN,
1875  cpl_array_get_mean(afactors));
1876  cpl_propertylist_update_double(aPT->header, MUSE_HDR_PT_ILLUM_STDEV,
1877  cpl_array_get_stdev(afactors));
1878  cpl_array_delete(afactors);
1879  return CPL_ERROR_NONE;
1880 } /* muse_basicproc_apply_illum() */
1881 
1882 /*---------------------------------------------------------------------------*/
1899 /*---------------------------------------------------------------------------*/
1900 cpl_error_code
1902 {
1903  cpl_ensure_code(aPT && aPT->header && aPT->table && aTwilight,
1904  CPL_ERROR_NULL_INPUT);
1905 
1906  // XXX should check, if the twilight cube is of the same INS.MODE as the data
1907 
1908  unsigned char ifu = muse_utils_get_ifu(aPT->header);
1909 
1910  /* transfer the sky flat flux value */
1911  char *kw = cpl_sprintf(MUSE_HDR_FLAT_FLUX_SKY"%hhu", ifu);
1912  double flux = cpl_propertylist_get_double(aTwilight->header, kw);
1913  cpl_free(kw);
1914  cpl_propertylist_update_double(aPT->header, MUSE_HDR_FLAT_FLUX_SKY, flux);
1915 
1916  /* get the WCS from the twilight cube */
1917  int nx = cpl_image_get_size_x(cpl_imagelist_get(aTwilight->data, 0)),
1918  ny = cpl_image_get_size_y(cpl_imagelist_get(aTwilight->data, 0)),
1919  nz = cpl_imagelist_get_size(aTwilight->data);
1920  cpl_msg_debug(__func__, "Working with %d planes in twilight cube", nz);
1921  double cd12 = muse_pfits_get_cd(aTwilight->header, 1, 2),
1922  cd21 = muse_pfits_get_cd(aTwilight->header, 2, 1);
1923  if (cd12 > DBL_EPSILON || cd21 > DBL_EPSILON) {
1924  cpl_msg_warning(__func__, "Twilight cube contains WCS cross-terms (CD1_2"
1925  "=%e, CD2_1=%e), results will be inaccurate!", cd12, cd21);
1926  }
1927  // XXX guard against more rubbish (CTYPEi, CUNITi, cross-terms with axis3
1928  double crval1 = muse_pfits_get_crval(aTwilight->header, 1),
1929  crpix1 = muse_pfits_get_crpix(aTwilight->header, 1),
1930  cd11 = muse_pfits_get_cd(aTwilight->header, 1, 1),
1931  crval2 = muse_pfits_get_crval(aTwilight->header, 2),
1932  crpix2 = muse_pfits_get_crpix(aTwilight->header, 2),
1933  cd22 = muse_pfits_get_cd(aTwilight->header, 2, 2),
1934  crval3 = muse_pfits_get_crval(aTwilight->header, 3),
1935  crpix3 = muse_pfits_get_crpix(aTwilight->header, 3),
1936  cd33 = muse_pfits_get_cd(aTwilight->header, 3, 3);
1937 
1938  /* loop through the pixel table and apply the correction */
1939  float *data = cpl_table_get_data_float(aPT->table, MUSE_PIXTABLE_DATA),
1940  *stat = cpl_table_get_data_float(aPT->table, MUSE_PIXTABLE_STAT),
1941  *xpos = cpl_table_get_data_float(aPT->table, MUSE_PIXTABLE_XPOS),
1942  *ypos = cpl_table_get_data_float(aPT->table, MUSE_PIXTABLE_YPOS),
1943  *lbda = cpl_table_get_data_float(aPT->table, MUSE_PIXTABLE_LAMBDA);
1944  cpl_size irow, nrow = muse_pixtable_get_nrow(aPT),
1945  nfailed = 0;
1946  for (irow = 0; irow < nrow; irow++) {
1947  /* find closest spatial pixel in twilight cube *
1948  * using the coordinates from the pixel table */
1949  int x = lround((xpos[irow] - crval1) / cd11 + crpix1), /* nearest neighbor */
1950  y = lround((ypos[irow] - crval2) / cd22 + crpix2); /* nearest neighbor */
1951  /* boundary conditions */
1952  if (x < 1) {
1953  x = 1;
1954  }
1955  if (x > nx) {
1956  x = nx;
1957  }
1958  if (y < 1) {
1959  y = 1;
1960  }
1961  if (y > ny) {
1962  y = ny;
1963  }
1964  double z = (lbda[irow] - crval3) / cd33 + crpix3; /* plane */
1965 #if 0
1966  cpl_msg_debug(__func__, "%"CPL_SIZE_FORMAT": %.3f %.3f %.3f => %d %d %.3f",
1967  irow, xpos[irow], ypos[irow], lbda[irow], x, y, z);
1968 #endif
1969  /* get indices of the two closest image planes for linear interpolation */
1970  int zp1 = floor(z) - 1,
1971  zp2 = ceil(z) - 1;
1972  /* boundary conditions */
1973  if (zp1 < 1) {
1974  zp1 = 0;
1975  }
1976  if (zp1 >= nz) {
1977  zp1 = nz - 1;
1978  }
1979  if (zp2< 1) {
1980  zp2 = 0;
1981  }
1982  if (zp2 >= nz) {
1983  zp2 = nz - 1;
1984  }
1985  int err1, err2;
1986  double v1 = cpl_image_get(cpl_imagelist_get(aTwilight->data, zp1),
1987  x, y, &err1),
1988  v2 = cpl_image_get(cpl_imagelist_get(aTwilight->data, zp2),
1989  x, y, &err2);
1990  double villum = 1.;
1991  double f = 1.;
1992  /* linearly interpolate the given twilight factors from both planes */
1993  if (err1 && err2) {
1994  nfailed++;
1995  continue;
1996  }
1997  if (zp1 == zp2) {
1998  villum = v1; /* same planes, just take the first value */
1999  } else if (err1) {
2000  villum = v2;
2001  } else if (err2) {
2002  villum = v1;
2003  } else { /* real interpolation */
2004  f = fabs(z - 1 - zp1);
2005  villum = v1 * (1. - f) + v2 * f;
2006  }
2007  double fillum = 1. / villum; /* the inverse as correction factor */
2008 #if 0
2009  cpl_msg_debug(__func__, "%d/%d, %f/%f -> %f -> %f => %f", zp1, zp2, v1, v2,
2010  f, villum, fillum);
2011 #endif
2012 
2013  /* multiply the data value by the inverse twilight factor */
2014  data[irow] *= fillum;
2015  /* multiply the stat value by the squared inverse twilight factor */
2016  stat[irow] *= fillum*fillum;
2017  } /* for irow (all pixel table rows) */
2018  if (nfailed) {
2019  cpl_msg_warning(__func__, "Failed to correct twilight in %"CPL_SIZE_FORMAT
2020  " of %"CPL_SIZE_FORMAT", pixels in IFU %hhu!", nfailed,
2021  nrow, ifu);
2022  } else {
2023  cpl_msg_debug(__func__, "No failures during twilight correction of %"
2024  CPL_SIZE_FORMAT" pixels in IFU %hhu", nrow, ifu);
2025  }
2026 
2027  return CPL_ERROR_NONE;
2028 } /* muse_basicproc_apply_twilight() */
2029 
2030 /*----------------------------------------------------------------------------*/
2048 /*----------------------------------------------------------------------------*/
2049 cpl_error_code
2051 {
2052  cpl_ensure_code(aPT && aPT->header && aPT->table, CPL_ERROR_NULL_INPUT);
2053 
2054  /* for the moment set values for [WN]FM-AO-N] (Na2 filter) */
2055  float lmin = kMuseNa2LambdaMin,
2056  lmax = kMuseNa2LambdaMax;
2057  muse_ins_mode insmode = muse_pfits_get_mode(aPT->header);
2058  const char *mode = muse_pfits_get_insmode(aPT->header);
2059  if ((insmode == MUSE_MODE_WFM_AO_N) || (insmode == MUSE_MODE_NFM_AO_N)) {
2060  /* nothing to be done */
2061  } else if (insmode == MUSE_MODE_WFM_AO_E) {
2062  lmin = kMuseNaLambdaMin;
2063  lmax = kMuseNaLambdaMax;
2064  } else {
2065  cpl_msg_warning(__func__, "No notch filter for mode %s!", mode);
2066  return CPL_ERROR_ILLEGAL_INPUT;
2067  }
2068  cpl_msg_info(__func__, "%s mode: marking NaD region (%.1f..%.1f Angstrom) of "
2069  "IFU %d as 0x%08lx", mode, lmin, lmax, aIFU, EURO3D_NOTCH_NAD);
2070  cpl_table_unselect_all(aPT->table);
2071  cpl_table_or_selected_float(aPT->table, MUSE_PIXTABLE_LAMBDA,
2072  CPL_GREATER_THAN, lmin);
2073  cpl_table_and_selected_float(aPT->table, MUSE_PIXTABLE_LAMBDA,
2074  CPL_LESS_THAN, lmax);
2075  cpl_array *asel = cpl_table_where_selected(aPT->table);
2076  cpl_size isel, nsel = cpl_array_get_size(asel);
2077  const cpl_size *sel = cpl_array_get_data_cplsize_const(asel);
2078  int *dq = cpl_table_get_data_int(aPT->table, MUSE_PIXTABLE_DQ);
2079  for (isel = 0; isel < nsel; isel++) {
2080  dq[sel[isel]] = EURO3D_NOTCH_NAD;
2081  } /* for isel (all selected table rows) */
2082  cpl_array_delete(asel);
2083 
2084  return CPL_ERROR_NONE;
2085 } /* muse_basicproc_mask_notch_filter() */
2086 
2087 /*----------------------------------------------------------------------------*/
2113 /*----------------------------------------------------------------------------*/
2114 static int
2115 muse_basicproc_combine_compare_lamp(const cpl_frame *aFrame1, const cpl_frame *aFrame2)
2116 {
2117  cpl_ensure(aFrame1 && aFrame2, CPL_ERROR_NULL_INPUT, -1);
2118  const char *fn1 = cpl_frame_get_filename(aFrame1),
2119  *fn2 = cpl_frame_get_filename(aFrame2);
2120  cpl_propertylist *head1 = cpl_propertylist_load(fn1, 0),
2121  *head2 = cpl_propertylist_load(fn2, 0);
2122  if (!head1 || !head2) {
2123  cpl_propertylist_delete(head1); /* in case one was loaded... */
2124  cpl_propertylist_delete(head2);
2125  return -1;
2126  }
2127 
2128  /* Loop through all lamps in the header and find their status. The first *
2129  * missing shutter entry will cause the FITS query to throw an error, that's *
2130  * when we stop. Otherwise, we are done when the lamp status is not equal. */
2131  int nlamp = 1, status1, status2;
2132  cpl_errorstate prestate = cpl_errorstate_get();
2133  do {
2134  /* ensure that we are dealing with the same lamps! */
2135  const char *name1 = muse_pfits_get_lamp_name(head1, nlamp),
2136  *name2 = muse_pfits_get_lamp_name(head2, nlamp);
2137  cpl_errorstate_set(prestate); /* lamps may be missing */
2138  if (name1 && name2 && strncmp(name1, name2, strlen(name1) + 1)) {
2139  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT,
2140  "Files \"%s\" and \"%s\" have incompatible lamp "
2141  "setups", fn1, fn2);
2142  cpl_propertylist_delete(head1);
2143  cpl_propertylist_delete(head2);
2144  return -1;
2145  }
2146  name1 = muse_pfits_get_shut_name(head1, nlamp);
2147  name2 = muse_pfits_get_shut_name(head2, nlamp);
2148  if (name1 && name2 && strncmp(name1, name2, strlen(name1) + 1)) {
2149  cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT,
2150  "Files \"%s\" and \"%s\" have incompatible shutter "
2151  "setups", fn1, fn2);
2152  cpl_propertylist_delete(head1);
2153  cpl_propertylist_delete(head2);
2154  return -1;
2155  }
2156 
2157  status1 = muse_pfits_get_lamp_status(head1, nlamp);
2158  status2 = muse_pfits_get_lamp_status(head2, nlamp);
2159  cpl_errorstate_set(prestate); /* lamps may be missing */
2160  if (status1 != status2) {
2161  break;
2162  }
2163  status1 = muse_pfits_get_shut_status(head1, nlamp);
2164  status2 = muse_pfits_get_shut_status(head2, nlamp);
2165  if (status1 != status2) {
2166  break;
2167  }
2168  nlamp++;
2169  } while (cpl_errorstate_is_equal(prestate));
2170  cpl_errorstate_set(prestate);
2171 
2172  cpl_propertylist_delete(head1);
2173  cpl_propertylist_delete(head2);
2174  return status1 == status2;
2175 } /* muse_basicproc_combine_compare_lamp() */
2176 
2177 /*---------------------------------------------------------------------------*/
2203 /*---------------------------------------------------------------------------*/
2206  unsigned char aIFU,
2207  muse_basicproc_params *aBPars,
2208  cpl_frameset ***aLabeledFrames)
2209 {
2210  if (aLabeledFrames) { /* NULL out return pointer, in case it's given... */
2211  *aLabeledFrames = NULL; /* ... so it's always NULL in case of problems */
2212  }
2213  cpl_ensure(aProcessing, CPL_ERROR_NULL_INPUT, NULL);
2214 
2215  /* find out different lamps in input */
2216  cpl_frameset *rawframes = muse_frameset_find_tags(aProcessing->inframes,
2217  aProcessing->intags, aIFU,
2218  CPL_FALSE);
2219  char *prefix = cpl_sprintf("muse.%s", aProcessing->name);
2220  muse_combinepar *cpars = muse_combinepar_new(aProcessing->parameters,
2221  prefix);
2222  cpl_free(prefix);
2223 #if 0
2224  printf("rawframes\n");
2225  cpl_frameset_dump(rawframes, stdout);
2226  fflush(stdout);
2227 #endif
2228  cpl_size nlabels,
2229  *labels = cpl_frameset_labelise(rawframes,
2230  muse_basicproc_combine_compare_lamp,
2231  &nlabels);
2232  if (!labels || nlabels <= 1) {
2233  /* if labeling didn't work, process the list of one lamp and return */
2234  cpl_free(labels);
2235  cpl_frameset_delete(rawframes);
2236  muse_imagelist *list = muse_basicproc_load(aProcessing, aIFU, aBPars),
2237  *images = NULL;
2238  if (nlabels == 1) {
2239  muse_image *image = muse_combine_images(cpars, list);
2240  images = muse_imagelist_new();
2241  muse_imagelist_set(images, image, 0);
2242  if (aLabeledFrames) {
2243  *aLabeledFrames = cpl_calloc(1, sizeof(cpl_frameset *));
2244  (*aLabeledFrames)[0] = cpl_frameset_duplicate(aProcessing->usedframes);
2245  } /* if */
2246  } /* if only one label */
2247  muse_imagelist_delete(list);
2248  muse_combinepar_delete(cpars);
2249  return images;
2250  } /* if one of no labels */
2251 
2252 #if 0
2253  cpl_array *alabels = cpl_array_wrap_int(labels, cpl_frameset_get_size(rawframes));
2254  cpl_array_dump(alabels, 0, 1000, stdout);
2255  fflush(stdout);
2256  cpl_array_unwrap(alabels);
2257 #endif
2258  /* output list of lampwise combined images */
2259  muse_imagelist *images = muse_imagelist_new();
2260  if (aLabeledFrames) {
2261  *aLabeledFrames = cpl_calloc(nlabels, sizeof(cpl_frameset *));
2262  }
2263 
2264  /* duplicate aProcessing into a local structure the contents of which *
2265  * we can manipulate here (don't change it directly for threadsafety!) */
2266  muse_processing *proc = cpl_malloc(sizeof(muse_processing));
2267  memcpy(proc, aProcessing, sizeof(muse_processing));
2268  cpl_frameset *inframes = proc->inframes;
2269  /* copy frames with all extra frames somewhere else */
2270  cpl_frameset *auxframes = muse_frameset_find_tags(inframes, aProcessing->intags,
2271  aIFU, CPL_TRUE);
2272  /* loop through labels for all lamps */
2273  int ilabel, ipos = 0;
2274  for (ilabel = 0; ilabel < nlabels; ilabel++) {
2275  /* create new sub-frameset for this lamp */
2276  cpl_frameset *frames = cpl_frameset_extract(rawframes, labels, ilabel);
2277  /* append other files for the initial processing */
2278  cpl_frameset_join(frames, auxframes);
2279  /* substitute aProcessing->inframes */
2280  proc->inframes = frames;
2281  /* load and combine frames for this sub-frameset */
2282  muse_imagelist *list = muse_basicproc_load(proc, aIFU, aBPars);
2283  /* reinstate original aProcessing->inframes */
2284  proc->inframes = inframes;
2285  if (!list) { /* break this loop to fail the function below */
2286  muse_imagelist_delete(images);
2287  cpl_frameset_delete(frames);
2288  images = NULL;
2289  /* if muse_basicproc_load() fails then because of some missing *
2290  * calibration, and then it will already fail for the first *
2291  * ilabel; it is therefore enough to free the pointer */
2292  if (aLabeledFrames) {
2293  cpl_free(*aLabeledFrames);
2294  *aLabeledFrames = NULL;
2295  }
2296  break;
2297  }
2298 
2299  muse_image *lampimage = muse_combine_images(cpars, list);
2300  if (!lampimage) {
2301  cpl_msg_error(__func__, "Image combination failed for IFU %hhu for lamp "
2302  "with label %d of %"CPL_SIZE_FORMAT, aIFU, ilabel + 1, nlabels);
2303  muse_imagelist_delete(list);
2304  cpl_frameset_delete(frames);
2305  continue;
2306  }
2307 
2308  if (aLabeledFrames) {
2309  /* if a given frame was used now, copy its group */
2310  cpl_size iframe, nframes = cpl_frameset_get_size(frames);
2311  for (iframe = 0; iframe < nframes; iframe++) {
2312  cpl_frame *frame = cpl_frameset_get_position(frames, iframe);
2313  const char *fn = cpl_frame_get_filename(frame),
2314  *tag = cpl_frame_get_tag(frame);
2315  cpl_size iuframe, nuframes = cpl_frameset_get_size(aProcessing->usedframes);
2316  for (iuframe = 0;
2317  (iuframe < nuframes) && fn && tag; /* only check with valid info */
2318  iuframe++) {
2319  cpl_frame *uframe = cpl_frameset_get_position(aProcessing->usedframes,
2320  iuframe);
2321  const char *fnu = cpl_frame_get_filename(uframe),
2322  *tagu = cpl_frame_get_tag(uframe);
2323  if (fnu && !strncmp(fn, fnu, strlen(fn) + 1) &&
2324  tagu && !strncmp(tag, tagu, strlen(tag) + 1)) {
2325  cpl_frame_set_group(frame, cpl_frame_get_group(uframe));
2326  break;
2327  }
2328  } /* for uframe (all usedframes) */
2329  } /* for frame */
2330  (*aLabeledFrames)[ipos] = frames;
2331  } else {
2332  cpl_frameset_delete(frames);
2333  }
2334 
2335  /* transfer NSATURATION headers from invidual images to lamp-combined image */
2336  unsigned int k;
2337  for (k = 0; k < muse_imagelist_get_size(list); k++) {
2338  char *keyword = cpl_sprintf(QC_WAVECAL_PREFIXi" "QC_BASIC_NSATURATED, k+1);
2339  int nsaturated = cpl_propertylist_get_int(muse_imagelist_get(list, k)->header,
2340  MUSE_HDR_TMP_NSAT);
2341  cpl_propertylist_update_int(lampimage->header, keyword, nsaturated);
2342  cpl_free(keyword);
2343  }
2344  muse_imagelist_delete(list);
2345  /* append to imagelist */
2346  muse_imagelist_set(images, lampimage, ipos++);
2347  } /* for ilabel (labels) */
2348  cpl_free(labels);
2349  cpl_free(proc);
2350  muse_combinepar_delete(cpars);
2351  cpl_frameset_delete(rawframes);
2352  cpl_frameset_delete(auxframes);
2353 
2354  if (images && muse_imagelist_get_size(images) == 0) {
2355  muse_imagelist_delete(images);
2356  images = NULL;
2357  if (aLabeledFrames) {
2358  cpl_free(*aLabeledFrames);
2359  *aLabeledFrames = NULL;
2360  }
2361  }
2362 
2363  return images;
2364 } /* muse_basicproc_combine_images_lampwise() */
2365 
2366 /*---------------------------------------------------------------------------*/
2388 /*---------------------------------------------------------------------------*/
2389 cpl_error_code
2391  double aHalfWidth, double aBinWidth,
2392  float aLo, float aHi, unsigned char aIter)
2393 {
2394  cpl_ensure_code(aPt && aLines, CPL_ERROR_NULL_INPUT);
2395  cpl_ensure_code(cpl_array_get_type(aLines) == CPL_TYPE_DOUBLE ||
2396  cpl_array_get_type(aLines) == CPL_TYPE_FLOAT,
2397  CPL_ERROR_ILLEGAL_INPUT);
2398 
2399  double lmin = cpl_propertylist_get_float(aPt->header, MUSE_HDR_PT_LLO),
2400  lmax = cpl_propertylist_get_float(aPt->header, MUSE_HDR_PT_LHI);
2401  double shift = 0., wshift = 0.;
2402  cpl_array *errors = cpl_array_new(4, CPL_TYPE_DOUBLE);
2403  int i, n = cpl_array_get_size(aLines), nvalid = 0;
2404  for (i = 0; i < n; i++) {
2405  int err;
2406  double lambdasign = cpl_array_get(aLines, i, &err),
2407  lambda = fabs(lambdasign);
2408  if (err || lambda >= lmax || lambda <= lmin) {
2409  cpl_msg_debug(__func__, "Invalid wavelength at position %d of %d in "
2410  "skylines", i + 1, n);
2411  continue;
2412  }
2413  nvalid++;
2414  double center = muse_utils_pixtable_fit_line_gaussian(aPt, lambdasign, aHalfWidth,
2415  aBinWidth, aLo, aHi, aIter,
2416  NULL, errors),
2417  cerr = cpl_array_get_double(errors, 0, NULL);
2418  shift += (lambda - center) / cerr;
2419  wshift += 1. / cerr;
2420  cpl_msg_debug(__func__, "dlambda = %.4f +/- %.4f (for skyline at %.4f "
2421  "Angstrom)", lambda - center, cerr, lambda);
2422  } /* for i (all entries in aLines) */
2423  cpl_array_delete(errors);
2424  shift /= wshift;
2425  if (nvalid > 0 && isfinite(shift)) {
2426  cpl_msg_info(__func__, "Skyline correction (%d lines): shifting data of IFU "
2427  "%hhu by %.4f Angstrom", nvalid, muse_utils_get_ifu(aPt->header),
2428  shift);
2429  cpl_table_add_scalar(aPt->table, MUSE_PIXTABLE_LAMBDA, shift);
2430  cpl_propertylist_update_float(aPt->header, QC_SCIBASIC_SHIFT, shift);
2431  } else {
2432  cpl_propertylist_update_float(aPt->header, QC_SCIBASIC_SHIFT, 0.);
2433  }
2434  return CPL_ERROR_NONE;
2435 } /* muse_basicproc_shift_pixtable() */
2436 
2437 /*---------------------------------------------------------------------------*/
2449 /*---------------------------------------------------------------------------*/
2450 cpl_error_code
2451 muse_basicproc_stats_append_header(cpl_image *aImage, cpl_propertylist *aHeader,
2452  const char *aPrefix, unsigned aStats)
2453 {
2454  cpl_ensure_code(aImage, CPL_ERROR_NULL_INPUT);
2455 
2456  int nx = cpl_image_get_size_x(aImage),
2457  ny = cpl_image_get_size_y(aImage);
2458  return muse_basicproc_stats_append_header_window(aImage, aHeader, aPrefix,
2459  aStats, 1, 1, nx, ny);
2460 } /* muse_basicproc_stats_append_header() */
2461 
2462 /*---------------------------------------------------------------------------*/
2479 /*---------------------------------------------------------------------------*/
2480 cpl_error_code
2482  cpl_propertylist *aHeader,
2483  const char *aPrefix, unsigned aStats,
2484  int aX1, int aY1, int aX2, int aY2)
2485 {
2486  cpl_ensure_code(aImage && aHeader, CPL_ERROR_NULL_INPUT);
2487  cpl_ensure_code(aPrefix, CPL_ERROR_ILLEGAL_INPUT);
2488  cpl_ensure_code(aPrefix, CPL_ERROR_ILLEGAL_INPUT);
2489 
2490  cpl_stats *stats = cpl_stats_new_from_image_window(aImage, aStats,
2491  aX1, aY1, aX2, aY2);
2492  if (!stats) {
2493  return cpl_error_get_code();
2494  }
2495 
2496  char keyword[KEYWORD_LENGTH];
2497  if (aStats & CPL_STATS_MEDIAN) {
2498  snprintf(keyword, KEYWORD_LENGTH, "%s MEDIAN", aPrefix);
2499  cpl_propertylist_append_float(aHeader, keyword,
2500  cpl_stats_get_median(stats));
2501  }
2502  if (aStats & CPL_STATS_MEAN) {
2503  snprintf(keyword, KEYWORD_LENGTH, "%s MEAN", aPrefix);
2504  cpl_propertylist_append_float(aHeader, keyword, cpl_stats_get_mean(stats));
2505  }
2506  if (aStats & CPL_STATS_STDEV) {
2507  snprintf(keyword, KEYWORD_LENGTH, "%s STDEV", aPrefix);
2508  cpl_propertylist_append_float(aHeader, keyword, cpl_stats_get_stdev(stats));
2509  }
2510  if (aStats & CPL_STATS_MIN) {
2511  snprintf(keyword, KEYWORD_LENGTH, "%s MIN", aPrefix);
2512  cpl_propertylist_append_float(aHeader, keyword, cpl_stats_get_min(stats));
2513  }
2514  if (aStats & CPL_STATS_MAX) {
2515  snprintf(keyword, KEYWORD_LENGTH, "%s MAX", aPrefix);
2516  cpl_propertylist_append_float(aHeader, keyword, cpl_stats_get_max(stats));
2517  }
2518  if (aStats & CPL_STATS_FLUX) {
2519  snprintf(keyword, KEYWORD_LENGTH, "%s INTFLUX", aPrefix);
2520  cpl_propertylist_append_float(aHeader, keyword, cpl_stats_get_flux(stats));
2521  }
2522 
2523  cpl_stats_delete(stats);
2524 
2525  return CPL_ERROR_NONE;
2526 } /* muse_basicproc_stats_append_header_window() */
2527 
2528 /*---------------------------------------------------------------------------*/
2543 /*---------------------------------------------------------------------------*/
2544 cpl_error_code
2545 muse_basicproc_qc_saturated(muse_image *aImage, const char *aPrefix)
2546 {
2547  cpl_ensure_code(aImage && aImage->dq && aImage->header && aPrefix,
2548  CPL_ERROR_NULL_INPUT);
2549 
2550  cpl_mask *mask = cpl_mask_threshold_image_create(aImage->dq,
2551  EURO3D_SATURATED - 0.1,
2552  EURO3D_SATURATED + 0.1);
2553  int nsaturated = cpl_mask_count(mask);
2554  cpl_mask_delete(mask);
2555  /* check if the prefix has a trailing space, add it if not */
2556  char *keyword = NULL;
2557  if (aPrefix[strlen(aPrefix)-1] == ' ') {
2558  keyword = cpl_sprintf("%s%s", aPrefix, QC_BASIC_NSATURATED);
2559  } else {
2560  keyword = cpl_sprintf("%s %s", aPrefix, QC_BASIC_NSATURATED);
2561  }
2562  cpl_error_code rc = cpl_propertylist_update_int(aImage->header, keyword,
2563  nsaturated);
2564  cpl_free(keyword);
2565  return rc;
2566 } /* muse_basicproc_qc_saturated() */
2567 
cpl_error_code muse_quadrants_overscan_correct(muse_image *aImage, muse_image *aRefImage)
Adapt bias level to reference image using overscan statistics.
#define MUSE_PIXTABLE_DQ
Definition: muse_pixtable.h:49
muse_imagelist * muse_basicproc_load(muse_processing *aProcessing, unsigned char aIFU, muse_basicproc_params *aBPars)
Load the raw input files from disk and do basic processing.
#define MUSE_PIXTABLE_XPOS
Definition: muse_pixtable.h:51
Structure definition of a MUSE datacube.
Definition: muse_datacube.h:48
Structure definition for a collection of muse_images.
void muse_image_delete(muse_image *aImage)
Deallocate memory associated to a muse_image object.
Definition: muse_image.c:85
void muse_pixtable_extracted_delete(muse_pixtable **aPixtables)
Delete a pixel table array.
double muse_pfits_get_cd(const cpl_propertylist *aHeaders, unsigned int aAxisI, unsigned int aAxisJ)
find out the WCS coordinate at the reference point
Definition: muse_pfits.c:446
int muse_image_divide(muse_image *aImage, muse_image *aDivisor)
Divide a muse_image by another with correct treatment of bad pixels and variance. ...
Definition: muse_image.c:639
const char * muse_pfits_get_insmode(const cpl_propertylist *aHeaders)
find out the observation mode
Definition: muse_pfits.c:1383
double muse_utils_pixtable_fit_line_gaussian(muse_pixtable *aPixtable, double aLambda, double aHalfWidth, double aBinSize, float aLo, float aHi, unsigned char aIter, cpl_array *aResults, cpl_array *aErrors)
Fit a 1D Gaussian to a given wavelength range in a pixel table.
Definition: muse_utils.c:2442
unsigned short muse_pixtable_origin_get_slice(uint32_t aOrigin)
Get the slice number from the encoded 32bit origin number.
cpl_size * muse_quadrants_get_window(const muse_image *aImage, unsigned char aQuadrant)
Determine the data window of a given quadrant on the CCD.
int muse_image_scale(muse_image *aImage, double aScale)
Scale a muse_image with correct treatment of variance.
Definition: muse_image.c:703
int muse_pfits_get_read_id(const cpl_propertylist *aHeaders)
find out the readout mode id
Definition: muse_pfits.c:512
int muse_utils_get_extension_for_ifu(const char *aFilename, unsigned char aIFU)
Return extension number that corresponds to this IFU/channel number.
Definition: muse_utils.c:118
cpl_size muse_pixtable_extracted_get_size(muse_pixtable **aPixtables)
Get the size of an array of extracted pixel tables.
double muse_pfits_get_crval(const cpl_propertylist *aHeaders, unsigned int aAxis)
find out the WCS coordinate at the reference point
Definition: muse_pfits.c:423
const char * name
unsigned char muse_utils_get_ifu(const cpl_propertylist *aHeaders)
Find out the IFU/channel from which this header originated.
Definition: muse_utils.c:98
muse_imagelist * muse_basicproc_combine_images_lampwise(muse_processing *aProcessing, unsigned char aIFU, muse_basicproc_params *aBPars, cpl_frameset ***aLabeledFrames)
Combine several images into a lampwise image list.
muse_pixtable ** muse_pixtable_extracted_get_slices(muse_pixtable *aPixtable)
Extract one pixel table per IFU and slice.
double muse_pfits_get_gain(const cpl_propertylist *aHeaders, unsigned char aQuadrant)
find the detector gain (in units of count/adu)
Definition: muse_pfits.c:686
cpl_image * data
the data extension
Definition: muse_image.h:46
muse_image * muse_image_load_from_raw(const char *aFilename, int aExtension)
Load raw image into the data extension of a MUSE image.
Definition: muse_image.c:292
cpl_size muse_pixtable_get_nrow(const muse_pixtable *aPixtable)
get the number of rows within the pixel table
int muse_pfits_get_shut_status(const cpl_propertylist *aHeaders, int aShutter)
query the status of one shutter
Definition: muse_pfits.c:1569
#define MUSE_HDR_PT_LHI
FITS header keyword contains the upper limit of the data in spectral direction.
int muse_cosmics_dcr(muse_image *aImage, unsigned int aXBox, unsigned int aYBox, unsigned int aPasses, float aThres)
Quickly mark cosmic rays in an image using the DCR algorithm.
muse_image * muse_image_duplicate(const muse_image *aImage)
Duplicate the three image extensions and the FITS headers of a MUSE image.
Definition: muse_image.c:510
#define MUSE_HDR_PT_ILLUMi
cpl_image * stat
the statistics extension
Definition: muse_image.h:64
int muse_image_subtract(muse_image *aImage, muse_image *aSubtract)
Subtract a muse_image from another with correct treatment of bad pixels and variance.
Definition: muse_image.c:593
void muse_imagelist_delete(muse_imagelist *aList)
Free the memory of the MUSE image list.
#define MUSE_HDR_PT_ILLUM_MEAN
muse_imagelist * muse_basicproc_load_reduced(muse_processing *aProcessing, unsigned char aIFU)
Load reduced input files from disk.
muse_basicproc_params * muse_basicproc_params_new(cpl_parameterlist *aParameters, const char *aPrefix)
Create a new structure of basic processing parameters.
muse_image * flatimage
muse_image * muse_combine_images(muse_combinepar *aCPars, muse_imagelist *aImages)
Combine several images into one.
Definition: muse_combine.c:741
Structure definition of MUSE three extension FITS file.
Definition: muse_image.h:40
cpl_error_code muse_quadrants_overscan_polyfit_vertical(muse_image *aImage, unsigned aIgnore, unsigned char aOrder, double aSigma, const double aFRMS, double aFChiSq)
Correct quadrants by polynomial representation of vertical overscan.
unsigned int ovscignore
const char * muse_pfits_get_shut_name(const cpl_propertylist *aHeaders, int aShutter)
query the name of one shutter
Definition: muse_pfits.c:1548
cpl_table * table
The pixel table.
void muse_basicproc_params_delete(muse_basicproc_params *aBPars)
Free a structure of basic processing parameters.
cpl_propertylist * header
the FITS header
Definition: muse_image.h:72
cpl_boolean muse_processing_check_intags(muse_processing *aProcessing, const char *aTag, int aNChars)
Check that a tag is part of the input tags of a processing structure.
int muse_image_variance_create(muse_image *aImage, muse_image *aBias)
Create the photon noise-based variance in the stat extension.
Definition: muse_image.c:747
cpl_table * muse_cpltable_load(const char *aFile, const char *aExtension, const muse_cpltable_def aDefinition[])
Load a table from disk (and check against definition).
#define MUSE_PIXTABLE_DATA
Definition: muse_pixtable.h:48
unsigned int muse_imagelist_get_size(muse_imagelist *aList)
Return the number of stored images.
cpl_frameset * usedframes
void muse_combinepar_delete(muse_combinepar *aCPars)
Clear the combination parameters.
Definition: muse_combine.c:715
cpl_image * dq
the data quality extension
Definition: muse_image.h:56
cpl_error_code muse_pixtable_restrict_wavelength(muse_pixtable *aPixtable, double aLow, double aHigh)
Restrict a pixel table to a certain wavelength range.
double muse_pfits_get_crpix(const cpl_propertylist *aHeaders, unsigned int aAxis)
find out the WCS reference point
Definition: muse_pfits.c:401
int muse_pfits_get_biny(const cpl_propertylist *aHeaders)
find out the binning factor in y direction
Definition: muse_pfits.c:566
Structure definition of MUSE pixel table.
cpl_boolean muse_quadrants_overscan_check(muse_image *aImage, muse_image *aRefImage, double aSigma)
Compare overscan statistics of all quadrants to those of reference image.
cpl_error_code muse_basicproc_shift_pixtable(muse_pixtable *aPt, cpl_array *aLines, double aHalfWidth, double aBinWidth, float aLo, float aHi, unsigned char aIter)
Compute wavelength corrections for science data based on reference sky lines.
const char * muse_pfits_get_lamp_name(const cpl_propertylist *aHeaders, int aLamp)
query the name of one lamp
Definition: muse_pfits.c:1506
muse_image * muse_imagelist_get(muse_imagelist *aList, unsigned int aIdx)
Get the muse_image of given list index.
cpl_error_code muse_basicproc_apply_twilight(muse_pixtable *aPT, muse_datacube *aTwilight)
Apply an attached flat-field to a pixel table.
cpl_error_code muse_basicproc_stats_append_header_window(cpl_image *aImage, cpl_propertylist *aHeader, const char *aPrefix, unsigned aStats, int aX1, int aY1, int aX2, int aY2)
Compute image statistics of an image window and add them to a header.
int muse_quality_set_saturated(muse_image *aImage)
Set all pixels above the saturation limit in the bad pixel image.
Definition: muse_quality.c:473
const char * muse_pfits_get_read_name(const cpl_propertylist *aHeaders)
find out the readout mode name
Definition: muse_pfits.c:530
int muse_imagelist_is_uniform(muse_imagelist *aList)
Check that all images in the muse_imagelist have the same size.
const muse_cpltable_def muse_badpix_table_def[]
muse_combinepar * muse_combinepar_new(cpl_parameterlist *aParameters, const char *aPrefix)
Create a new set of combination parameters.
Definition: muse_combine.c:672
cpl_imagelist * data
the cube containing the actual data values
Definition: muse_datacube.h:76
void muse_processing_append_used(muse_processing *aProcessing, cpl_frame *aFrame, cpl_frame_group aGroup, int aDuplicate)
Add a frame to the set of used frames.
muse_pixtable * muse_pixtable_create(muse_image *aImage, cpl_table *aTrace, cpl_table *aWave, cpl_table *aGeoTable)
Create the pixel table for one CCD.
#define MUSE_PIXTABLE_ORIGIN
Definition: muse_pixtable.h:54
#define MUSE_HDR_PT_LLO
FITS header keyword contains the lower limit of the data in spectral direction.
const char * muse_pfits_get_chip_id(const cpl_propertylist *aHeaders)
find out the chip id
Definition: muse_pfits.c:602
cpl_error_code muse_quadrants_overscan_stats(muse_image *aImage, const char *aRejection, unsigned int aIgnore)
Compute overscan statistics of all quadrants and save in FITS header.
cpl_error_code muse_basicproc_stats_append_header(cpl_image *aImage, cpl_propertylist *aHeader, const char *aPrefix, unsigned aStats)
Compute image statistics of an image and add them to a header.
cpl_table * muse_basicproc_get_illum(muse_imagelist *aImages, cpl_table *aTrace, cpl_table *aWave, cpl_table *aGeo)
Get an illum/attached flat-field from an imagelist and prepare it for use.
int muse_pfits_get_binx(const cpl_propertylist *aHeaders)
find out the binning factor in x direction
Definition: muse_pfits.c:548
cpl_propertylist * header
the FITS header
Definition: muse_datacube.h:57
#define MUSE_PIXTABLE_STAT
Definition: muse_pixtable.h:50
cpl_error_code muse_image_save(muse_image *aImage, const char *aFilename)
Save the three image extensions and the FITS headers of a MUSE image to a file.
Definition: muse_image.c:405
muse_image * muse_image_load(const char *aFilename)
Load the three extensions and the FITS headers of a MUSE image from a file.
Definition: muse_image.c:228
int muse_pfits_get_lamp_status(const cpl_propertylist *aHeaders, int aLamp)
query the status of one lamp
Definition: muse_pfits.c:1527
cpl_array * intags
muse_imagelist * muse_imagelist_new(void)
Create a new (empty) MUSE image list.
#define MUSE_PIXTABLE_LAMBDA
Definition: muse_pixtable.h:53
cpl_frameset * inframes
cpl_error_code muse_basicproc_mask_notch_filter(muse_pixtable *aPT, unsigned char aIFU)
Mask the range of the NaD notch filter in the given pixel table.
muse_basicproc_params * muse_basicproc_params_new_from_propertylist(const cpl_propertylist *aHeader)
Create a structure of basic processing parameters from a FITS header.
double muse_pfits_get_exptime(const cpl_propertylist *aHeaders)
find out the exposure time
Definition: muse_pfits.c:382
cpl_error_code muse_processing_check_input(muse_processing *aProcessing, unsigned char aIFU)
Check the input files for completeness.
Structure of basic processing parameters.
const char * muse_pfits_get_pro_catg(const cpl_propertylist *aHeaders)
find out the PRO category
Definition: muse_pfits.c:159
#define MUSE_HDR_PT_ILLUM_STDEV
cpl_error_code muse_basicproc_apply_illum(muse_pixtable *aPT, cpl_table *aAttached)
Apply an illum/attached flat-field to a pixel table.
cpl_frameset * muse_frameset_check_raw(const cpl_frameset *aFrames, const cpl_array *aTags, unsigned char aIFU)
return frameset containing good raw input data
Definition: muse_utils.c:284
unsigned int size
cpl_error_code muse_image_adu_to_count(muse_image *aImage)
Convert the data units from raw adu to count (= electron) units.
Definition: muse_image.c:811
cpl_frameset * muse_frameset_find(const cpl_frameset *aFrames, const char *aTag, unsigned char aIFU, cpl_boolean aInvert)
return frameset containing data from an IFU/channel with a certain tag
Definition: muse_utils.c:160
cpl_parameter * muse_cplparamerterlist_find_prefix(cpl_parameterlist *aParameters, const char *aPrefix, const char *aName)
Return the full recipe parameter belonging to prefix and shortname.
muse_image * muse_image_load_from_extensions(const char *aFilename, unsigned char aIFU)
Load the three extensions and the FITS headers of a MUSE image from extensions of a merged file...
Definition: muse_image.c:262
#define MUSE_PIXTABLE_YPOS
Definition: muse_pixtable.h:52
void muse_pixtable_delete(muse_pixtable *aPixtable)
Deallocate memory associated to a pixel table object.
cpl_error_code muse_imagelist_set(muse_imagelist *aList, muse_image *aImage, unsigned int aIdx)
Set the muse_image of given list index.
const char * muse_pfits_get_chip_name(const cpl_propertylist *aHeaders)
find out the chip name
Definition: muse_pfits.c:584
cpl_frame * muse_frameset_find_master(const cpl_frameset *aFrames, const char *aTag, unsigned char aIFU)
find the master frame according to its CCD number and tag
Definition: muse_utils.c:505
muse_ins_mode muse_pfits_get_mode(const cpl_propertylist *aHeaders)
find out the observation mode
Definition: muse_pfits.c:1352
muse_image * muse_imagelist_unset(muse_imagelist *aList, unsigned int aIdx)
Unset the muse_image at given list index from the list.
cpl_parameterlist * parameters
muse_image * muse_quadrants_trim_image(muse_image *aImage)
Trim the input image of pre- and over-scan regions of all quadrants.
cpl_parameterlist * muse_cplparameterlist_from_propertylist(const cpl_propertylist *aHeader, int aRecNum)
Recreate a cpl_parameterlist from the RECi headers of an output MUSE product.
cpl_propertylist * header
The FITS header.
cpl_frameset * muse_frameset_find_tags(const cpl_frameset *aFrames, const cpl_array *aTags, unsigned char aIFU, cpl_boolean aInvert)
return frameset containing data from an IFU/channel with the given tag(s)
Definition: muse_utils.c:253
cpl_error_code muse_basicproc_qc_saturated(muse_image *aImage, const char *aPrefix)
Add QC parameter about saturated pixels to a muse_image.