Process multiple IFS images in parallel

import time
import sys
codefolder = '../../../../crispy'
if codefolder not in sys.path: sys.path.append(codefolder)
from crispy.PISCESparams import Params
codefolder = '../../../crispy'

par = Params(codefolder)

# activate this if you want to get rid of all console messages
# but want to keep the file
# from tools.initLogger import getLogger
# t t

Import files to reduce, doesn’t matter how many

import glob
# call the reduction routine with an example image
filelist= glob.glob(par.wavecalDir+'IFS???nm.fits')
from crispy.tools.image import Image

No parallelization

# import parallel tools
from IFS import reduceIFSMap
start = time.time()
for filename in filelist:
    reduceIFSMap(par,filename)
print('Elapsed time: %fs' % (time.time()-start))

crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS605nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins

/Users/mrizzo/anaconda/lib/python2.7/site-packages/numpy/lib/nanfunctions.py:703: RuntimeWarning: Mean of empty slice
  warnings.warn("Mean of empty slice", RuntimeWarning)

crispy - INFO - Writing data to ../../code/SimResults/IFS605nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS615nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins

WARNING: AstropyDeprecationWarning: "clobber" was deprecated in version 1.3 and will be removed in a future version. Use argument "overwrite" instead. [astropy.utils.decorators]

crispy - INFO - Writing data to ../../code/SimResults/IFS615nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS625nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS625nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS635nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS635nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS645nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS645nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS655nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS655nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS665nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS665nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS675nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS675nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS685nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS685nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS695nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS695nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS705nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS705nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS715nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS715nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS725nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS725nm_red_optext.fits
Elapsed time: 201.540511s

Parallelized version

import multiprocessing
from tools.par_utils import Task, Consumer

start = time.time()

tasks = multiprocessing.Queue()
results = multiprocessing.Queue()
ncpus = multiprocessing.cpu_count()
consumers = [ Consumer(tasks, results)
              for i in range(ncpus) ]
for w in consumers:
    w.start()

# you call the function here, with all its arguments in a list
for i in range(len(filelist)):
        tasks.put(Task(i, reduceIFSMap, (par, filelist[i])))

for i in range(ncpus):
    tasks.put(None)

# if the function returns something, that's where you process them
for i in range(len(filelist)):
    index, result = results.get()
    # index tells us which task this was
    print("File %s processed" % filelist[index])
    # if the function you put in Task returned something, it is result

print('Elapsed time: %fs' % (time.time()-start))

crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS615nm.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS625nm.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS605nm.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS655nm.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS665nm.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS635nm.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS645nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS675nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS645nm_red_optext.fits
crispy - INFO - Writing data to ../../code/SimResults/IFS615nm_red_optext.fits
crispy - INFO - Writing data to ../../code/SimResults/IFS605nm_red_optext.fits
crispy - INFO - Writing data to ../../code/SimResults/IFS675nm_red_optext.fits
crispy - INFO - Writing data to ../../code/SimResults/IFS635nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS685nm.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS705nm.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS695nm.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS715nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS655nm_red_optext.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../code/SimResults/IFS625nm_red_optext.fits
crispy - INFO - Read data from HDU 0 of ../../code/ReferenceFiles/Calibra_170306/IFS725nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Reduced cube will have 17 wavelength bins
File ../../code/ReferenceFiles/Calibra_170306/IFS615nm.fits processed
File ../../code/ReferenceFiles/Calibra_170306/IFS645nm.fits processed
File ../../code/ReferenceFiles/Calibra_170306/IFS605nm.fits processed
File ../../code/ReferenceFiles/Calibra_170306/IFS675nm.fits processed
File ../../code/ReferenceFiles/Calibra_170306/IFS635nm.fits processed
File ../../code/ReferenceFiles/Calibra_170306/IFS655nm.fits processed
File ../../code/ReferenceFiles/Calibra_170306/IFS625nm.fits processed
crispy - INFO - Writing data to ../../code/SimResults/IFS665nm_red_optext.fits
File ../../code/ReferenceFiles/Calibra_170306/IFS665nm.fits processed
crispy - INFO - Writing data to ../../code/SimResults/IFS695nm_red_optext.fits
crispy - INFO - Writing data to ../../code/SimResults/IFS705nm_red_optext.fits
File ../../code/ReferenceFiles/Calibra_170306/IFS695nm.fits processed
File ../../code/ReferenceFiles/Calibra_170306/IFS705nm.fits processed
crispy - INFO - Writing data to ../../code/SimResults/IFS725nm_red_optext.fits
crispy - INFO - Writing data to ../../code/SimResults/IFS685nm_red_optext.fits
File ../../code/ReferenceFiles/Calibra_170306/IFS725nm.fits processed
File ../../code/ReferenceFiles/Calibra_170306/IFS685nm.fits processed
crispy - INFO - Writing data to ../../code/SimResults/IFS715nm_red_optext.fits
File ../../code/ReferenceFiles/Calibra_170306/IFS715nm.fits processed
Elapsed time: 45.754292s

Built-in parallelized reduction

We implemented a parallelized version of reduceIFSMap called reduceIFSMapList

from crispy.IFS import reduceIFSMapList
help(reduceIFSMapList)

Help on function reduceIFSMapList in module crispy.IFS:

reduceIFSMapList(par, IFSimageNameList, method='optext', parallel=True)
    Main reduction function

    Uses various routines to extract an IFS detector map into a spectral-spatial cube.

    Parameters
    ----------
    par :   Parameter instance
            Contains all IFS parameters
    IFSimageNameList : list
            List of strings containing the paths to the image files
    method : 'lstsq', 'optext'
            Method used for reduction.
            'lstsq': use the knowledge of the PSFs at each location and each wavelength and fits
            the microspectrum as a weighted sum of these PSFs in the least-square sense. Can weigh the data by its variance.
            'optext': use a matched filter to appropriately weigh each pixel and assign the fluxes, making use of the inverse
            wavlength calibration map. Then remap each microspectrum onto the desired wavelengths

import time
start= time.time()
reduceIFSMapList(par,filelist)
print('Elapsed time: %fs' % (time.time()-start))

crispy - INFO - Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS635nm.fits

INFO:crispy:Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS635nm.fits

crispy - INFO - Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS605nm.fits

INFO:crispy:Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS605nm.fits

crispy - INFO - Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS655nm.fits

INFO:crispy:Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS655nm.fits

crispy - INFO - Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS695nm.fits

INFO:crispy:Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS695nm.fits

crispy - INFO - Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS645nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Reduced cube will have 17 wavelength bins

INFO:crispy:Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS645nm.fits
INFO:crispy:Reduced cube will have 17 wavelength bins

crispy - INFO - Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS705nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins

INFO:crispy:Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS705nm.fits
INFO:crispy:Reduced cube will have 17 wavelength bins

crispy - INFO - Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS715nm.fits

INFO:crispy:Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS715nm.fits
INFO:crispy:Reduced cube will have 17 wavelength bins

crispy - INFO - Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS675nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins

INFO:crispy:Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS675nm.fits
INFO:crispy:Reduced cube will have 17 wavelength bins

crispy - INFO - Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS625nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Reduced cube will have 17 wavelength bins

INFO:crispy:Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS625nm.fits
INFO:crispy:Reduced cube will have 17 wavelength bins

crispy - INFO - Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS665nm.fits

INFO:crispy:Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS665nm.fits
INFO:crispy:Reduced cube will have 17 wavelength bins

crispy - INFO - Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS615nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins

INFO:crispy:Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS615nm.fits
INFO:crispy:Reduced cube will have 17 wavelength bins

crispy - INFO - Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS685nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins

INFO:crispy:Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS685nm.fits
INFO:crispy:Reduced cube will have 17 wavelength bins

crispy - INFO - Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS725nm.fits
crispy - INFO - Reduced cube will have 17 wavelength bins

INFO:crispy:Read data from HDU 0 of ../../../crispy/ReferenceFiles/Calibra_170306/IFS725nm.fits

crispy - INFO - Elapsed time: 49.567595s
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../../crispy/SimResults/IFS695nm_red_optext.fits
crispy - INFO - Writing data to ../../../crispy/SimResults/IFS605nm_red_optext.fits
crispy - INFO - Writing data to ../../../crispy/SimResults/IFS645nm_red_optext.fits
crispy - INFO - Writing data to ../../../crispy/SimResults/IFS625nm_red_optext.fits
crispy - INFO - Writing data to ../../../crispy/SimResults/IFS655nm_red_optext.fits
crispy - INFO - Writing data to ../../../crispy/SimResults/IFS635nm_red_optext.fits
crispy - INFO - Writing data to ../../../crispy/SimResults/IFS705nm_red_optext.fits
crispy - INFO - Writing data to ../../../crispy/SimResults/IFS715nm_red_optext.fits
crispy - INFO - Writing data to ../../../crispy/SimResults/IFS675nm_red_optext.fits
crispy - INFO - Writing data to ../../../crispy/SimResults/IFS665nm_red_optext.fits
crispy - INFO - Writing data to ../../../crispy/SimResults/IFS615nm_red_optext.fits
crispy - INFO - Writing data to ../../../crispy/SimResults/IFS685nm_red_optext.fits
crispy - INFO - Reduced cube will have 17 wavelength bins
crispy - INFO - Writing data to ../../../crispy/SimResults/IFS725nm_red_optext.fits

INFO:crispy:Elapsed time: 49.567595s
INFO:crispy:Reduced cube will have 17 wavelength bins
INFO:crispy:Reduced cube will have 17 wavelength bins
INFO:crispy:Reduced cube will have 17 wavelength bins
INFO:crispy:Reduced cube will have 17 wavelength bins
/local/data/nicolaus2/mrizzo/anaconda/lib/python2.7/site-packages/numpy/lib/nanfunctions.py:675: RuntimeWarning: Mean of empty slice
  warnings.warn("Mean of empty slice", RuntimeWarning)
/local/data/nicolaus2/mrizzo/anaconda/lib/python2.7/site-packages/numpy/lib/nanfunctions.py:675: RuntimeWarning: Mean of empty slice
  warnings.warn("Mean of empty slice", RuntimeWarning)
/local/data/nicolaus2/mrizzo/anaconda/lib/python2.7/site-packages/numpy/lib/nanfunctions.py:675: RuntimeWarning: Mean of empty slice
  warnings.warn("Mean of empty slice", RuntimeWarning)
/local/data/nicolaus2/mrizzo/anaconda/lib/python2.7/site-packages/numpy/lib/nanfunctions.py:675: RuntimeWarning: Mean of empty slice
  warnings.warn("Mean of empty slice", RuntimeWarning)
/local/data/nicolaus2/mrizzo/anaconda/lib/python2.7/site-packages/numpy/lib/nanfunctions.py:675: RuntimeWarning: Mean of empty slice
  warnings.warn("Mean of empty slice", RuntimeWarning)
/local/data/nicolaus2/mrizzo/anaconda/lib/python2.7/site-packages/numpy/lib/nanfunctions.py:675: RuntimeWarning: Mean of empty slice
  warnings.warn("Mean of empty slice", RuntimeWarning)
/local/data/nicolaus2/mrizzo/anaconda/lib/python2.7/site-packages/numpy/lib/nanfunctions.py:675: RuntimeWarning: Mean of empty slice
  warnings.warn("Mean of empty slice", RuntimeWarning)
/local/data/nicolaus2/mrizzo/anaconda/lib/python2.7/site-packages/numpy/lib/nanfunctions.py:675: RuntimeWarning: Mean of empty slice
  warnings.warn("Mean of empty slice", RuntimeWarning)
INFO:crispy:Writing data to ../../../crispy/SimResults/IFS605nm_red_optext.fits
INFO:crispy:Writing data to ../../../crispy/SimResults/IFS645nm_red_optext.fits
INFO:crispy:Writing data to ../../../crispy/SimResults/IFS635nm_red_optext.fits
INFO:crispy:Writing data to ../../../crispy/SimResults/IFS655nm_red_optext.fits
INFO:crispy:Writing data to ../../../crispy/SimResults/IFS695nm_red_optext.fits
INFO:crispy:Writing data to ../../../crispy/SimResults/IFS705nm_red_optext.fits
INFO:crispy:Writing data to ../../../crispy/SimResults/IFS715nm_red_optext.fits
INFO:crispy:Writing data to ../../../crispy/SimResults/IFS675nm_red_optext.fits
/local/data/nicolaus2/mrizzo/anaconda/lib/python2.7/site-packages/numpy/lib/nanfunctions.py:675: RuntimeWarning: Mean of empty slice
  warnings.warn("Mean of empty slice", RuntimeWarning)
/local/data/nicolaus2/mrizzo/anaconda/lib/python2.7/site-packages/numpy/lib/nanfunctions.py:675: RuntimeWarning: Mean of empty slice
  warnings.warn("Mean of empty slice", RuntimeWarning)
/local/data/nicolaus2/mrizzo/anaconda/lib/python2.7/site-packages/numpy/lib/nanfunctions.py:675: RuntimeWarning: Mean of empty slice
  warnings.warn("Mean of empty slice", RuntimeWarning)
/local/data/nicolaus2/mrizzo/anaconda/lib/python2.7/site-packages/numpy/lib/nanfunctions.py:675: RuntimeWarning: Mean of empty slice
  warnings.warn("Mean of empty slice", RuntimeWarning)
INFO:crispy:Writing data to ../../../crispy/SimResults/IFS625nm_red_optext.fits
INFO:crispy:Writing data to ../../../crispy/SimResults/IFS665nm_red_optext.fits
INFO:crispy:Writing data to ../../../crispy/SimResults/IFS615nm_red_optext.fits
INFO:crispy:Writing data to ../../../crispy/SimResults/IFS685nm_red_optext.fits
INFO:crispy:Reduced cube will have 17 wavelength bins
INFO:crispy:Writing data to ../../../crispy/SimResults/IFS725nm_red_optext.fits

Elapsed time: 49.575726s