Match_LOFAR_combined_generic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ML match for LOFAR and the combined PanSTARRS WISE catalogue: Generic matching code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configuration\n",
    "\n",
    "### Load libraries and setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from astropy.table import Table, join\n",
    "from astropy import units as u\n",
    "from astropy.coordinates import SkyCoord, search_around_sky\n",
    "from IPython.display import clear_output\n",
    "import pickle\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from mltier1 import (get_center, Field, MultiMLEstimatorU, parallel_process, get_sigma_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### General configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "save_intermediate = True\n",
    "plot_intermediate = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "idp = \"idata/gaus_pdf_v0.8\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if not os.path.isdir(idp):\n",
    "    os.makedirs(idp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Area limits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Busy week Edinburgh 2017\n",
    "ra_down = 172.09\n",
    "ra_up = 187.5833\n",
    "dec_down = 46.106\n",
    "dec_up = 56.1611"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Busy week Hatfield 2017\n",
    "ra_down = 170.\n",
    "ra_up = 190.\n",
    "dec_down = 46.8\n",
    "dec_up = 55.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Full field July 2017\n",
    "ra_down = 160.\n",
    "ra_up = 232.\n",
    "dec_down = 42.\n",
    "dec_up = 62."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "field = Field(170.0, 190.0, 46.8, 55.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "field_full = Field(160.0, 232.0, 42.0, 62.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined_all = Table.read(\"pw.fits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#lofar_all = Table.read(\"data/LOFAR_HBA_T1_DR1_catalog_v0.9.gaus.fits\")\n",
    "lofar_all = Table.read(\"data/LOFAR_HBA_T1_DR1_catalog_v0.99.gaus.fits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['AllWISE', 'objID', 'ra', 'dec', 'raErr', 'decErr', 'W1mag',\n",
       "       'W1magErr', 'i', 'iErr'],\n",
       "      dtype='<U8')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(combined_all.colnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Source_Name', 'RA', 'E_RA', 'E_RA_tot', 'DEC', 'E_DEC',\n",
       "       'E_DEC_tot', 'Peak_flux', 'E_Peak_flux', 'E_Peak_flux_tot',\n",
       "       'Total_flux', 'E_Total_flux', 'E_Total_flux_tot', 'Maj', 'E_Maj',\n",
       "       'Min', 'E_Min', 'DC_Maj', 'E_DC_Maj', 'DC_Min', 'E_DC_Min', 'PA',\n",
       "       'E_PA', 'DC_PA', 'E_DC_PA', 'Isl_rms', 'S_Code', 'Mosaic_ID',\n",
       "       'Isl_id', 'Gaus_id'],\n",
       "      dtype='<U16')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(lofar_all.colnames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter catalogues"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following line has been corrected in the latest versions to use all the sources, including the extended. Hence the running of the \"-extended\" version of this notebook is no longer necessary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lofar = field_full.filter_catalogue(lofar_all, colnames=(\"RA\", \"DEC\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined = field_full.filter_catalogue(combined_all, \n",
    "                               colnames=(\"ra\", \"dec\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Additional data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined[\"colour\"] = combined[\"i\"] - combined[\"W1mag\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined_aux_index = np.arange(len(combined))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sky coordinates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "coords_combined = SkyCoord(combined['ra'], \n",
    "                           combined['dec'], \n",
    "                           unit=(u.deg, u.deg), \n",
    "                           frame='icrs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "coords_lofar = SkyCoord(lofar['RA'], \n",
    "                       lofar['DEC'], \n",
    "                       unit=(u.deg, u.deg), \n",
    "                       frame='icrs')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Class of sources in the combined catalogue\n",
    "\n",
    "The sources are grouped depending on the available photometric data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined_matched = (~np.isnan(combined[\"i\"]) & ~np.isnan(combined[\"W1mag\"])) # Matched i-W1 sources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined_panstarrs = (~np.isnan(combined[\"i\"]) & np.isnan(combined[\"W1mag\"])) # Sources with only i-band"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined_wise =(np.isnan(combined[\"i\"]) & ~np.isnan(combined[\"W1mag\"])) # Sources with only W1-band"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined_i = combined_matched | combined_panstarrs\n",
    "combined_w1 = combined_matched | combined_wise\n",
    "#combined_only_i = combined_panstarrs & ~combined_matched\n",
    "#combined_only_w1 = combined_wise & ~combined_matched"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total    -  26674548\n",
      "i and W1 -  8196213\n",
      "Only i   -  13454849\n",
      "With i   -  21651062\n",
      "Only W1  -  5023475\n",
      "With W1  -  13219688\n"
     ]
    }
   ],
   "source": [
    "print(\"Total    - \", len(combined))\n",
    "print(\"i and W1 - \", np.sum(combined_matched))\n",
    "print(\"Only i   - \", np.sum(combined_panstarrs))\n",
    "print(\"With i   - \", np.sum(combined_i))\n",
    "print(\"Only W1  - \", np.sum(combined_wise))\n",
    "print(\"With W1  - \", np.sum(combined_w1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Colour categories\n",
    "\n",
    "The colour categories will be used after the first ML match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "colour_limits = [0.0, 0.5, 1.0, 1.25, 1.5, 1.75, 2.0, 2.25, 2.5, 2.75, 3.0, 3.5, 4.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/disk2/jsm/prog/anaconda/envs/py36/lib/python3.6/site-packages/astropy/table/column.py:928: RuntimeWarning: invalid value encountered in less\n",
      "  return getattr(self.data, oper)(other)\n",
      "/disk2/jsm/prog/anaconda/envs/py36/lib/python3.6/site-packages/astropy/table/column.py:928: RuntimeWarning: invalid value encountered in greater_equal\n",
      "  return getattr(self.data, oper)(other)\n"
     ]
    }
   ],
   "source": [
    "# Start with the W1-only, i-only and \"less than lower colour\" bins\n",
    "colour_bin_def = [{\"name\":\"only W1\", \"condition\": combined_wise},\n",
    "                  {\"name\":\"only i\", \"condition\": combined_panstarrs},\n",
    "                  {\"name\":\"-inf to {}\".format(colour_limits[0]), \n",
    "                   \"condition\": (combined[\"colour\"] < colour_limits[0])}]\n",
    "\n",
    "# Get the colour bins\n",
    "for i in range(len(colour_limits)-1):\n",
    "    name = \"{} to {}\".format(colour_limits[i], colour_limits[i+1])\n",
    "    condition = ((combined[\"colour\"] >= colour_limits[i]) & \n",
    "                 (combined[\"colour\"] < colour_limits[i+1]))\n",
    "    colour_bin_def.append({\"name\":name, \"condition\":condition})\n",
    "\n",
    "# Add the \"more than higher colour\" bin\n",
    "colour_bin_def.append({\"name\":\"{} to inf\".format(colour_limits[-1]), \n",
    "                       \"condition\": (combined[\"colour\"] >= colour_limits[-1])})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined[\"category\"] = np.nan\n",
    "for i in range(len(colour_bin_def)):\n",
    "    combined[\"category\"][colour_bin_def[i][\"condition\"]] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(np.isnan(combined[\"category\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We get the number of sources of the combined catalogue in each colour category. It will be used at a later stage to compute the $Q_0$ values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "numbers_combined_bins = np.array([np.sum(a[\"condition\"]) for a in colour_bin_def])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5023475, 13454849,  1048864,   679553,   911508,   654963,\n",
       "         774322,   830266,   804997,   713558,   580621,   438821,\n",
       "         309428,   319368,    95816,    34128])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numbers_combined_bins"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Maximum Likelihood"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bin_list, centers, Q_0_colour, n_m, q_m = pickle.load(open(\"lofar_params.pckl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "likelihood_ratio_function = MultiMLEstimatorU(Q_0_colour, n_m, q_m, centers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ML match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "radius = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "selection = ~np.isnan(combined[\"category\"]) # Avoid the dreaded sources with no actual data\n",
    "catalogue = combined[selection]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def apply_ml(i, likelihood_ratio_function):\n",
    "    idx_0 = idx_i[idx_lofar == i]\n",
    "    d2d_0 = d2d[idx_lofar == i]\n",
    "    \n",
    "    category = catalogue[\"category\"][idx_0].astype(int)\n",
    "    mag = catalogue[\"i\"][idx_0]\n",
    "    mag[category == 0] = catalogue[\"W1mag\"][idx_0][category == 0]\n",
    "    \n",
    "    lofar_ra = lofar[i][\"RA\"]\n",
    "    lofar_dec = lofar[i][\"DEC\"]\n",
    "    lofar_pa = lofar[i][\"PA\"]\n",
    "    lofar_maj_err = lofar[i][\"E_Maj\"]\n",
    "    lofar_min_err = lofar[i][\"E_Min\"]\n",
    "    c_ra = catalogue[\"ra\"][idx_0]\n",
    "    c_dec = catalogue[\"dec\"][idx_0]\n",
    "    c_ra_err = catalogue[\"raErr\"][idx_0]\n",
    "    c_dec_err = catalogue[\"decErr\"][idx_0]\n",
    "    \n",
    "    sigma, sigma_maj, sigma_min = get_sigma_all(lofar_maj_err, lofar_min_err, lofar_pa, \n",
    "                      lofar_ra, lofar_dec, \n",
    "                      c_ra, c_dec, c_ra_err, c_dec_err)\n",
    "\n",
    "    lr_0 = likelihood_ratio_function(mag, d2d_0.arcsec, sigma, sigma_maj, sigma_min, category)\n",
    "    \n",
    "    chosen_index = np.argmax(lr_0)\n",
    "    result = [combined_aux_index[selection][idx_0[chosen_index]], # Index\n",
    "              (d2d_0.arcsec)[chosen_index],                        # distance\n",
    "              lr_0[chosen_index]]                                  # LR\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the cross-match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "idx_lofar, idx_i, d2d, d3d = search_around_sky(\n",
    "    coords_lofar, coords_combined[selection], radius*u.arcsec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "idx_lofar_unique = np.unique(idx_lofar)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the ML matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ml(i):\n",
    "    return apply_ml(i, likelihood_ratio_function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 365K/365K [2:50:22<00:00, 35.7it/s]\n",
      "100%|██████████| 365440/365440 [00:01<00:00, 347861.73it/s]\n"
     ]
    }
   ],
   "source": [
    "res = parallel_process(idx_lofar_unique, ml, n_jobs=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lofar[\"lr\"] = np.nan                   # Likelihood ratio\n",
    "lofar[\"lr_dist\"] = np.nan              # Distance to the selected source\n",
    "lofar[\"lr_index\"] = np.nan             # Index of the PanSTARRS source in combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(lofar[\"lr_index\"][idx_lofar_unique], \n",
    " lofar[\"lr_dist\"][idx_lofar_unique], \n",
    " lofar[\"lr\"][idx_lofar_unique]) = list(map(list, zip(*res)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_sources = len(idx_lofar_unique)\n",
    "combined_aux_index = np.arange(len(combined))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Threshold and selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lofar[\"lrt\"] = lofar[\"lr\"]\n",
    "lofar[\"lrt\"][np.isnan(lofar[\"lr\"])] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "threshold_sel = 0.358"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lofar[\"lr_index_sel\"] = lofar[\"lr_index\"]\n",
    "lofar[\"lr_index_sel\"][lofar[\"lrt\"] < threshold_sel] = np.nan"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save combined catalogue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined[\"lr_index_sel\"] = combined_aux_index.astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pwl = join(lofar, combined, join_type='left', keys='lr_index_sel')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pwl_columns = pwl.colnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dec 1.0\n",
      "W1mag 1.0\n",
      "i 1.0\n",
      "colour 1.0\n",
      "category 1.0\n"
     ]
    }
   ],
   "source": [
    "for col in pwl_columns:\n",
    "    fv = pwl[col].fill_value\n",
    "    if (isinstance(fv, np.float64) and (fv != 1e+20)):\n",
    "        print(col, fv)\n",
    "        pwl[col].fill_value = 1e+20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columns_save = ['Source_Name', 'RA', 'E_RA', 'E_RA_tot', 'DEC', 'E_DEC',\n",
    "       'E_DEC_tot', 'Peak_flux', 'E_Peak_flux', 'E_Peak_flux_tot',\n",
    "       'Total_flux', 'E_Total_flux', 'E_Total_flux_tot', 'Maj', 'E_Maj',\n",
    "       'Min', 'E_Min', 'DC_Maj', 'E_DC_Maj', 'DC_Min', 'E_DC_Min', 'PA',\n",
    "       'E_PA', 'DC_PA', 'E_DC_PA', 'Isl_rms', 'S_Code', 'Mosaic_ID',\n",
    "       'Isl_id', 'Gaus_id',\n",
    "     'AllWISE', 'objID', 'ra', 'dec', 'raErr', 'decErr',\n",
    "     'W1mag', 'W1magErr', 'i', 'iErr', 'colour', 'category',\n",
    "     'lr', 'lr_dist']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pwl[columns_save].filled().write('lofar_gaus_pw_pdf.fits', format=\"fits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}