Match_LOFAR_combined_above-threshold.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ML match for LOFAR and the combined PanSTARRS WISE catalogue: Get all sources above the threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook the maximum likelihood cross-match between the LOFAR HETDEX catalogue and the combined PansSTARRS WISE catalogue is computed. However, we take all the sources that have likelihood ratios above the threshold found in the first run."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configuration\n",
    "\n",
    "### Load libraries and setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from astropy.table import Table, join\n",
    "from astropy import units as u\n",
    "from astropy.coordinates import SkyCoord, search_around_sky\n",
    "from IPython.display import clear_output\n",
    "import pickle\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from mltier1 import (get_center, get_n_m, estimate_q_m, Field, MultiMLEstimatorU,\n",
    "                     parallel_process, get_sigma_all, get_q_m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### General configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "save_intermediate = True\n",
    "plot_intermediate = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "idp = \"idata/final\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if not os.path.isdir(idp):\n",
    "    os.makedirs(idp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Area limits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Busy week Edinburgh 2017\n",
    "ra_down = 172.09\n",
    "ra_up = 187.5833\n",
    "dec_down = 46.106\n",
    "dec_up = 56.1611"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Busy week Hatfield 2017\n",
    "ra_down = 170.\n",
    "ra_up = 190.\n",
    "dec_down = 46.8\n",
    "dec_up = 55.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Full field July 2017\n",
    "ra_down = 160.\n",
    "ra_up = 232.\n",
    "dec_down = 42.\n",
    "dec_up = 62."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "field = Field(170.0, 190.0, 46.8, 55.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "field_full = Field(160.0, 232.0, 42.0, 62.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined_all = Table.read(\"pw.fits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lofar_all = Table.read(\"data/LOFAR_HBA_T1_DR1_catalog_v0.9.srl.fits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['AllWISE', 'objID', 'ra', 'dec', 'raErr', 'decErr', 'W1mag',\n",
       "       'W1magErr', 'i', 'iErr'],\n",
       "      dtype='<U8')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(combined_all.colnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Source_Name', 'RA', 'E_RA', 'E_RA_tot', 'DEC', 'E_DEC',\n",
       "       'E_DEC_tot', 'Peak_flux', 'E_Peak_flux', 'E_Peak_flux_tot',\n",
       "       'Total_flux', 'E_Total_flux', 'E_Total_flux_tot', 'Maj', 'E_Maj',\n",
       "       'Min', 'E_Min', 'PA', 'E_PA', 'Isl_rms', 'S_Code', 'Mosaic_ID',\n",
       "       'Isl_id'],\n",
       "      dtype='<U16')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(lofar_all.colnames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter catalogues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lofar = field_full.filter_catalogue(lofar_all, colnames=(\"RA\", \"DEC\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined = field_full.filter_catalogue(combined_all, \n",
    "                               colnames=(\"ra\", \"dec\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Additional data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined[\"colour\"] = combined[\"i\"] - combined[\"W1mag\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined_aux_index = np.arange(len(combined))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sky coordinates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "coords_combined = SkyCoord(combined['ra'], \n",
    "                           combined['dec'], \n",
    "                           unit=(u.deg, u.deg), \n",
    "                           frame='icrs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "coords_lofar = SkyCoord(lofar['RA'], \n",
    "                       lofar['DEC'], \n",
    "                       unit=(u.deg, u.deg), \n",
    "                       frame='icrs')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Class of sources in the combined catalogue\n",
    "\n",
    "The sources are grouped depending on the available photometric data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined_matched = (~np.isnan(combined[\"i\"]) & ~np.isnan(combined[\"W1mag\"])) # Matched i-W1 sources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined_panstarrs = (~np.isnan(combined[\"i\"]) & np.isnan(combined[\"W1mag\"])) # Sources with only i-band"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined_wise =(np.isnan(combined[\"i\"]) & ~np.isnan(combined[\"W1mag\"])) # Sources with only W1-band"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined_i = combined_matched | combined_panstarrs\n",
    "combined_w1 = combined_matched | combined_wise\n",
    "#combined_only_i = combined_panstarrs & ~combined_matched\n",
    "#combined_only_w1 = combined_wise & ~combined_matched"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total    -  26674548\n",
      "i and W1 -  8196213\n",
      "Only i   -  13454849\n",
      "With i   -  21651062\n",
      "Only W1  -  5023475\n",
      "With W1  -  13219688\n"
     ]
    }
   ],
   "source": [
    "print(\"Total    - \", len(combined))\n",
    "print(\"i and W1 - \", np.sum(combined_matched))\n",
    "print(\"Only i   - \", np.sum(combined_panstarrs))\n",
    "print(\"With i   - \", np.sum(combined_i))\n",
    "print(\"Only W1  - \", np.sum(combined_wise))\n",
    "print(\"With W1  - \", np.sum(combined_w1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Colour categories\n",
    "\n",
    "The colour categories will be used after the first ML match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "colour_limits = [-0.5, 0.0, 0.5, 1.0, 1.25, 1.5, 1.75, 2.0, 2.25, 2.5, 2.75, 3.0, 3.5, 4.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/disk2/jsm/prog/anaconda/envs/py36/lib/python3.6/site-packages/astropy/table/column.py:928: RuntimeWarning: invalid value encountered in less\n",
      "  return getattr(self.data, oper)(other)\n",
      "/disk2/jsm/prog/anaconda/envs/py36/lib/python3.6/site-packages/astropy/table/column.py:928: RuntimeWarning: invalid value encountered in greater_equal\n",
      "  return getattr(self.data, oper)(other)\n"
     ]
    }
   ],
   "source": [
    "# Start with the W1-only, i-only and \"less than lower colour\" bins\n",
    "colour_bin_def = [{\"name\":\"only W1\", \"condition\": combined_wise},\n",
    "                  {\"name\":\"only i\", \"condition\": combined_panstarrs},\n",
    "                  {\"name\":\"-inf to {}\".format(colour_limits[0]), \n",
    "                   \"condition\": (combined[\"colour\"] < colour_limits[0])}]\n",
    "\n",
    "# Get the colour bins\n",
    "for i in range(len(colour_limits)-1):\n",
    "    name = \"{} to {}\".format(colour_limits[i], colour_limits[i+1])\n",
    "    condition = ((combined[\"colour\"] >= colour_limits[i]) & \n",
    "                 (combined[\"colour\"] < colour_limits[i+1]))\n",
    "    colour_bin_def.append({\"name\":name, \"condition\":condition})\n",
    "\n",
    "# Add the \"more than higher colour\" bin\n",
    "colour_bin_def.append({\"name\":\"{} to inf\".format(colour_limits[-1]), \n",
    "                       \"condition\": (combined[\"colour\"] >= colour_limits[-1])})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined[\"category\"] = np.nan\n",
    "for i in range(len(colour_bin_def)):\n",
    "    combined[\"category\"][colour_bin_def[i][\"condition\"]] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(np.isnan(combined[\"category\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We get the number of sources of the combined catalogue in each colour category. It will be used at a later stage to compute the $Q_0$ values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "numbers_combined_bins = np.array([np.sum(a[\"condition\"]) for a in colour_bin_def])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5023475, 13454849,   614840,   434024,   679553,   911508,\n",
       "         654963,   774322,   830266,   804997,   713558,   580621,\n",
       "         438821,   309428,   319368,    95816,    34128])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numbers_combined_bins"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Maximum Likelihood"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bin_list, centers, Q_0_colour, n_m, q_m = pickle.load(open(\"lofar_params.pckl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "likelihood_ratio_function = MultiMLEstimator(Q_0_colour, n_m, q_m, centers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ML match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "radius = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr_threshold = 0.36"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "selection = ~np.isnan(combined[\"category\"]) # Avoid the dreaded sources with no actual data\n",
    "catalogue = combined[selection]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def apply_ml(i, likelihood_ratio_function):\n",
    "    idx_0 = idx_i[idx_lofar == i]\n",
    "    d2d_0 = d2d[idx_lofar == i]\n",
    "    \n",
    "    category = catalogue[\"category\"][idx_0].astype(int)\n",
    "    mag = catalogue[\"i\"][idx_0]\n",
    "    mag[category == 0] = catalogue[\"W1mag\"][idx_0][category == 0]\n",
    "    \n",
    "    lofar_ra = lofar[i][\"RA\"]\n",
    "    lofar_dec = lofar[i][\"DEC\"]\n",
    "    lofar_pa = lofar[i][\"PA\"]\n",
    "    lofar_maj_err = lofar[i][\"E_Maj\"]\n",
    "    lofar_min_err = lofar[i][\"E_Min\"]\n",
    "    c_ra = catalogue[\"ra\"][idx_0]\n",
    "    c_dec = catalogue[\"dec\"][idx_0]\n",
    "    c_ra_err = catalogue[\"raErr\"][idx_0]\n",
    "    c_dec_err = catalogue[\"decErr\"][idx_0]\n",
    "    \n",
    "    sigma, sigma_maj, sigma_min = get_sigma_all(lofar_maj_err, lofar_min_err, lofar_pa, \n",
    "                      lofar_ra, lofar_dec, \n",
    "                      c_ra, c_dec, c_ra_err, c_dec_err)\n",
    "\n",
    "    lr_0 = likelihood_ratio_function(mag, d2d_0.arcsec, sigma, sigma_maj, sigma_min, category)\n",
    "    \n",
    "    chosen_index = (lr_0 >= lr_threshold)\n",
    "    result = [combined_aux_index[selection][idx_0[chosen_index]], # Index\n",
    "              (d2d_0.arcsec)[chosen_index],                        # distance\n",
    "              lr_0[chosen_index]]                                  # LR\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the cross-match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "idx_lofar, idx_i, d2d, d3d = search_around_sky(\n",
    "    coords_lofar, coords_combined[selection], radius*u.arcsec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "idx_lofar_unique = np.unique(idx_lofar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_sources = len(idx_lofar_unique)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the ML matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ml(i):\n",
    "    return apply_ml(i, likelihood_ratio_function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 312K/312K [2:28:19<00:00, 35.0it/s]\n",
      "100%|██████████| 311874/311874 [00:00<00:00, 335946.24it/s]\n"
     ]
    }
   ],
   "source": [
    "res = parallel_process(idx_lofar_unique, ml, n_jobs=8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Selection and match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lofar_aux_index = np.arange(len(lofar))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = []\n",
    "lr_dist = []\n",
    "lr_index = []\n",
    "lr_order = []\n",
    "lr_lofar_index = []\n",
    "\n",
    "for i, idx in enumerate(idx_lofar_unique):\n",
    "    result = res[i]\n",
    "    n = len(result[0])\n",
    "    lofar_index = lofar_aux_index[idx]\n",
    "    if n > 0:\n",
    "        order = np.argsort(result[2])[::-1]\n",
    "        lr.extend(result[2][order])\n",
    "        lr_dist.extend(result[1][order])\n",
    "        lr_index.extend(result[0][order])\n",
    "        lr_order.extend(np.arange(n, dtype=int) + 1)\n",
    "        lr_lofar_index.extend(np.ones(n, dtype=int)*lofar_index)\n",
    "    else:\n",
    "        lr.append(np.nan)\n",
    "        lr_dist.append(np.nan)\n",
    "        lr_index.append(np.nan)\n",
    "        lr_order.append(np.nan)\n",
    "        lr_lofar_index.append(lofar_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aux_table = Table()\n",
    "aux_table['aux_index'] = lr_lofar_index\n",
    "aux_table['lr'] = lr\n",
    "aux_table['lr_dist'] = lr_dist\n",
    "aux_table['lr_index'] = lr_index\n",
    "aux_table['lr_order'] = lr_order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "&lt;Table length=325770&gt;\n",
       "<table id=\"table140529238978400\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>aux_index</th><th>lr</th><th>lr_dist</th><th>lr_index</th><th>lr_order</th></tr></thead>\n",
       "<thead><tr><th>int64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th></tr></thead>\n",
       "<tr><td>0</td><td>1224.06561556</td><td>0.256393266029</td><td>10243809.0</td><td>1.0</td></tr>\n",
       "<tr><td>1</td><td>359.378211742</td><td>0.55383604885</td><td>10206441.0</td><td>1.0</td></tr>\n",
       "<tr><td>2</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td></tr>\n",
       "<tr><td>3</td><td>1719.36703</td><td>0.212102609363</td><td>10195667.0</td><td>1.0</td></tr>\n",
       "<tr><td>4</td><td>529.002075527</td><td>0.300592058961</td><td>10171052.0</td><td>1.0</td></tr>\n",
       "<tr><td>5</td><td>0.580790135351</td><td>2.24311275138</td><td>10318938.0</td><td>1.0</td></tr>\n",
       "<tr><td>6</td><td>614.176429091</td><td>0.181201605232</td><td>10320755.0</td><td>1.0</td></tr>\n",
       "<tr><td>7</td><td>112.009893324</td><td>1.2520854396</td><td>10216972.0</td><td>1.0</td></tr>\n",
       "<tr><td>8</td><td>270.392988058</td><td>1.20606318915</td><td>10339350.0</td><td>1.0</td></tr>\n",
       "<tr><td>8</td><td>5.67326137866</td><td>1.07704111747</td><td>10339351.0</td><td>2.0</td></tr>\n",
       "<tr><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td></tr>\n",
       "<tr><td>325688</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td></tr>\n",
       "<tr><td>325689</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td></tr>\n",
       "<tr><td>325690</td><td>0.48500595098</td><td>13.2393295354</td><td>14562507.0</td><td>1.0</td></tr>\n",
       "<tr><td>325690</td><td>0.371481680061</td><td>10.0441360907</td><td>26521592.0</td><td>2.0</td></tr>\n",
       "<tr><td>325691</td><td>1.5333394225</td><td>0.359093118484</td><td>26352886.0</td><td>1.0</td></tr>\n",
       "<tr><td>325691</td><td>0.737285833627</td><td>8.36125539124</td><td>26275082.0</td><td>2.0</td></tr>\n",
       "<tr><td>325692</td><td>1.4100985987</td><td>8.94180949023</td><td>14393277.0</td><td>1.0</td></tr>\n",
       "<tr><td>325692</td><td>0.579629471613</td><td>10.820825309</td><td>26657452.0</td><td>2.0</td></tr>\n",
       "<tr><td>325693</td><td>2.8081435628</td><td>10.6355466372</td><td>14625420.0</td><td>1.0</td></tr>\n",
       "<tr><td>325693</td><td>0.55398818036</td><td>10.880053618</td><td>14652394.0</td><td>2.0</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table length=325770>\n",
       "aux_index       lr          lr_dist      lr_index  lr_order\n",
       "  int64      float64        float64      float64   float64 \n",
       "--------- -------------- -------------- ---------- --------\n",
       "        0  1224.06561556 0.256393266029 10243809.0      1.0\n",
       "        1  359.378211742  0.55383604885 10206441.0      1.0\n",
       "        2            nan            nan        nan      nan\n",
       "        3     1719.36703 0.212102609363 10195667.0      1.0\n",
       "        4  529.002075527 0.300592058961 10171052.0      1.0\n",
       "        5 0.580790135351  2.24311275138 10318938.0      1.0\n",
       "        6  614.176429091 0.181201605232 10320755.0      1.0\n",
       "        7  112.009893324   1.2520854396 10216972.0      1.0\n",
       "        8  270.392988058  1.20606318915 10339350.0      1.0\n",
       "        8  5.67326137866  1.07704111747 10339351.0      2.0\n",
       "      ...            ...            ...        ...      ...\n",
       "   325688            nan            nan        nan      nan\n",
       "   325689            nan            nan        nan      nan\n",
       "   325690  0.48500595098  13.2393295354 14562507.0      1.0\n",
       "   325690 0.371481680061  10.0441360907 26521592.0      2.0\n",
       "   325691   1.5333394225 0.359093118484 26352886.0      1.0\n",
       "   325691 0.737285833627  8.36125539124 26275082.0      2.0\n",
       "   325692   1.4100985987  8.94180949023 14393277.0      1.0\n",
       "   325692 0.579629471613   10.820825309 26657452.0      2.0\n",
       "   325693   2.8081435628  10.6355466372 14625420.0      1.0\n",
       "   325693  0.55398818036   10.880053618 14652394.0      2.0"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aux_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lofar[\"aux_index\"] = lofar_aux_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lofar_lr = join(lofar, aux_table, join_type='outer', keys='aux_index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "&lt;Table masked=True length=339587&gt;\n",
       "<table id=\"table140529285528880\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>Source_Name</th><th>RA</th><th>E_RA</th><th>E_RA_tot</th><th>DEC</th><th>E_DEC</th><th>E_DEC_tot</th><th>Peak_flux</th><th>E_Peak_flux</th><th>E_Peak_flux_tot</th><th>Total_flux</th><th>E_Total_flux</th><th>E_Total_flux_tot</th><th>Maj</th><th>E_Maj</th><th>Min</th><th>E_Min</th><th>PA</th><th>E_PA</th><th>Isl_rms</th><th>S_Code</th><th>Mosaic_ID</th><th>Isl_id</th><th>aux_index</th><th>lr</th><th>lr_dist</th><th>lr_index</th><th>lr_order</th></tr></thead>\n",
       "<thead><tr><th></th><th>deg</th><th>arcsec</th><th>arcsec</th><th>deg</th><th>arcsec</th><th>arcsec</th><th>mJy / beam</th><th>mJy / beam</th><th>mJy / beam</th><th>mJy</th><th>mJy</th><th>mJy</th><th>arcsec</th><th>arcsec</th><th>arcsec</th><th>arcsec</th><th>deg</th><th>deg</th><th>mJy / beam</th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th></tr></thead>\n",
       "<thead><tr><th>str24</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th><th>str1</th><th>str8</th><th>int32</th><th>int64</th><th>float64</th><th>float64</th><th>float64</th><th>float64</th></tr></thead>\n",
       "<tr><td>ILTJ122108.441+491340.94</td><td>185.285168771</td><td>0.0850700841174</td><td>0.150016049008</td><td>49.2280397678</td><td>0.094025008434</td><td>0.155269436656</td><td>1.48718776646</td><td>0.0525447165803</td><td>0.30204311836</td><td>1.45167723509</td><td>0.0918727162887</td><td>0.304524658761</td><td>6.19600662707</td><td>0.228961053509</td><td>5.67333350682</td><td>0.191652766315</td><td>153.935376162</td><td>18.4908886458</td><td>0.0529464996362</td><td>S</td><td>P22Hetde</td><td>188</td><td>0</td><td>1224.06561556</td><td>0.256393266029</td><td>10243809.0</td><td>1.0</td></tr>\n",
       "<tr><td>ILTJ122107.851+491951.84</td><td>185.282710884</td><td>0.473938135007</td><td>0.489780819921</td><td>49.3310690203</td><td>0.474978105451</td><td>0.490787221111</td><td>0.494206901892</td><td>0.0658548772285</td><td>0.118770717476</td><td>0.847902990364</td><td>0.0975490731081</td><td>0.195635888597</td><td>9.12706565844</td><td>1.35780957336</td><td>6.76932220205</td><td>0.807971003466</td><td>46.4592997429</td><td>23.1085152836</td><td>0.0616828474449</td><td>S</td><td>P22Hetde</td><td>196</td><td>1</td><td>359.378211742</td><td>0.55383604885</td><td>10206441.0</td><td>1.0</td></tr>\n",
       "<tr><td>ILTJ122108.572+493121.89</td><td>185.285716341</td><td>0.533848043611</td><td>0.547961339344</td><td>49.52274915</td><td>0.367264254936</td><td>0.387493133749</td><td>0.357484003783</td><td>0.0532574812958</td><td>0.0891524078889</td><td>0.438294187244</td><td>0.087484556291</td><td>0.123845142708</td><td>7.79035197589</td><td>1.33867766644</td><td>5.66738515187</td><td>0.732001245106</td><td>67.3769455634</td><td>23.8529803004</td><td>0.0519686545886</td><td>S</td><td>P22Hetde</td><td>202</td><td>2</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td></tr>\n",
       "<tr><td>ILTJ122104.027+490823.80</td><td>185.266778578</td><td>0.0680609727689</td><td>0.141068039479</td><td>49.1399451347</td><td>0.0689599603528</td><td>0.141503964185</td><td>2.18501193876</td><td>0.0561302300848</td><td>0.440592430292</td><td>2.39268273523</td><td>0.0943671437026</td><td>0.487752380486</td><td>6.30982940769</td><td>0.162893477077</td><td>6.24977288427</td><td>0.159758583039</td><td>158.418653032</td><td>107.214019258</td><td>0.0549563665118</td><td>S</td><td>P22Hetde</td><td>204</td><td>3</td><td>1719.36703</td><td>0.212102609363</td><td>10195667.0</td><td>1.0</td></tr>\n",
       "<tr><td>ILTJ122105.030+491440.01</td><td>185.270960099</td><td>0.585020092161</td><td>0.597926754695</td><td>49.2444492774</td><td>0.508482783423</td><td>0.523280648205</td><td>0.291824013394</td><td>0.0552949639572</td><td>0.0803989006813</td><td>0.350985385828</td><td>0.0906813300159</td><td>0.114676646515</td><td>7.2605736876</td><td>1.50440230517</td><td>5.96540847045</td><td>1.03365648739</td><td>125.162791534</td><td>44.7934049203</td><td>0.0535216240678</td><td>S</td><td>P22Hetde</td><td>205</td><td>4</td><td>529.002075527</td><td>0.300592058961</td><td>10171052.0</td><td>1.0</td></tr>\n",
       "<tr><td>ILTJ122106.722+492420.20</td><td>185.278009798</td><td>0.476205256172</td><td>0.491974940169</td><td>49.4056116521</td><td>0.363568823749</td><td>0.383992428767</td><td>0.308482936742</td><td>0.0504988664517</td><td>0.0797283161953</td><td>0.306304270094</td><td>0.0879554002244</td><td>0.107186961258</td><td>6.40838974909</td><td>1.12431943609</td><td>5.57965677846</td><td>0.852251276603</td><td>85.3961636616</td><td>52.8570913248</td><td>0.0507702061441</td><td>S</td><td>P22Hetde</td><td>206</td><td>5</td><td>0.580790135351</td><td>2.24311275138</td><td>10318938.0</td><td>1.0</td></tr>\n",
       "<tr><td>ILTJ122106.172+492908.04</td><td>185.275717593</td><td>0.183850706166</td><td>0.221515186626</td><td>49.4855672648</td><td>0.257120714935</td><td>0.285269973528</td><td>0.641852509513</td><td>0.0532738711074</td><td>0.138985938503</td><td>0.671735160982</td><td>0.091377926785</td><td>0.162477846378</td><td>6.73049491464</td><td>0.611268985317</td><td>5.59946675829</td><td>0.424736261495</td><td>12.4682229602</td><td>21.5508992193</td><td>0.0530273173354</td><td>S</td><td>P22Hetde</td><td>210</td><td>6</td><td>614.176429091</td><td>0.181201605232</td><td>10320755.0</td><td>1.0</td></tr>\n",
       "<tr><td>ILTJ122104.649+492207.60</td><td>185.269372038</td><td>0.515425109037</td><td>0.530029186719</td><td>49.3687791357</td><td>0.37925775751</td><td>0.398878856772</td><td>0.349770333809</td><td>0.0544436946502</td><td>0.088643597308</td><td>0.358176497234</td><td>0.0955404857905</td><td>0.119413569379</td><td>7.27808684182</td><td>1.354813951</td><td>5.06678218192</td><td>0.659476070658</td><td>122.170405666</td><td>22.0608157459</td><td>0.0554672114959</td><td>S</td><td>P22Hetde</td><td>211</td><td>7</td><td>112.009893324</td><td>1.2520854396</td><td>10216972.0</td><td>1.0</td></tr>\n",
       "<tr><td>ILTJ122107.312+493759.86</td><td>185.280465285</td><td>0.574867040653</td><td>0.587996607284</td><td>49.6332943738</td><td>0.81022683125</td><td>0.819594664346</td><td>0.301244883956</td><td>0.0569938555714</td><td>0.0829351480211</td><td>0.528145587087</td><td>0.0841966227453</td><td>0.135079908677</td><td>9.55625229668</td><td>2.06868702874</td><td>6.60653434819</td><td>1.09240387499</td><td>154.585287717</td><td>27.2674448446</td><td>0.0537575579074</td><td>S</td><td>P22Hetde</td><td>212</td><td>8</td><td>270.392988058</td><td>1.20606318915</td><td>10339350.0</td><td>1.0</td></tr>\n",
       "<tr><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td></tr>\n",
       "<tr><td>ILTJ134709.873+494747.92</td><td>206.791138823</td><td>10.2608863243</td><td>10.2621447879</td><td>49.796645035</td><td>10.4509355437</td><td>10.452171125</td><td>0.136857475655</td><td>0.0801158374039</td><td>0.0846625427721</td><td>6.59446034754</td><td>0.0817616106204</td><td>1.32142394862</td><td>42.282220218</td><td>24.7670680073</td><td>41.0258078805</td><td>24.0015909953</td><td>26.9549901434</td><td>179.846450053</td><td>0.202775700018</td><td>S</td><td>P206+50</td><td>9964</td><td>325688</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td></tr>\n",
       "<tr><td>ILTJ134632.301+485421.18</td><td>206.634588782</td><td>6.09050055912</td><td>6.09262049928</td><td>48.9058837738</td><td>8.20323694906</td><td>8.20481102342</td><td>0.0597379590874</td><td>0.0196921954311</td><td>0.0230331828268</td><td>4.18135735789</td><td>0.0199715633002</td><td>0.836509914774</td><td>59.0972313295</td><td>19.5695548958</td><td>42.6451760327</td><td>13.9940561139</td><td>166.42997423</td><td>55.2200606894</td><td>0.0595674064243</td><td>S</td><td>P206+50</td><td>9965</td><td>325689</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td></tr>\n",
       "<tr><td>ILTJ134454.662+491951.32</td><td>206.227759779</td><td>8.71976713446</td><td>8.72124798219</td><td>49.3309227454</td><td>10.0949662552</td><td>10.0962454002</td><td>0.0578780941924</td><td>0.0252661695745</td><td>0.027791622415</td><td>3.61793467655</td><td>0.0256672432086</td><td>0.724042029392</td><td>61.6611012311</td><td>27.1506478165</td><td>36.4984671231</td><td>15.7963254274</td><td>35.8648709586</td><td>53.3979794776</td><td>0.0725644931663</td><td>S</td><td>P206+50</td><td>9967</td><td>325690</td><td>0.371481680061</td><td>10.0441360907</td><td>26521592.0</td><td>2.0</td></tr>\n",
       "<tr><td>ILTJ134454.662+491951.32</td><td>206.227759779</td><td>8.71976713446</td><td>8.72124798219</td><td>49.3309227454</td><td>10.0949662552</td><td>10.0962454002</td><td>0.0578780941924</td><td>0.0252661695745</td><td>0.027791622415</td><td>3.61793467655</td><td>0.0256672432086</td><td>0.724042029392</td><td>61.6611012311</td><td>27.1506478165</td><td>36.4984671231</td><td>15.7963254274</td><td>35.8648709586</td><td>53.3979794776</td><td>0.0725644931663</td><td>S</td><td>P206+50</td><td>9967</td><td>325690</td><td>0.48500595098</td><td>13.2393295354</td><td>14562507.0</td><td>1.0</td></tr>\n",
       "<tr><td>ILTJ134426.185+492250.83</td><td>206.109104993</td><td>7.35337747571</td><td>7.355133431</td><td>49.3807881244</td><td>8.14633251011</td><td>8.1479175777</td><td>0.0646593122396</td><td>0.0216523667916</td><td>0.025220191396</td><td>5.24164882945</td><td>0.0219178391556</td><td>1.04855886326</td><td>59.5889525737</td><td>20.0032386041</td><td>48.9792776867</td><td>16.3616237264</td><td>28.8371736979</td><td>82.7450691357</td><td>0.0702290708432</td><td>S</td><td>P206+50</td><td>9968</td><td>325691</td><td>0.737285833627</td><td>8.36125539124</td><td>26275082.0</td><td>2.0</td></tr>\n",
       "<tr><td>ILTJ134426.185+492250.83</td><td>206.109104993</td><td>7.35337747571</td><td>7.355133431</td><td>49.3807881244</td><td>8.14633251011</td><td>8.1479175777</td><td>0.0646593122396</td><td>0.0216523667916</td><td>0.025220191396</td><td>5.24164882945</td><td>0.0219178391556</td><td>1.04855886326</td><td>59.5889525737</td><td>20.0032386041</td><td>48.9792776867</td><td>16.3616237264</td><td>28.8371736979</td><td>82.7450691357</td><td>0.0702290708432</td><td>S</td><td>P206+50</td><td>9968</td><td>325691</td><td>1.5333394225</td><td>0.359093118484</td><td>26352886.0</td><td>1.0</td></tr>\n",
       "<tr><td>ILTJ134416.177+484518.63</td><td>206.067405501</td><td>4.27904823131</td><td>4.2820650688</td><td>48.7551761932</td><td>2.35101550671</td><td>2.35650194153</td><td>0.197490968078</td><td>0.0240902457249</td><td>0.0462649677184</td><td>19.4004590358</td><td>0.0243342686365</td><td>3.88016811344</td><td>83.8064149769</td><td>10.2995547168</td><td>42.2084335982</td><td>5.11029566712</td><td>103.05693549</td><td>12.4492036664</td><td>0.0860225482029</td><td>S</td><td>P206+50</td><td>9970</td><td>325692</td><td>0.579629471613</td><td>10.820825309</td><td>26657452.0</td><td>2.0</td></tr>\n",
       "<tr><td>ILTJ134416.177+484518.63</td><td>206.067405501</td><td>4.27904823131</td><td>4.2820650688</td><td>48.7551761932</td><td>2.35101550671</td><td>2.35650194153</td><td>0.197490968078</td><td>0.0240902457249</td><td>0.0462649677184</td><td>19.4004590358</td><td>0.0243342686365</td><td>3.88016811344</td><td>83.8064149769</td><td>10.2995547168</td><td>42.2084335982</td><td>5.11029566712</td><td>103.05693549</td><td>12.4492036664</td><td>0.0860225482029</td><td>S</td><td>P206+50</td><td>9970</td><td>325692</td><td>1.4100985987</td><td>8.94180949023</td><td>14393277.0</td><td>1.0</td></tr>\n",
       "<tr><td>ILTJ134311.277+491857.55</td><td>205.796987079</td><td>10.0774803606</td><td>10.0787617249</td><td>49.3159867305</td><td>11.2244774561</td><td>11.2256278956</td><td>0.198956672145</td><td>0.108410789679</td><td>0.115482681016</td><td>9.42576116904</td><td>0.110676392851</td><td>1.8883983183</td><td>57.4020158334</td><td>31.7328982534</td><td>29.7163017148</td><td>15.9602674185</td><td>139.285397713</td><td>58.5452540984</td><td>0.274070509477</td><td>S</td><td>P206+50</td><td>9971</td><td>325693</td><td>2.8081435628</td><td>10.6355466372</td><td>14625420.0</td><td>1.0</td></tr>\n",
       "<tr><td>ILTJ134311.277+491857.55</td><td>205.796987079</td><td>10.0774803606</td><td>10.0787617249</td><td>49.3159867305</td><td>11.2244774561</td><td>11.2256278956</td><td>0.198956672145</td><td>0.108410789679</td><td>0.115482681016</td><td>9.42576116904</td><td>0.110676392851</td><td>1.8883983183</td><td>57.4020158334</td><td>31.7328982534</td><td>29.7163017148</td><td>15.9602674185</td><td>139.285397713</td><td>58.5452540984</td><td>0.274070509477</td><td>S</td><td>P206+50</td><td>9971</td><td>325693</td><td>0.55398818036</td><td>10.880053618</td><td>14652394.0</td><td>2.0</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table masked=True length=339587>\n",
       "      Source_Name              RA            E_RA      ...  lr_index  lr_order\n",
       "                              deg           arcsec     ...                    \n",
       "         str24              float64        float64     ...  float64   float64 \n",
       "------------------------ ------------- --------------- ... ---------- --------\n",
       "ILTJ122108.441+491340.94 185.285168771 0.0850700841174 ... 10243809.0      1.0\n",
       "ILTJ122107.851+491951.84 185.282710884  0.473938135007 ... 10206441.0      1.0\n",
       "ILTJ122108.572+493121.89 185.285716341  0.533848043611 ...        nan      nan\n",
       "ILTJ122104.027+490823.80 185.266778578 0.0680609727689 ... 10195667.0      1.0\n",
       "ILTJ122105.030+491440.01 185.270960099  0.585020092161 ... 10171052.0      1.0\n",
       "ILTJ122106.722+492420.20 185.278009798  0.476205256172 ... 10318938.0      1.0\n",
       "ILTJ122106.172+492908.04 185.275717593  0.183850706166 ... 10320755.0      1.0\n",
       "ILTJ122104.649+492207.60 185.269372038  0.515425109037 ... 10216972.0      1.0\n",
       "ILTJ122107.312+493759.86 185.280465285  0.574867040653 ... 10339350.0      1.0\n",
       "                     ...           ...             ... ...        ...      ...\n",
       "ILTJ134709.873+494747.92 206.791138823   10.2608863243 ...        nan      nan\n",
       "ILTJ134632.301+485421.18 206.634588782   6.09050055912 ...        nan      nan\n",
       "ILTJ134454.662+491951.32 206.227759779   8.71976713446 ... 26521592.0      2.0\n",
       "ILTJ134454.662+491951.32 206.227759779   8.71976713446 ... 14562507.0      1.0\n",
       "ILTJ134426.185+492250.83 206.109104993   7.35337747571 ... 26275082.0      2.0\n",
       "ILTJ134426.185+492250.83 206.109104993   7.35337747571 ... 26352886.0      1.0\n",
       "ILTJ134416.177+484518.63 206.067405501   4.27904823131 ... 26657452.0      2.0\n",
       "ILTJ134416.177+484518.63 206.067405501   4.27904823131 ... 14393277.0      1.0\n",
       "ILTJ134311.277+491857.55 205.796987079   10.0774803606 ... 14625420.0      1.0\n",
       "ILTJ134311.277+491857.55 205.796987079   10.0774803606 ... 14652394.0      2.0"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lofar_lr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save combined catalogue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined[\"lr_index\"] = combined_aux_index.astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for col in ['lr', 'lr_dist', 'lr_index', 'lr_order']:\n",
    "    lofar_lr[col].fill_value = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pwl = join(lofar_lr.filled(), combined, join_type='left', keys='lr_index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "339587"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pwl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pwl_columns = pwl.colnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dec 1.0\n",
      "W1mag 1.0\n",
      "i 1.0\n",
      "colour 1.0\n",
      "category 1.0\n"
     ]
    }
   ],
   "source": [
    "for col in pwl_columns:\n",
    "    fv = pwl[col].fill_value\n",
    "    #print(col, fv)\n",
    "    if (isinstance(fv, np.float64) and (fv != 1e+20)):\n",
    "        print(col, fv)\n",
    "        pwl[col].fill_value = 1e+20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columns_save = ['Source_Name', 'RA', 'E_RA', 'E_RA_tot', 'DEC', 'E_DEC', 'E_DEC_tot',\n",
    " 'Peak_flux', 'E_Peak_flux', 'E_Peak_flux_tot', 'Total_flux', 'E_Total_flux', 'E_Total_flux_tot',\n",
    " 'Maj', 'E_Maj', 'Min', 'E_Min', 'PA', 'E_PA', 'Isl_rms', 'S_Code', 'Mosaic_ID', 'Isl_id',\n",
    " 'AllWISE', 'objID', 'ra', 'dec', 'raErr', 'decErr',\n",
    " 'W1mag', 'W1magErr', 'i', 'iErr', 'colour', 'category',\n",
    " 'lr', 'lr_dist', 'lr_order']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pwl[columns_save].filled().write('lofar_multi_lr_pw.fits', format=\"fits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}