Save_main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save the LOFAR combined catalogue\n",
    "\n",
    "This notebook can retrieve the saved data and create the combined final catalogue. It is very useful if the main notebook crashed or after running the script version."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from astropy.table import Table, join\n",
    "from astropy import units as u\n",
    "from astropy.coordinates import SkyCoord, search_around_sky\n",
    "from IPython.display import clear_output\n",
    "import pickle\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from mltier1 import (get_center, get_n_m, estimate_q_m, Field, SingleMLEstimator, MultiMLEstimator,\n",
    "                     parallel_process, get_sigma, get_q_m, get_threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "last_iter = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lofar = Table.read(\"idata/main/lofar_m{}.fits\".format(last_iter))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined = Table.read(\"pw.fits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined[\"colour\"] = combined[\"i\"] - combined[\"W1mag\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined_matched = (~np.isnan(combined[\"i\"]) & ~np.isnan(combined[\"W1mag\"])) # Matched i-W1 sources\n",
    "combined_panstarrs = (~np.isnan(combined[\"i\"]) & np.isnan(combined[\"W1mag\"])) # Sources with only i-band\n",
    "combined_wise =(np.isnan(combined[\"i\"]) & ~np.isnan(combined[\"W1mag\"])) # Sources with only W1-band\n",
    "combined_i = combined_matched | combined_panstarrs\n",
    "combined_w1 = combined_matched | combined_wise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "colour_limits = [-0.5, 0.0, 0.5, 1.0, 1.25, 1.5, 1.75, 2.0, 2.25, 2.5, 2.75, 3.0, 3.5, 4.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jsm/programs/soft/anaconda/envs/py36/lib/python3.6/site-packages/ipykernel/__main__.py:5: RuntimeWarning: invalid value encountered in less\n",
      "/home/jsm/programs/soft/anaconda/envs/py36/lib/python3.6/site-packages/ipykernel/__main__.py:10: RuntimeWarning: invalid value encountered in greater_equal\n",
      "/home/jsm/programs/soft/anaconda/envs/py36/lib/python3.6/site-packages/ipykernel/__main__.py:11: RuntimeWarning: invalid value encountered in less\n",
      "/home/jsm/programs/soft/anaconda/envs/py36/lib/python3.6/site-packages/ipykernel/__main__.py:16: RuntimeWarning: invalid value encountered in greater_equal\n"
     ]
    }
   ],
   "source": [
    "# Start with the W1-only, i-only and \"less than lower colour\" bins\n",
    "colour_bin_def = [{\"name\":\"only W1\", \"condition\": combined_wise},\n",
    "                  {\"name\":\"only i\", \"condition\": combined_panstarrs},\n",
    "                  {\"name\":\"-inf to {}\".format(colour_limits[0]), \n",
    "                   \"condition\": (combined[\"colour\"] < colour_limits[0])}]\n",
    "\n",
    "# Get the colour bins\n",
    "for i in range(len(colour_limits)-1):\n",
    "    name = \"{} to {}\".format(colour_limits[i], colour_limits[i+1])\n",
    "    condition = ((combined[\"colour\"] >= colour_limits[i]) & \n",
    "                 (combined[\"colour\"] < colour_limits[i+1]))\n",
    "    colour_bin_def.append({\"name\":name, \"condition\":condition})\n",
    "\n",
    "# Add the \"more than higher colour\" bin\n",
    "colour_bin_def.append({\"name\":\"{} to inf\".format(colour_limits[-1]), \n",
    "                       \"condition\": (combined[\"colour\"] >= colour_limits[-1])})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined[\"category\"] = np.nan\n",
    "for i in range(len(colour_bin_def)):\n",
    "    combined[\"category\"][colour_bin_def[i][\"condition\"]] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combined[\"lr_index_sel_{}\".format(last_iter)] = np.arange(len(combined)).astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "combined[\"i\"].fill_value = 1e+20\n",
    "combined[\"W1mag\"].fill_value = 1e+20\n",
    "combined[\"colour\"].fill_value = 1e+20\n",
    "combined[\"category\"].fill_value = 1e+20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lofar.remove_column(\"i\")\n",
    "lofar.remove_column(\"category\")\n",
    "lofar.remove_column(\"W1mag\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lofar_combined = join(lofar, combined, join_type='left', keys='lr_index_sel_{}'.format(last_iter))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source_id N/A\n",
      "RA 1e+20\n",
      "E_RA 1e+20\n",
      "E_RA_tot 1e+20\n",
      "DEC 1e+20\n",
      "E_DEC 1e+20\n",
      "E_DEC_tot 1e+20\n",
      "Peak_flux 1e+20\n",
      "E_Peak_flux 1e+20\n",
      "E_Peak_flux_tot 1e+20\n",
      "Total_flux 1e+20\n",
      "E_Total_flux 1e+20\n",
      "E_Total_flux_tot 1e+20\n",
      "Maj 1e+20\n",
      "E_Maj 1e+20\n",
      "Min 1e+20\n",
      "E_Min 1e+20\n",
      "PA 1e+20\n",
      "E_PA 1e+20\n",
      "Isl_rms 1e+20\n",
      "S_Code N/A\n",
      "Mosaic_ID N/A\n",
      "lr_i 1e+20\n",
      "lr_dist_i 1e+20\n",
      "lr_index_i 1e+20\n",
      "lr_index_sel_i 1e+20\n",
      "lr_w1 1e+20\n",
      "lr_dist_w1 1e+20\n",
      "lr_index_w1 1e+20\n",
      "lr_index_sel_w1 1e+20\n",
      "lr_index_1 1e+20\n",
      "lr_dist_1 1e+20\n",
      "lr_1 1e+20\n",
      "lr_type_1 999999\n",
      "lr_index_2 1e+20\n",
      "lr_dist_2 1e+20\n",
      "lr_2 1e+20\n",
      "lr_index_sel_2 1e+20\n",
      "lr_index_3 1e+20\n",
      "lr_dist_3 1e+20\n",
      "lr_3 1e+20\n",
      "lr_index_sel_3 1e+20\n",
      "lr_index_4 1e+20\n",
      "lr_dist_4 1e+20\n",
      "lr_4 1e+20\n",
      "lr_index_sel_4 1e+20\n",
      "lr_index_5 1e+20\n",
      "lr_dist_5 1e+20\n",
      "lr_5 1e+20\n",
      "lr_index_sel_5 1e+20\n",
      "lr_index_6 1e+20\n",
      "lr_dist_6 1e+20\n",
      "lr_6 1e+20\n",
      "lr_index_sel_6 1e+20\n",
      "lr_index_7 1e+20\n",
      "lr_dist_7 1e+20\n",
      "lr_7 1e+20\n",
      "lr_index_sel_7 1e+20\n",
      "lr_index_8 1e+20\n",
      "lr_dist_8 1e+20\n",
      "lr_8 1e+20\n",
      "lr_index_sel_8 1e+20\n",
      "lr_index_9 1e+20\n",
      "lr_dist_9 1e+20\n",
      "lr_9 1e+20\n",
      "lr_index_sel_9 1e+20\n",
      "lr_index_10 1e+20\n",
      "lr_dist_10 1e+20\n",
      "lr_10 1e+20\n",
      "lr_index_sel_10 1e+20\n",
      "AllWISE N/A\n",
      "objID -99\n",
      "ra 1e+20\n",
      "dec 1e+20\n",
      "raErr 1e+20\n",
      "decErr 1e+20\n",
      "W1mag 1e+20\n",
      "W1magErr 1e+20\n",
      "i 1e+20\n",
      "iErr 1e+20\n",
      "colour 1e+20\n",
      "category 1e+20\n"
     ]
    }
   ],
   "source": [
    "for col in lofar_combined.colnames:\n",
    "    try:\n",
    "        print(col, lofar_combined[col].fill_value)\n",
    "    except AttributeError:\n",
    "        print(col, \"NO_FILL\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lofar_combined.write(\"lofar_matched_all.fits\", format=\"fits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (py36)",
   "language": "python",
   "name": "py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}