status.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "suburban-childhood",
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "developing-silly",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "enhanced-medium",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dutch-credit",
   "metadata": {},
   "outputs": [],
   "source": [
    "from processfedndata import read_status, smoothen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "suitable-bikini",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "hundred-birmingham",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "turkish-translator",
   "metadata": {},
   "outputs": [],
   "source": [
    "def smoothen(xs, ys, size=10):\n",
    "    _xs = []\n",
    "    _ys = []\n",
    "    _x = 0\n",
    "    _y1 = 0\n",
    "    _y2 = 0\n",
    "    for i, (x, (y1, y2)) in enumerate(zip(xs, ys)):\n",
    "        _x += x\n",
    "        _y1 += y1\n",
    "        _y2 += y2\n",
    "        if (i + 1) % size == 0:\n",
    "            _xs.append(_x / size)\n",
    "            _ys.append((_y1 / size, _y2 / size))\n",
    "            _x = 0\n",
    "            _y1 = 0\n",
    "            _y2 = 0\n",
    "    return _xs, _ys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "pediatric-going",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_status(results,\n",
    "                steps_per_round,\n",
    "                metric=\"loss\",\n",
    "                time=False,\n",
    "                smooth=0,\n",
    "                fig=None,\n",
    "                ax=None,\n",
    "                do_plot=True,\n",
    "                a_name=\"\"):\n",
    "    ys = defaultdict(list)\n",
    "    for y, name, timestamp in results:\n",
    "        ys[name].append(y[metric])\n",
    "    if time:\n",
    "        xs = {}\n",
    "        start = None\n",
    "        for i, (_, name, timestamp) in enumerate(results):\n",
    "            time = datetime.datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S.%f')\n",
    "            if start is None:\n",
    "                start = time\n",
    "            time_passed = time - start\n",
    "            xs[name] = time_passed.total_seconds() / 60\n",
    "        xs = xs.values()\n",
    "    else:\n",
    "        xs = np.arange(0, len(ys) * steps_per_round, steps_per_round)\n",
    "    ys = list(ys.values())\n",
    "\n",
    "    if not fig and do_plot:\n",
    "        fig, ax = plt.subplots(figsize=(19, 6))\n",
    "    if smooth:\n",
    "        xs, ys = smoothen(xs, ys, smooth)\n",
    "    ys1, ys2 = zip(*ys)\n",
    "    if do_plot:\n",
    "        plt.plot(xs, ys1, \"-\", label=f\"sv - {a_name}\")\n",
    "        plt.plot(xs, ys2, \"-\", label=f\"no - {a_name}\")\n",
    "    return xs, ys, fig, ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "national-tension",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(fn1, l1, fn2, l2, time_spent, smooth, title, xlabel, ylabel=\"Loss\", a_name=\"\", b_name=\"\"):\n",
    "    data1 = read_status(fn1)\n",
    "    data2 = read_status(fn2)\n",
    "    smv = 0\n",
    "    if smooth:\n",
    "        smv = 10 # int(len(data1) / len(data2))\n",
    "    print(len(data1), len(data2))\n",
    "    # print_status(data1, l1, \"loss\")\n",
    "    _, _, fig, ax = plot_status(data1, l1, \"loss\", time=time_spent, smooth=smv, a_name=a_name)\n",
    "    _, _, fig, ax = plot_status(data2, l2, \"loss\", time=time_spent, smooth=0, fig=fig, ax=ax, a_name=b_name)\n",
    "    plt.title(title)\n",
    "    plt.xlabel(xlabel)\n",
    "    plt.ylabel(ylabel)\n",
    "    plt.legend()\n",
    "    _title = \"_\".join(title.lower().split())\n",
    "    _xlabel = \"_\".join(xlabel.lower().split())\n",
    "    plt.savefig(f\"../figs/{_title}_{_xlabel}.pdf\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "difficult-violence",
   "metadata": {},
   "source": [
    "## Experiments\n",
    "\n",
    "In order to test the effect of the number of steps between each model aggregation step, we run ELECTRA training\n",
    "with 100, 1000, 2000, and 5000 steps for every aggregation loop."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "smoking-listing",
   "metadata": {},
   "outputs": [],
   "source": [
    "l1k = \"electra_small_sv+no_1000-step-round.control.status\"\n",
    "l1k_ctd = \"electra_small_sv+no_1000-step-round.control.status.continued\"\n",
    "l1k_extra = \"electra_small_sv+no_1000-step-no_extra_round.control.status\"\n",
    "l100 = \"electra_small_sv+no_100-step-round.control.status\"\n",
    "l2k = \"electra_small_sv+no_2000-step-round.control.status\"\n",
    "l5k = \"electra_small_sv+no_5000-step-round.control.status\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "hairy-clause",
   "metadata": {},
   "source": [
    "### 100 vs. 1000 Steps\n",
    "\n",
    "While more frequent aggregations result in better model performance in the same total number of steps, this takes\n",
    "much longer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "australian-liquid",
   "metadata": {},
   "outputs": [],
   "source": [
    "main(l100, 100, l1k, 1000, time_spent=False, smooth=True, title=\"100 vs. 1000 Steps\", xlabel=\"Training Steps\", a_name=\"100\", b_name=\"1000\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "stuffed-toyota",
   "metadata": {},
   "source": [
    "Plotting performance instead over time spent instead of the number of steps shows, that it is reasonable to reduce\n",
    "performance for the sake of speed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "thermal-gothic",
   "metadata": {},
   "outputs": [],
   "source": [
    "main(l100, 100, l1k, 1000, time_spent=True, smooth=True, title=\"100 vs. 1000 Steps\", xlabel=\"Training Time in Minutes\", a_name=\"100\", b_name=\"1000\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "severe-walter",
   "metadata": {},
   "source": [
    "### 1000 vs. 2000 Steps\n",
    "\n",
    "To see how much performance degrades while gaining speed in return we compare 1000 and 2000 steps per federated learning round."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "closing-passage",
   "metadata": {},
   "outputs": [],
   "source": [
    "main(l1k, 1000, l2k, 2000, False, False, title=\"1000 vs. 2000 Steps\", xlabel=\"Training Steps\", a_name=\"1000\", b_name=\"2000\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "political-syracuse",
   "metadata": {},
   "source": [
    "The drop in performance is small but visible, but over time spent we do not get any real benefit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "painted-agent",
   "metadata": {},
   "outputs": [],
   "source": [
    "main(l1k, 1000, l2k, 2000, True, False, title=\"1000 vs. 2000 Steps\", xlabel=\"Training Time in Minutes\", a_name=\"1000\", b_name=\"2000\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "organic-photograph",
   "metadata": {},
   "source": [
    "### 1000 vs. 5000 Steps\n",
    "\n",
    "To see if the speed gain justifies a performance drop we further increase the number of steps per round."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "anonymous-solomon",
   "metadata": {},
   "outputs": [],
   "source": [
    "main(l1k, 1000, l5k, 5000, False, False, title=\"1000 vs. 5000 Steps\", xlabel=\"Training Steps\", a_name=\"1000\", b_name=\"5000\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "excess-noise",
   "metadata": {},
   "source": [
    "While the model is trained for about 300000 more steps in nearly the same amount of time, again the performance\n",
    "drop argues against this higher amount of steps per round."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "varying-milwaukee",
   "metadata": {},
   "outputs": [],
   "source": [
    "main(l1k, 1000, l5k, 5000, True, False, title=\"1000 vs. 5000 Steps\", xlabel=\"Training Time in Minutes\", a_name=\"1000\", b_name=\"5000\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "small-treaty",
   "metadata": {},
   "source": [
    "### With and without optimizer parameters\n",
    "\n",
    "Language models such as ELECTRA use the Adam optimizer for training, which requires to keep track of additional momentum variables for every parameter.\n",
    "When training large LMs in a federated fashion, the increased number of parameters required to be aggregated can result into long waiting times when sending the client's model data to the combiner/reducer.\n",
    "\n",
    "We therefore test training ELECTRA without aggregating the optimizer parameters, federating only model parameters, while keeping the client's local optimization variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "designed-jewel",
   "metadata": {},
   "outputs": [],
   "source": [
    "smv = 10\n",
    "data1 = read_status(l1k)\n",
    "data2 = read_status(l1k_ctd)\n",
    "xs1, ys1, fig, ax = plot_status(data1, 1000, do_plot=False, smooth=smv)\n",
    "xs2, ys2, _, _ = plot_status(data2, 1000, do_plot=False, smooth=smv)\n",
    "xs = np.concatenate([xs1, xs1[-1] + xs2])\n",
    "ys = np.concatenate([ys1, ys2])\n",
    "fig, ax = plt.subplots(figsize=(19, 6))\n",
    "xs3, ys3, _, _ = plot_status(read_status(l1k_extra), 1000, do_plot=False, smooth=smv)\n",
    "plt.title(\"1000 Steps: With vs. Without Optimization Parameters\")\n",
    "plt.xlabel(\"Training Steps\")\n",
    "plt.ylabel(\"Loss\")\n",
    "ys1, ys2 = zip(*ys)\n",
    "plt.plot(xs, ys1, label=\"sv - Federated Adam\")\n",
    "plt.plot(xs, ys2, label=\"no - Federated Adam\")\n",
    "ys3, ys4 = zip(*ys3)\n",
    "plt.plot(xs3, ys3, label=\"sv - Local Adam\")\n",
    "plt.plot(xs3, ys4, label=\"no - Local Adam\")\n",
    "plt.legend()\n",
    "plt.savefig(\"../figs/local_v_global_steps.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "living-andorra",
   "metadata": {},
   "outputs": [],
   "source": [
    "smv = 10\n",
    "data1 = read_status(l1k)\n",
    "data2 = read_status(l1k_ctd)\n",
    "xs1, ys1, fig, ax = plot_status(data1, 1000, do_plot=False, smooth=smv, time=True)\n",
    "xs2, ys2, _, _ = plot_status(data2, 1000, do_plot=False, smooth=smv, time=True)\n",
    "xs1 = np.array(xs1)\n",
    "xs2 = np.array(xs2)\n",
    "xs = np.concatenate([xs1, xs1[-1] + xs2])\n",
    "ys = np.concatenate([ys1, ys2])\n",
    "fig, ax = plt.subplots(figsize=(19, 6))\n",
    "xs3, ys3, _, _ = plot_status(read_status(l1k_extra), 1000, do_plot=False, smooth=smv, time=True)\n",
    "plt.title(\"1000 Steps: With vs. Without Optimization Parameters\")\n",
    "plt.xlabel(\"Training Time in Minutes\")\n",
    "plt.ylabel(\"Loss\")\n",
    "ys1, ys2 = zip(*ys)\n",
    "plt.plot(xs, ys1, label=\"sv - Federated Adam\")\n",
    "plt.plot(xs, ys2, label=\"no - Federated Adam\")\n",
    "ys3, ys4 = zip(*ys3)\n",
    "plt.plot(xs3, ys3, label=\"sv - Local Adam\")\n",
    "plt.plot(xs3, ys4, label=\"no - Local Adam\")\n",
    "plt.legend()\n",
    "plt.savefig(\"../figs/local_v_global_time.pdf\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}