briscola.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from copy import deepcopy\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "from sklearn.metrics import explained_variance_score\n",
    "from torch import optim, nn\n",
    "\n",
    "from src.agents import Agent\n",
    "from src.agents.NNAgent import NNAgent\n",
    "from src.agents.RandomAgent import RandomAgent\n",
    "from time import time\n",
    "import wandb\n",
    "from src.utils.AgentPool import AgentPool\n",
    "from src.utils.training_utils import get_state_representation\n",
    "\n",
    "from src.vectorizers.VectorizedEnv import VectorizedEnv\n",
    "from src.envs.two_player_briscola.TwoPlayerBriscola import TwoPlayerBriscola\n",
    "\n",
    "from src.envs.two_player_briscola.BriscolaConstants import Constants\n",
    "\n",
    "from src.utils.training_utils import play_all_moves_of_players, compute_rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"n_envs\" : 2048,\n",
    "    \"n_steps\": Constants.deck_cards // 2,\n",
    "    \"lr\": 3e-3,\n",
    "    \"lr_decay\": 0.997,\n",
    "    \"lr_min\": 1e-4,\n",
    "    \"mini_batch_size\": 2048,\n",
    "    \"total_timesteps\": 50_000_000,\n",
    "    \"gamma\": 1.,\n",
    "    \"lambda\": 0.95,\n",
    "    \"update_epochs\": 2,\n",
    "    \"clip_coef\": 0.2,\n",
    "    \"normalize_advantage\": True,\n",
    "    \"clip_value_loss\": True,\n",
    "    \"value_coef\": 0.5,\n",
    "    \"entropy_coef\": 1e-2,\n",
    "    \"entropy_decay\": 0.998,\n",
    "    \"max_grad_norm\": 0.5,\n",
    "    \"ratio_win_reward\": 0.1,\n",
    "    \"win_reward_increase\": 0.001,\n",
    "    \"n_opponents\": 4,\n",
    "    \"self_play_opponents\": 2,\n",
    "    \"max_pool_size\": 128,\n",
    "    \"add_model_every_x_step\": 2,\n",
    "    \"nu\": 0.1,\n",
    "    \"hidden_size\": 256,\n",
    "    \"activation\": nn.Mish,\n",
    "    \"briscola_penalization\": 0.,\n",
    "    \"briscola_penalization_decay\": 0.99\n",
    "}\n",
    "params[\"batch_size\"] = params[\"n_envs\"] * params[\"n_steps\"]\n",
    "params[\"num_updates\"] = params[\"total_timesteps\"] // params[\"batch_size\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "run = wandb.init(\n",
    "        name=\"briscola penalization continue\",\n",
    "        project=\"briscolaBot\",\n",
    "        entity=\"lettera\",\n",
    "        config=params,\n",
    "        save_code=True,\n",
    "        sync_tensorboard=False,\n",
    "        mode=\"online\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vec_env = VectorizedEnv(lambda: TwoPlayerBriscola(), params[\"n_envs\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "device = \"cpu\"\n",
    "observation_shape = vec_env.single_observation_space()[\"observation\"].shape\n",
    "action_size = vec_env.single_action_space().n\n",
    "\n",
    "player_policy = NNAgent(observation_shape, action_size, hidden_size=params[\"hidden_size\"], activation=params[\"activation\"]).to(device)\n",
    "# player_policy.load_state_dict(torch.load(\"pretrained_models/briscola-bot-v3.pt\")) # Load pretrained model\n",
    "\n",
    "player_name = vec_env[0].agents[0]\n",
    "opponent_name = vec_env[0].agents[1]\n",
    "\n",
    "trained_previous = NNAgent(observation_shape, action_size, hidden_size=256).to(device)\n",
    "trained_previous.load_state_dict(torch.load(\"pretrained_models/briscola-bot-v3.pt\"))\n",
    "\n",
    "trained_v2 = NNAgent(observation_shape, action_size, hidden_size=256, activation=nn.Mish).to(device)\n",
    "trained_v2.load_state_dict(torch.load(\"pretrained_models/briscola-bot-v2.pt\"))\n",
    "\n",
    "pool = AgentPool(params[\"max_pool_size\"], nu=params[\"nu\"])\n",
    "\n",
    "optimizer = optim.Adam(player_policy.parameters(), lr=params[\"lr\"], eps=1e-5)\n",
    "\n",
    "obs = torch.zeros((params[\"n_steps\"], params[\"n_envs\"]) + observation_shape).to(device)\n",
    "actions = torch.zeros((params[\"n_steps\"], params[\"n_envs\"]), dtype=torch.int64).to(device)\n",
    "actions_masks = torch.zeros((params[\"n_steps\"], params[\"n_envs\"]) + (action_size,), dtype=torch.int64).to(device)\n",
    "logprobs = torch.zeros((params[\"n_steps\"], params[\"n_envs\"])).to(device)\n",
    "rewards = torch.zeros((params[\"n_steps\"], params[\"n_envs\"])).to(device)\n",
    "dones = torch.zeros((params[\"n_steps\"], params[\"n_envs\"]), dtype=torch.int8).to(device)\n",
    "values = torch.zeros((params[\"n_steps\"], params[\"n_envs\"])).to(device)\n",
    "\n",
    "global_step = 0\n",
    "start_time = time()\n",
    "for update in range(params[\"num_updates\"]):\n",
    "    # Decay lr\n",
    "    current_lr = optimizer.param_groups[0][\"lr\"]\n",
    "    optimizer.param_groups[0][\"lr\"] = max(params[\"lr_min\"], current_lr*params[\"lr_decay\"])\n",
    "\n",
    "    # Decay entropy\n",
    "    params[\"entropy_coef\"] *= params[\"entropy_decay\"]\n",
    "\n",
    "    # Decay briscola penalization\n",
    "    params[\"briscola_penalization\"] *= params[\"briscola_penalization_decay\"]\n",
    "\n",
    "    # Increase ratio_win_reward\n",
    "    params[\"ratio_win_reward\"] = min(params[\"ratio_win_reward\"] + params[\"win_reward_increase\"], 1)\n",
    "\n",
    "    # Add agent to pool\n",
    "    if update % params[\"add_model_every_x_step\"] == 0:\n",
    "        pool.add_agent(deepcopy(player_policy))\n",
    "\n",
    "    # Sample agents\n",
    "    opponent_policies, opponent_indexes = pool.sample_agents(params[\"n_opponents\"] - params[\"self_play_opponents\"])\n",
    "    opponent_policies += [player_policy] * params[\"self_play_opponents\"]\n",
    "\n",
    "    # Play episodes\n",
    "    vec_env.reset()\n",
    "    play_all_moves_of_players(vec_env, opponent_policies, opponent_name)\n",
    "    next_obs, action_mask, reward, next_done = get_state_representation(vec_env)\n",
    "    for step in range(params[\"n_steps\"]):\n",
    "        global_step += params[\"n_envs\"]\n",
    "\n",
    "        obs[step] = next_obs\n",
    "        dones[step] = next_done\n",
    "\n",
    "        with torch.no_grad():\n",
    "            action, logprob, _, value = player_policy.get_action_and_value(next_obs.to(device), action_mask.to(device))\n",
    "            values[step] = value.flatten()\n",
    "        actions[step] = action\n",
    "        actions_masks[step] = action_mask.to(device)\n",
    "        logprobs[step] = logprob\n",
    "\n",
    "        vec_env.step(actions[step].cpu().numpy(), briscola_penalization=params[\"briscola_penalization\"])\n",
    "        play_all_moves_of_players(vec_env, opponent_policies, opponent_name)\n",
    "        next_obs, action_mask, reward, next_done = get_state_representation(vec_env)\n",
    "        wins = torch.tensor([env.get_game_outcome(player_name) for env in vec_env], dtype=torch.float32)\n",
    "        rewards[step] = (1 - params[\"ratio_win_reward\"]) * reward + (params[\"ratio_win_reward\"] * next_done * wins).to(device)\n",
    "\n",
    "    # Update rating\n",
    "    scores = [env.get_game_outcome(opponent_name) for env in vec_env.get_envs()]\n",
    "    mean_score_per_opponent = np.empty_like(opponent_indexes, dtype=np.float64)\n",
    "    for i in range(opponent_indexes.size):\n",
    "        start, end = (i * len(scores)) // opponent_indexes.size, ((i + 1) * len(scores)) // opponent_indexes.size\n",
    "        mean_score_per_opponent[i] = np.mean(scores[start:end])\n",
    "\n",
    "    agent_rating = pool.update_ratings(0., mean_score_per_opponent, opponent_indexes)\n",
    "\n",
    "    # Bootstrap value\n",
    "    with torch.no_grad():\n",
    "        next_value = player_policy.get_value(next_obs).reshape(1, -1)\n",
    "        advantages = torch.zeros_like(rewards).to(device)\n",
    "        last_gae_lambda = 0\n",
    "\n",
    "        for t in reversed(range(params[\"n_steps\"])):\n",
    "            if t == params[\"n_steps\"] - 1:\n",
    "                next_non_terminal = 1. - next_done\n",
    "                next_values = next_value\n",
    "            else:\n",
    "                next_non_terminal = 1. - dones[t+1]\n",
    "                next_values = values[t+1]\n",
    "\n",
    "            delta = rewards[t] + params[\"gamma\"] * next_values * next_non_terminal - values[t]\n",
    "            last_gae_lambda = delta + params[\"gamma\"] * params[\"lambda\"] * next_non_terminal * last_gae_lambda\n",
    "            advantages[t] = last_gae_lambda\n",
    "\n",
    "        returns = advantages + values\n",
    "\n",
    "    # Optimize net\n",
    "    b_obs = obs.reshape((-1,) + observation_shape)\n",
    "    b_logprobs = logprobs.reshape(-1)\n",
    "    b_actions = actions.reshape(-1)\n",
    "    b_action_masks = actions_masks.reshape((-1, action_size))\n",
    "    b_advantages = advantages.reshape(-1)\n",
    "    b_returns = returns.reshape(-1)\n",
    "    b_values = values.reshape(-1)\n",
    "\n",
    "    clip_fraction = []\n",
    "    b_indexes = np.arange(params[\"batch_size\"])\n",
    "    for epoch in range(params[\"update_epochs\"]):\n",
    "        np.random.shuffle(b_indexes)\n",
    "        for start in range(0, params[\"batch_size\"], params[\"mini_batch_size\"]):\n",
    "            end = start + params[\"mini_batch_size\"]\n",
    "            mb_indexes = b_indexes[start:end]\n",
    "\n",
    "            _, newlogprob, entropy, newvalue = player_policy.get_action_and_value(b_obs[mb_indexes], b_action_masks[mb_indexes], b_actions[mb_indexes])\n",
    "            logratio = newlogprob - b_logprobs[mb_indexes]\n",
    "            ratio = logratio.exp()\n",
    "\n",
    "            with torch.no_grad():\n",
    "                # calculate approx_kl https://joschu.net/blog/kl-approx.html\n",
    "                old_approx_kl = (-logratio).mean()\n",
    "                approx_kl = ((ratio - 1) - logratio).mean()\n",
    "                clip_fraction.append(((ratio - 1.0).abs() > params[\"clip_coef\"]).float().mean().item())\n",
    "\n",
    "            mb_advantages = b_advantages[mb_indexes]\n",
    "            if params[\"normalize_advantage\"]:\n",
    "                mb_advantages = (mb_advantages - mb_advantages.mean()) / (mb_advantages.std() + 1e-8)\n",
    "\n",
    "            # Policy loss\n",
    "            pg_loss1 = -mb_advantages * ratio\n",
    "            pg_loss2 = -mb_advantages * torch.clamp(ratio, 1-params[\"clip_coef\"], 1+params[\"clip_coef\"])\n",
    "            pg_loss = torch.max(pg_loss1, pg_loss2).mean()\n",
    "\n",
    "            # Value loss\n",
    "            newvalue = newvalue.view(-1)\n",
    "            if params[\"clip_value_loss\"]:\n",
    "                v_loss_unclipped = (newvalue - b_returns[mb_indexes]) ** 2\n",
    "                v_clipped = b_values[mb_indexes] + torch.clamp(\n",
    "                    newvalue - b_values[mb_indexes],\n",
    "                    -params[\"clip_coef\"],\n",
    "                    params[\"clip_coef\"],\n",
    "                )\n",
    "                v_loss_clipped = (v_clipped - b_returns[mb_indexes]) ** 2\n",
    "                v_loss_max = torch.max(v_loss_unclipped, v_loss_clipped)\n",
    "                value_loss = 0.5 * v_loss_max.mean()\n",
    "            else:\n",
    "                value_loss = 0.5 * ((newvalue - b_returns[mb_indexes]) ** 2).mean()\n",
    "\n",
    "            entropy_loss = entropy.mean()\n",
    "            loss = pg_loss - params[\"entropy_coef\"] * entropy_loss + params[\"value_coef\"] * value_loss\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            nn.utils.clip_grad_norm_(player_policy.parameters(), params[\"max_grad_norm\"])\n",
    "            optimizer.step()\n",
    "\n",
    "    # Log game outcome\n",
    "    explained_var = explained_variance_score(b_returns.cpu().numpy(), b_values.cpu().numpy())\n",
    "    if update % 8 == 0:\n",
    "        outcome_vs_random, rating_vs_random = compute_rating(player_policy, RandomAgent(action_size))\n",
    "        outcome_vs_past_iterations, rating_vs_past_iterations = compute_rating(player_policy, pool.get_agent(-50))\n",
    "        _, rating_vs_trained_previous = compute_rating(player_policy, trained_previous)\n",
    "        _, rating_vs_v2 = compute_rating(player_policy, trained_v2)\n",
    "\n",
    "    wandb.log({\n",
    "        \"global_step\": global_step,\n",
    "        \"learning_rate\": optimizer.param_groups[0][\"lr\"],\n",
    "        \"ratio_win_reward\": params[\"ratio_win_reward\"],\n",
    "        \"entropy_coef\": params[\"entropy_coef\"],\n",
    "\n",
    "        \"value_loss\": value_loss.item(),\n",
    "        \"policy_loss\": pg_loss.item(),\n",
    "        \"entropy\": entropy_loss.item(),\n",
    "        \"total_loss\": loss.item(),\n",
    "\n",
    "        \"old_approx_kl\": old_approx_kl.item(),\n",
    "        \"approx_kl\": approx_kl.item(),\n",
    "        \"clipfrac\": np.mean(clip_fraction),\n",
    "        \"explained_variance\": explained_var,\n",
    "        \"SPS\": int(global_step / (time() - start_time)),\n",
    "\n",
    "        \"reward_per_game\": torch.sum(rewards, dim=0).mean(),\n",
    "        \"points_per_game\": sum([env.game_state.agent_points[player_name] for env in vec_env]) / params[\"n_envs\"],\n",
    "        \"mean_outcome\": sum([env.get_game_outcome(player_name) for env in vec_env]) / params[\"n_envs\"],\n",
    "\n",
    "        \"outcome_vs_random\": outcome_vs_random,\n",
    "        \"rating_vs_random\": rating_vs_random,\n",
    "\n",
    "        \"outcome_vs_50_past_iterations\": outcome_vs_past_iterations,\n",
    "        \"rating_vs_50_past_iterations\": rating_vs_past_iterations,\n",
    "\n",
    "        \"rating_vs_best\": rating_vs_trained_previous,\n",
    "        \"rating_vs_v2\": rating_vs_v2,\n",
    "\n",
    "        \"pool_ratings\": wandb.Histogram(pool.ratings),\n",
    "        \"pool_std\": np.std(pool.ratings)\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========== Diagnostic Run torch.onnx.export version 2.0.0.dev20230204 ==========\n",
      "verbose: False, log level: Level.ERROR\n",
      "======================= 0 NONE 0 NOTE 0 WARNING 0 ERROR ========================\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from src.utils.onnx_utils import export_to_onnx\n",
    "export_to_onnx(trained_previous, \"agent.onnx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>SPS</td><td>█▇███████▇▆▅▅▄▃▃▂▂▂▂▁▁▁▁▁▁▁▁▂▂▂▂▂▂▂▃▃▃▃▃</td></tr><tr><td>approx_kl</td><td>█▄▄▃▃▅▃▃▂▆▂▂▂▂▂▂▁▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>clipfrac</td><td>██▇▇▇▆▆▅▆▆▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>entropy</td><td>█▇█▄▄▇▆███▆▇▄▅▄▇▃▅▄▇▄▃▆▅▆▆▆▃▃▄▁▆▃▅▆▁▄▆▃▅</td></tr><tr><td>entropy_coef</td><td>██▇▇▇▆▆▅▅▅▅▅▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁</td></tr><tr><td>explained_variance</td><td>▇▆▆▅▁▆▄▆▆▇▆▄▆█▆▆█▆▅▅▇▃▆▆▆▆▅▆▅▇█▇▅▆▃▇▆▆▆▅</td></tr><tr><td>global_step</td><td>▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███</td></tr><tr><td>learning_rate</td><td>██▇▆▆▆▅▅▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>mean_outcome</td><td>▁▆▄▁█▄▂▁▅▂▄▂▄▃▂▆▅▆█▆█▄▄▄▇▄▁▄▆▇▃▄▅▅▆▇▆▄▄▄</td></tr><tr><td>old_approx_kl</td><td>█▇▆▆▆▄▅▄▅▅▂▃▆▄▃▃▂▂▁▃▂▃▂▂▂▁▂▂▂▂▁▃▁▂▁▂▂▂▁▂</td></tr><tr><td>outcome_vs_50_past_iterations</td><td>▆▃▃▇▄▆▅▄▆▆▄▇▆▅▆▄▂▆▆▅▃▂▄▇▄▃▇▆▇▆▅▄▁▆▇▆▂▆█▃</td></tr><tr><td>outcome_vs_random</td><td>█▆▅▅▅▅▅▆▅▇▆▅▃▅▃▁▅▅▃▅▇▆▂▂▃█▅▆▅▁▃▄▇▁▅▅▄▆▄▄</td></tr><tr><td>points_per_game</td><td>▃▅▃▂▆▄▂▁▄▃▄▃▄▃▃▆▅▆▆▅▇▄▄▃█▄▄▃▅▆▄▅▃▆▃▅▅▄▃▄</td></tr><tr><td>policy_loss</td><td>▁▃▁▃▂▁▂▂▄▂▄▃▆▃▃▄▅▅▂▂▃▇▆▇▇▅▅▇███▆▇▆█▇█▇▆▇</td></tr><tr><td>pool_std</td><td>█▆▄▅▆█▇▅▄▅▄▄▂▂▂▁▁▂▂▂▃▂▂▂▂▂▃▃▃▃▃▃▃▃▃▂▂▃▃▄</td></tr><tr><td>rating_vs_50_past_iterations</td><td>▆▃▃▇▄▆▅▄▆▆▄▇▆▅▆▄▂▆▆▅▃▂▄▇▄▃▇▆▇▆▅▄▁▆▇▆▂▆█▃</td></tr><tr><td>rating_vs_best</td><td>▄▄▅▄▄▅▅▅▂▄▃▂▆▆▅▁█▇▅▅▂▅▄█▄▅▄▄▄▄▆▆▅▇▇▆▇▃▄▃</td></tr><tr><td>rating_vs_random</td><td>█▆▅▄▄▅▄▆▅▇▆▅▃▄▃▁▅▄▃▅▇▆▂▂▃█▅▆▅▁▃▄▇▁▄▅▄▅▃▃</td></tr><tr><td>rating_vs_v2</td><td>▄▁▅▂▂▅█▅▄▄▃▆▅▄▆▄█▄▂▅▅▇▂▂▄█▃█▃▄▂▅██▆▆▅▆▆▆</td></tr><tr><td>ratio_win_reward</td><td>▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>reward_per_game</td><td>▁▆▄▁█▄▂▁▅▂▄▂▄▃▂▆▅▆█▆█▄▄▄▇▄▁▄▆▇▃▄▅▅▆▇▆▄▄▄</td></tr><tr><td>total_loss</td><td>▁▂▁▂▄▁▃▁▃▄▃▃▆▁▄▅▅▅▃▃▄▇▇▆▆▆▆██▇▇▇▇▆█▇█▇▆█</td></tr><tr><td>value_loss</td><td>▃▁▃▃▇▃▆▂▂▇▃▄▅▁▅▅▅▆▇▆▅▅█▅▅▇▆█▆▄▄▇▇▆▇▄▆▆▆█</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>SPS</td><td>3005</td></tr><tr><td>approx_kl</td><td>0.00051</td></tr><tr><td>clipfrac</td><td>0.00409</td></tr><tr><td>entropy</td><td>0.08023</td></tr><tr><td>entropy_coef</td><td>0.00011</td></tr><tr><td>explained_variance</td><td>0.51164</td></tr><tr><td>global_step</td><td>45096960</td></tr><tr><td>learning_rate</td><td>0.0001</td></tr><tr><td>mean_outcome</td><td>0.50928</td></tr><tr><td>old_approx_kl</td><td>0.00119</td></tr><tr><td>outcome_vs_50_past_iterations</td><td>0.51758</td></tr><tr><td>outcome_vs_random</td><td>0.92578</td></tr><tr><td>points_per_game</td><td>60.17383</td></tr><tr><td>policy_loss</td><td>-0.00283</td></tr><tr><td>pool_std</td><td>0.0088</td></tr><tr><td>rating_vs_50_past_iterations</td><td>0.07034</td></tr><tr><td>rating_vs_best</td><td>0.01953</td></tr><tr><td>rating_vs_random</td><td>2.52362</td></tr><tr><td>rating_vs_v2</td><td>0.36736</td></tr><tr><td>ratio_win_reward</td><td>1</td></tr><tr><td>reward_per_game</td><td>0.50928</td></tr><tr><td>total_loss</td><td>0.01819</td></tr><tr><td>value_loss</td><td>0.04205</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">briscola penalization continue</strong> at: <a href=\"https://wandb.ai/lettera/briscolaBot/runs/8vwbmvox\" target=\"_blank\">https://wandb.ai/lettera/briscolaBot/runs/8vwbmvox</a><br/>Synced 7 W&B file(s), 0 media file(s), 15 artifact file(s) and 1 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>.\\wandb\\run-20230214_184936-8vwbmvox\\logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.save(player_policy.state_dict(), 'agent.pt')\n",
    "# Save as artifact for version control.\n",
    "artifact = wandb.Artifact('model', type='model')\n",
    "artifact.add_file('agent.pt')\n",
    "run.log_artifact(artifact)\n",
    "wandb.run.finish()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "f38bca3c9600444284abf86780be38c386e032d708dbe2e8970d98e9b2d0aa36"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}