udacity-project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "gather": {
     "logged": 1598275788035
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Workspace name: quick-starts-ws-125527\n",
      "Azure region: southcentralus\n",
      "Subscription id: 94e14ad4-bf97-47e8-aae0-f9b85a7befa8\n",
      "Resource group: aml-quickstarts-125527\n"
     ]
    }
   ],
   "source": [
    "from azureml.core import Workspace, Experiment\n",
    "\n",
    "ws = Workspace.get(name=\"quick-starts-ws-125527\")\n",
    "exp = Experiment(workspace=ws, name=\"quick-starts-ws-125463\")\n",
    "\n",
    "print('Workspace name: ' + ws.name, \n",
    "      'Azure region: ' + ws.location, \n",
    "      'Subscription id: ' + ws.subscription_id, \n",
    "      'Resource group: ' + ws.resource_group, sep = '\\n')\n",
    "\n",
    "run = exp.start_logging()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "gather": {
     "logged": 1598275788675
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found existing cluster which we'll use instead of creating a new one.\n"
     ]
    }
   ],
   "source": [
    "from azureml.core.compute import ComputeTarget, AmlCompute\n",
    "from azureml.core.compute_target import ComputeTargetException\n",
    "\n",
    "# TODO: Create compute cluster\n",
    "# Use vm_size = \"Standard_D2_V2\" in your provisioning configuration.\n",
    "# max_nodes should be no greater than 4.\n",
    "\n",
    "### YOUR CODE HERE \n",
    "cluster_name=\"cpu_cluster\" #The compute cluster created before in the Azure portal\n",
    "try:\n",
    "    # Checking for existing compute target\n",
    "    train_cluster = ComputeTarget(workspace=ws, name=\"cpu-cluster\")\n",
    "    print(\"Found existing cluster which we'll use instead of creating a new one.\")\n",
    "except ComputeTargetException:\n",
    "    # If it doesn't exist, create a compute cluster\n",
    "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2', max_nodes=4)\n",
    "    train_cluster = ComputeTarget.create(ws, \"train_cluster\", compute_config)\n",
    "    train_cluster.wait_for_completion(show_output=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "gather": {
     "logged": 1598275789986
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "outputs": [],
   "source": [
    "from azureml.widgets import RunDetails\n",
    "from azureml.train.sklearn import SKLearn\n",
    "from azureml.train.hyperdrive.run import PrimaryMetricGoal\n",
    "from azureml.train.hyperdrive.policy import BanditPolicy\n",
    "from azureml.train.hyperdrive.sampling import RandomParameterSampling\n",
    "from azureml.train.hyperdrive.runconfig import HyperDriveConfig\n",
    "from azureml.train.hyperdrive.parameter_expressions import uniform,choice\n",
    "import os\n",
    "\n",
    "# Specify parameter sampler\n",
    "### YOUR CODE HERE ###\n",
    "\n",
    "ps = RandomParameterSampling( {\n",
    "        \"--C\":uniform(0.05,1),\n",
    "        \"--max_iter\":choice(20,40,60,80,100)\n",
    "    }\n",
    ") \n",
    "\n",
    "\n",
    "# Specify a Policy\n",
    "### YOUR CODE HERE ###\n",
    "early_termination_policy = BanditPolicy(evaluation_interval=2,slack_factor=0.2)\n",
    "\n",
    "# Create a SKLearn estimator for use with train.py\n",
    "### YOUR CODE HERE ###\n",
    "from azureml.core import ScriptRunConfig\n",
    "from azureml.core import Environment\n",
    "script='train.py'\n",
    "est=SKLearn(source_directory='.',entry_script=script,compute_target=train_cluster)\n",
    "\n",
    "\n",
    "# Create a HyperDriveConfig using the estimator, hyperparameter sampler, and policy.\n",
    "### YOUR CODE HERE ###\n",
    "hyperdrive_config = HyperDriveConfig(estimator=est,\n",
    "                             hyperparameter_sampling=ps,\n",
    "                             policy=early_termination_policy,\n",
    "                             primary_metric_name='Accuracy',\n",
    "                             primary_metric_goal=PrimaryMetricGoal.MAXIMIZE,\n",
    "                             max_total_runs=4,\n",
    "                             max_concurrent_runs=4\n",
    "                                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING - If 'script' has been provided here and a script file name has been specified in 'run_config', 'script' provided in ScriptRunConfig initialization will take precedence.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7dc4be289b664de1a314b13c56eedd56",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "_HyperDriveWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/aml.mini.widget.v1": "{\"status\": \"Completed\", \"workbench_run_details_uri\": \"https://ml.azure.com/experiments/quick-starts-ws-125463/runs/HD_1d971ac1-9645-49b0-bd09-e80dd84cd694?wsid=/subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourcegroups/aml-quickstarts-125527/workspaces/quick-starts-ws-125527\", \"run_id\": \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694\", \"run_properties\": {\"run_id\": \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694\", \"created_utc\": \"2020-11-10T14:48:31.789935Z\", \"properties\": {\"primary_metric_config\": \"{\\\"name\\\": \\\"Accuracy\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"resume_from\": \"null\", \"runTemplate\": \"HyperDrive\", \"azureml.runsource\": \"hyperdrive\", \"platform\": \"AML\", \"ContentSnapshotId\": \"85945b44-aa5e-4665-8bb6-b0ee6279d871\", \"score\": \"0.9072837632776934\", \"best_child_run_id\": \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_0\", \"best_metric_status\": \"Succeeded\"}, \"tags\": {\"_aml_system_max_concurrent_jobs\": \"4\", \"max_concurrent_jobs\": \"4\", \"_aml_system_max_total_jobs\": \"4\", \"max_total_jobs\": \"4\", \"_aml_system_max_duration_minutes\": \"10080\", \"max_duration_minutes\": \"10080\", \"_aml_system_policy_config\": \"{\\\"name\\\": \\\"BANDIT\\\", \\\"properties\\\": {\\\"evaluation_interval\\\": 2, \\\"delay_evaluation\\\": 0, \\\"slack_factor\\\": 0.2}}\", \"policy_config\": \"{\\\"name\\\": \\\"BANDIT\\\", \\\"properties\\\": {\\\"evaluation_interval\\\": 2, \\\"delay_evaluation\\\": 0, \\\"slack_factor\\\": 0.2}}\", \"_aml_system_generator_config\": \"{\\\"name\\\": \\\"RANDOM\\\", \\\"parameter_space\\\": {\\\"--C\\\": [\\\"uniform\\\", [0.05, 1]], \\\"--max_iter\\\": [\\\"choice\\\", [[20, 40, 60, 80, 100]]]}}\", \"generator_config\": \"{\\\"name\\\": \\\"RANDOM\\\", \\\"parameter_space\\\": {\\\"--C\\\": [\\\"uniform\\\", [0.05, 1]], \\\"--max_iter\\\": [\\\"choice\\\", [[20, 40, 60, 80, 100]]]}}\", \"_aml_system_primary_metric_config\": \"{\\\"name\\\": \\\"Accuracy\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"primary_metric_config\": \"{\\\"name\\\": \\\"Accuracy\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"_aml_system_platform_config\": \"{\\\"ServiceAddress\\\": \\\"https://southcentralus.experiments.azureml.net\\\", \\\"ServiceArmScope\\\": \\\"subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourceGroups/aml-quickstarts-125527/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-125527/experiments/quick-starts-ws-125463\\\", \\\"SubscriptionId\\\": \\\"94e14ad4-bf97-47e8-aae0-f9b85a7befa8\\\", \\\"ResourceGroupName\\\": \\\"aml-quickstarts-125527\\\", \\\"WorkspaceName\\\": \\\"quick-starts-ws-125527\\\", \\\"ExperimentName\\\": \\\"quick-starts-ws-125463\\\", \\\"Definition\\\": {\\\"Overrides\\\": {\\\"script\\\": \\\"train.py\\\", \\\"arguments\\\": [], \\\"target\\\": \\\"cpu-cluster\\\", \\\"framework\\\": \\\"Python\\\", \\\"communicator\\\": \\\"None\\\", \\\"maxRunDurationSeconds\\\": null, \\\"nodeCount\\\": 1, \\\"environment\\\": {\\\"name\\\": null, \\\"version\\\": null, \\\"environmentVariables\\\": {\\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\"}, \\\"python\\\": {\\\"userManagedDependencies\\\": false, \\\"interpreterPath\\\": \\\"python\\\", \\\"condaDependenciesFile\\\": null, \\\"baseCondaEnvironment\\\": null, \\\"condaDependencies\\\": {\\\"name\\\": \\\"project_environment\\\", \\\"dependencies\\\": [\\\"python=3.6.2\\\", {\\\"pip\\\": [\\\"azureml-defaults\\\", \\\"scikit-learn==0.20.3\\\", \\\"scipy==1.2.1\\\", \\\"joblib==0.13.2\\\"]}], \\\"channels\\\": [\\\"anaconda\\\", \\\"conda-forge\\\"]}}, \\\"docker\\\": {\\\"enabled\\\": true, \\\"baseImage\\\": \\\"mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200423.v1\\\", \\\"baseDockerfile\\\": null, \\\"sharedVolumes\\\": true, \\\"shmSize\\\": \\\"2g\\\", \\\"arguments\\\": [], \\\"baseImageRegistry\\\": {\\\"address\\\": null, \\\"username\\\": null, \\\"password\\\": null, \\\"registryIdentity\\\": null}, \\\"platform\\\": {\\\"os\\\": \\\"Linux\\\", \\\"architecture\\\": \\\"amd64\\\"}}, \\\"spark\\\": {\\\"repositories\\\": [], \\\"packages\\\": [], \\\"precachePackages\\\": false}, \\\"databricks\\\": {\\\"mavenLibraries\\\": [], \\\"pypiLibraries\\\": [], \\\"rcranLibraries\\\": [], \\\"jarLibraries\\\": [], \\\"eggLibraries\\\": []}, \\\"r\\\": null, \\\"inferencingStackVersion\\\": null}, \\\"history\\\": {\\\"outputCollection\\\": true, \\\"snapshotProject\\\": true, \\\"directoriesToWatch\\\": [\\\"logs\\\"]}, \\\"spark\\\": {\\\"configuration\\\": {\\\"spark.app.name\\\": \\\"Azure ML Experiment\\\", \\\"spark.yarn.maxAppAttempts\\\": 1}}, \\\"hdi\\\": {\\\"yarnDeployMode\\\": \\\"cluster\\\"}, \\\"tensorflow\\\": {\\\"workerCount\\\": 1, \\\"parameterServerCount\\\": 1}, \\\"mpi\\\": {\\\"processCountPerNode\\\": 1, \\\"nodeCount\\\": 1}, \\\"paralleltask\\\": {\\\"maxRetriesPerWorker\\\": 0, \\\"workerCountPerNode\\\": 1, \\\"terminalExitCodes\\\": null}, \\\"dataReferences\\\": {}, \\\"data\\\": {}, \\\"outputData\\\": {}, \\\"sourceDirectoryDataStore\\\": null, \\\"amlcompute\\\": {\\\"vmSize\\\": null, \\\"vmPriority\\\": null, \\\"retainCluster\\\": false, \\\"name\\\": null, \\\"clusterMaxNodeCount\\\": 1}}, \\\"TargetDetails\\\": null, \\\"SnapshotId\\\": \\\"85945b44-aa5e-4665-8bb6-b0ee6279d871\\\", \\\"TelemetryValues\\\": {\\\"amlClientType\\\": \\\"azureml-sdk-train\\\", \\\"amlClientModule\\\": \\\"[Scrubbed]\\\", \\\"amlClientFunction\\\": \\\"[Scrubbed]\\\", \\\"tenantId\\\": \\\"660b3398-b80e-49d2-bc5b-ac1dc93b5254\\\", \\\"amlClientRequestId\\\": \\\"52a610a6-2e33-4098-9a8a-a32430a94afc\\\", \\\"amlClientSessionId\\\": \\\"3cb82550-f204-41cd-90d3-72393029d7c8\\\", \\\"subscriptionId\\\": \\\"94e14ad4-bf97-47e8-aae0-f9b85a7befa8\\\", \\\"estimator\\\": \\\"SKLearn\\\", \\\"samplingMethod\\\": \\\"RANDOM\\\", \\\"terminationPolicy\\\": \\\"Bandit\\\", \\\"primaryMetricGoal\\\": \\\"maximize\\\", \\\"maxTotalRuns\\\": 4, \\\"maxConcurrentRuns\\\": 4, \\\"maxDurationMinutes\\\": 10080, \\\"vmSize\\\": null}}}\", \"platform_config\": \"{\\\"ServiceAddress\\\": \\\"https://southcentralus.experiments.azureml.net\\\", \\\"ServiceArmScope\\\": \\\"subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourceGroups/aml-quickstarts-125527/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-125527/experiments/quick-starts-ws-125463\\\", \\\"SubscriptionId\\\": \\\"94e14ad4-bf97-47e8-aae0-f9b85a7befa8\\\", \\\"ResourceGroupName\\\": \\\"aml-quickstarts-125527\\\", \\\"WorkspaceName\\\": \\\"quick-starts-ws-125527\\\", \\\"ExperimentName\\\": \\\"quick-starts-ws-125463\\\", \\\"Definition\\\": {\\\"Overrides\\\": {\\\"script\\\": \\\"train.py\\\", \\\"arguments\\\": [], \\\"target\\\": \\\"cpu-cluster\\\", \\\"framework\\\": \\\"Python\\\", \\\"communicator\\\": \\\"None\\\", \\\"maxRunDurationSeconds\\\": null, \\\"nodeCount\\\": 1, \\\"environment\\\": {\\\"name\\\": null, \\\"version\\\": null, \\\"environmentVariables\\\": {\\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\"}, \\\"python\\\": {\\\"userManagedDependencies\\\": false, \\\"interpreterPath\\\": \\\"python\\\", \\\"condaDependenciesFile\\\": null, \\\"baseCondaEnvironment\\\": null, \\\"condaDependencies\\\": {\\\"name\\\": \\\"project_environment\\\", \\\"dependencies\\\": [\\\"python=3.6.2\\\", {\\\"pip\\\": [\\\"azureml-defaults\\\", \\\"scikit-learn==0.20.3\\\", \\\"scipy==1.2.1\\\", \\\"joblib==0.13.2\\\"]}], \\\"channels\\\": [\\\"anaconda\\\", \\\"conda-forge\\\"]}}, \\\"docker\\\": {\\\"enabled\\\": true, \\\"baseImage\\\": \\\"mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200423.v1\\\", \\\"baseDockerfile\\\": null, \\\"sharedVolumes\\\": true, \\\"shmSize\\\": \\\"2g\\\", \\\"arguments\\\": [], \\\"baseImageRegistry\\\": {\\\"address\\\": null, \\\"username\\\": null, \\\"password\\\": null, \\\"registryIdentity\\\": null}, \\\"platform\\\": {\\\"os\\\": \\\"Linux\\\", \\\"architecture\\\": \\\"amd64\\\"}}, \\\"spark\\\": {\\\"repositories\\\": [], \\\"packages\\\": [], \\\"precachePackages\\\": false}, \\\"databricks\\\": {\\\"mavenLibraries\\\": [], \\\"pypiLibraries\\\": [], \\\"rcranLibraries\\\": [], \\\"jarLibraries\\\": [], \\\"eggLibraries\\\": []}, \\\"r\\\": null, \\\"inferencingStackVersion\\\": null}, \\\"history\\\": {\\\"outputCollection\\\": true, \\\"snapshotProject\\\": true, \\\"directoriesToWatch\\\": [\\\"logs\\\"]}, \\\"spark\\\": {\\\"configuration\\\": {\\\"spark.app.name\\\": \\\"Azure ML Experiment\\\", \\\"spark.yarn.maxAppAttempts\\\": 1}}, \\\"hdi\\\": {\\\"yarnDeployMode\\\": \\\"cluster\\\"}, \\\"tensorflow\\\": {\\\"workerCount\\\": 1, \\\"parameterServerCount\\\": 1}, \\\"mpi\\\": {\\\"processCountPerNode\\\": 1, \\\"nodeCount\\\": 1}, \\\"paralleltask\\\": {\\\"maxRetriesPerWorker\\\": 0, \\\"workerCountPerNode\\\": 1, \\\"terminalExitCodes\\\": null}, \\\"dataReferences\\\": {}, \\\"data\\\": {}, \\\"outputData\\\": {}, \\\"sourceDirectoryDataStore\\\": null, \\\"amlcompute\\\": {\\\"vmSize\\\": null, \\\"vmPriority\\\": null, \\\"retainCluster\\\": false, \\\"name\\\": null, \\\"clusterMaxNodeCount\\\": 1}}, \\\"TargetDetails\\\": null, \\\"SnapshotId\\\": \\\"85945b44-aa5e-4665-8bb6-b0ee6279d871\\\", \\\"TelemetryValues\\\": {\\\"amlClientType\\\": \\\"azureml-sdk-train\\\", \\\"amlClientModule\\\": \\\"[Scrubbed]\\\", \\\"amlClientFunction\\\": \\\"[Scrubbed]\\\", \\\"tenantId\\\": \\\"660b3398-b80e-49d2-bc5b-ac1dc93b5254\\\", \\\"amlClientRequestId\\\": \\\"52a610a6-2e33-4098-9a8a-a32430a94afc\\\", \\\"amlClientSessionId\\\": \\\"3cb82550-f204-41cd-90d3-72393029d7c8\\\", \\\"subscriptionId\\\": \\\"94e14ad4-bf97-47e8-aae0-f9b85a7befa8\\\", \\\"estimator\\\": \\\"SKLearn\\\", \\\"samplingMethod\\\": \\\"RANDOM\\\", \\\"terminationPolicy\\\": \\\"Bandit\\\", \\\"primaryMetricGoal\\\": \\\"maximize\\\", \\\"maxTotalRuns\\\": 4, \\\"maxConcurrentRuns\\\": 4, \\\"maxDurationMinutes\\\": 10080, \\\"vmSize\\\": null}}}\", \"_aml_system_resume_child_runs\": \"null\", \"resume_child_runs\": \"null\", \"_aml_system_all_jobs_generated\": \"true\", \"all_jobs_generated\": \"true\", \"_aml_system_cancellation_requested\": \"false\", \"cancellation_requested\": \"false\", \"_aml_system_progress_metadata_evaluation_timestamp\": \"\\\"2020-11-10T14:48:33.091132\\\"\", \"progress_metadata_evaluation_timestamp\": \"\\\"2020-11-10T14:48:33.091132\\\"\", \"_aml_system_progress_metadata_digest\": \"\\\"5f8478055f18fee8ea86f9b823ae764da02e31b7164d453a57af2cdd5c2dc5f0\\\"\", \"progress_metadata_digest\": \"\\\"5f8478055f18fee8ea86f9b823ae764da02e31b7164d453a57af2cdd5c2dc5f0\\\"\", \"_aml_system_progress_metadata_active_timestamp\": \"\\\"2020-11-10T14:48:33.091132\\\"\", \"progress_metadata_active_timestamp\": \"\\\"2020-11-10T14:48:33.091132\\\"\", \"_aml_system_HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_0\": \"{\\\"--C\\\": 0.14994722607487992, \\\"--max_iter\\\": 80}\", \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_0\": \"{\\\"--C\\\": 0.14994722607487992, \\\"--max_iter\\\": 80}\", \"_aml_system_HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_1\": \"{\\\"--C\\\": 0.353066044803669, \\\"--max_iter\\\": 100}\", \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_1\": \"{\\\"--C\\\": 0.353066044803669, \\\"--max_iter\\\": 100}\", \"_aml_system_HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_2\": \"{\\\"--C\\\": 0.742512140100084, \\\"--max_iter\\\": 20}\", \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_2\": \"{\\\"--C\\\": 0.742512140100084, \\\"--max_iter\\\": 20}\", \"_aml_system_HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_3\": \"{\\\"--C\\\": 0.09923385581734294, \\\"--max_iter\\\": 20}\", \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_3\": \"{\\\"--C\\\": 0.09923385581734294, \\\"--max_iter\\\": 20}\", \"_aml_system_environment_preparation_status\": \"PREPARED\", \"environment_preparation_status\": \"PREPARED\", \"_aml_system_prepare_run_id\": \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_preparation\", \"prepare_run_id\": \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_preparation\", \"_aml_system_final_best_metric_update_retry_count\": \"1\", \"final_best_metric_update_retry_count\": \"1\"}, \"end_time_utc\": \"2020-11-10T14:52:45.439012Z\", \"status\": \"Completed\", \"log_files\": {\"azureml-logs/hyperdrive.txt\": \"https://mlstrg125527.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_1d971ac1-9645-49b0-bd09-e80dd84cd694/azureml-logs/hyperdrive.txt?sv=2019-02-02&sr=b&sig=%2FBRbKvYTgz9x1BEr4xnBe%2BOmqh1W%2Bf051AJvunrk%2FuU%3D&st=2020-11-10T14%3A43%3A02Z&se=2020-11-10T22%3A53%3A02Z&sp=r\"}, \"log_groups\": [[\"azureml-logs/hyperdrive.txt\"]], \"run_duration\": \"0:04:13\", \"hyper_parameters\": {\"--C\": [\"uniform\", [0.05, 1]], \"--max_iter\": [\"choice\", [[20, 40, 60, 80, 100]]]}}, \"child_runs\": [{\"run_id\": \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_1\", \"run_number\": 64, \"metric\": 0.90728376, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2020-11-10T14:49:17.407424Z\", \"end_time\": \"2020-11-10T14:50:11.666601Z\", \"created_time\": \"2020-11-10T14:49:06.565593Z\", \"created_time_dt\": \"2020-11-10T14:49:06.565593Z\", \"duration\": \"0:01:05\", \"hyperdrive_id\": \"1d971ac1-9645-49b0-bd09-e80dd84cd694\", \"arguments\": null, \"param_--C\": 0.353066044803669, \"param_--max_iter\": 100, \"best_metric\": 0.90728376}, {\"run_id\": \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_2\", \"run_number\": 65, \"metric\": 0.90728376, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2020-11-10T14:49:17.801655Z\", \"end_time\": \"2020-11-10T14:51:55.327079Z\", \"created_time\": \"2020-11-10T14:49:06.874281Z\", \"created_time_dt\": \"2020-11-10T14:49:06.874281Z\", \"duration\": \"0:02:48\", \"hyperdrive_id\": \"1d971ac1-9645-49b0-bd09-e80dd84cd694\", \"arguments\": null, \"param_--C\": 0.742512140100084, \"param_--max_iter\": 20, \"best_metric\": 0.90728376}, {\"run_id\": \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_0\", \"run_number\": 66, \"metric\": 0.90728376, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2020-11-10T14:49:18.01874Z\", \"end_time\": \"2020-11-10T14:50:15.08814Z\", \"created_time\": \"2020-11-10T14:49:06.898504Z\", \"created_time_dt\": \"2020-11-10T14:49:06.898504Z\", \"duration\": \"0:01:08\", \"hyperdrive_id\": \"1d971ac1-9645-49b0-bd09-e80dd84cd694\", \"arguments\": null, \"param_--C\": 0.14994722607487992, \"param_--max_iter\": 80, \"best_metric\": 0.90728376}, {\"run_id\": \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_3\", \"run_number\": 67, \"metric\": 0.90728376, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2020-11-10T14:49:16.716769Z\", \"end_time\": \"2020-11-10T14:50:20.117082Z\", \"created_time\": \"2020-11-10T14:49:07.359561Z\", \"created_time_dt\": \"2020-11-10T14:49:07.359561Z\", \"duration\": \"0:01:12\", \"hyperdrive_id\": \"1d971ac1-9645-49b0-bd09-e80dd84cd694\", \"arguments\": null, \"param_--C\": 0.09923385581734294, \"param_--max_iter\": 20, \"best_metric\": 0.90728376}], \"children_metrics\": {\"categories\": [0], \"series\": {\"Regularization Strength:\": [{\"categories\": [64, 65, 66, 67], \"mode\": \"markers\", \"name\": \"Regularization Strength:\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.353066044803669, 0.742512140100084, 0.14994722607487992, 0.09923385581734294]}, {\"categories\": [64, 65, 66, 67], \"mode\": \"lines\", \"name\": \"Regularization Strength:_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.353066044803669, 0.742512140100084, 0.742512140100084, 0.742512140100084]}], \"Max iterations:\": [{\"categories\": [64, 65, 66, 67], \"mode\": \"markers\", \"name\": \"Max iterations:\", \"stepped\": false, \"type\": \"scatter\", \"data\": [100, 20, 80, 20]}, {\"categories\": [64, 65, 66, 67], \"mode\": \"lines\", \"name\": \"Max iterations:_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [100, 100, 100, 100]}], \"Accuracy\": [{\"categories\": [64, 65, 66, 67], \"mode\": \"markers\", \"name\": \"Accuracy\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9072837632776934, 0.9072837632776934, 0.9072837632776934, 0.9072837632776934]}, {\"categories\": [64, 65, 66, 67], \"mode\": \"lines\", \"name\": \"Accuracy_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9072837632776934, 0.9072837632776934, 0.9072837632776934, 0.9072837632776934]}]}, \"metricName\": null, \"primaryMetricName\": \"Accuracy\", \"showLegend\": false}, \"run_metrics\": [{\"name\": \"best_child_by_primary_metric\", \"run_id\": \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694\", \"categories\": [0], \"series\": [{\"data\": [{\"metric_name\": [\"Accuracy\", \"Accuracy\"], \"timestamp\": [\"2020-11-10 14:50:32.692825+00:00\", \"2020-11-10 14:50:32.692825+00:00\"], \"run_id\": [\"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_0\", \"HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_0\"], \"metric_value\": [0.9072837632776934, 0.9072837632776934], \"final\": [false, true]}]}]}], \"run_logs\": \"[2020-11-10T14:48:32.735634][API][INFO]Experiment created\\r\\n[2020-11-10T14:48:33.370343][GENERATOR][INFO]Trying to sample '4' jobs from the hyperparameter space\\r\\n[2020-11-10T14:48:33.562309][GENERATOR][INFO]Successfully sampled '4' jobs, they will soon be submitted to the execution target.\\r\\n[2020-11-10T14:48:34.4226654Z][SCHEDULER][INFO]The execution environment is being prepared. Please be patient as it can take a few minutes.\\r\\n[2020-11-10T14:49:02.948312][GENERATOR][INFO]Max number of jobs '4' reached for experiment.\\r\\n[2020-11-10T14:49:03.071673][GENERATOR][INFO]All jobs generated.\\r\\n[2020-11-10T14:49:05.9649238Z][SCHEDULER][INFO]Scheduling job, id='HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_3'\\r\\n[2020-11-10T14:49:05.9524875Z][SCHEDULER][INFO]Scheduling job, id='HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_1'\\r\\n[2020-11-10T14:49:05.9624147Z][SCHEDULER][INFO]Scheduling job, id='HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_2'\\r\\n[2020-11-10T14:49:05.9511953Z][SCHEDULER][INFO]Scheduling job, id='HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_0'\\r\\n[2020-11-10T14:49:05.9504340Z][SCHEDULER][INFO]The execution environment was successfully prepared.\\r\\n[2020-11-10T14:49:06.7107733Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_1'\\r\\n[2020-11-10T14:49:07.0839325Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_2'\\r\\n[2020-11-10T14:49:07.3416356Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_0'\\r\\n[2020-11-10T14:49:07.4844178Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_3'\\r\\n[2020-11-10T14:49:33.184683][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourceGroups/aml-quickstarts-125527/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-125527/experiments/**SCRUBBED**/runs/HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_0, https://southcentralus.experiments.azureml.net/subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourceGroups/aml-quickstarts-125527/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-125527/experiments/**SCRUBBED**/runs/HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_1, https://southcentralus.experiments.azureml.net/subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourceGroups/aml-quickstarts-125527/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-125527/experiments/**SCRUBBED**/runs/HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_2, https://southcentralus.experiments.azureml.net/subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourceGroups/aml-quickstarts-125527/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-125527/experiments/**SCRUBBED**/runs/HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2020-11-10T14:50:04.327394][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourceGroups/aml-quickstarts-125527/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-125527/experiments/**SCRUBBED**/runs/HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_0, https://southcentralus.experiments.azureml.net/subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourceGroups/aml-quickstarts-125527/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-125527/experiments/**SCRUBBED**/runs/HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_1, https://southcentralus.experiments.azureml.net/subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourceGroups/aml-quickstarts-125527/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-125527/experiments/**SCRUBBED**/runs/HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_2, https://southcentralus.experiments.azureml.net/subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourceGroups/aml-quickstarts-125527/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-125527/experiments/**SCRUBBED**/runs/HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2020-11-10T14:50:34.224325][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourceGroups/aml-quickstarts-125527/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-125527/experiments/**SCRUBBED**/runs/HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_2] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2020-11-10T14:52:45.930356][CONTROLLER][INFO]Experiment was 'ExperimentStatus.RUNNING', is 'ExperimentStatus.FINISHED'.\\n\\nRun is completed.\", \"graph\": {}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": false, \"log_level\": \"INFO\", \"sdk_version\": \"1.17.0\"}, \"loading\": false}"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RunId: HD_1d971ac1-9645-49b0-bd09-e80dd84cd694\n",
      "Web View: https://ml.azure.com/experiments/quick-starts-ws-125463/runs/HD_1d971ac1-9645-49b0-bd09-e80dd84cd694?wsid=/subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourcegroups/aml-quickstarts-125527/workspaces/quick-starts-ws-125527\n",
      "\n",
      "Streaming azureml-logs/hyperdrive.txt\n",
      "=====================================\n",
      "\n",
      "\"<START>[2020-11-10T14:48:32.735634][API][INFO]Experiment created<END>\\n\"\"<START>[2020-11-10T14:48:33.370343][GENERATOR][INFO]Trying to sample '4' jobs from the hyperparameter space<END>\\n\"\"<START>[2020-11-10T14:48:33.562309][GENERATOR][INFO]Successfully sampled '4' jobs, they will soon be submitted to the execution target.<END>\\n\"<START>[2020-11-10T14:48:34.4226654Z][SCHEDULER][INFO]The execution environment is being prepared. Please be patient as it can take a few minutes.<END>\n",
      "\n",
      "Execution Summary\n",
      "=================\n",
      "RunId: HD_1d971ac1-9645-49b0-bd09-e80dd84cd694\n",
      "Web View: https://ml.azure.com/experiments/quick-starts-ws-125463/runs/HD_1d971ac1-9645-49b0-bd09-e80dd84cd694?wsid=/subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourcegroups/aml-quickstarts-125527/workspaces/quick-starts-ws-125527\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'runId': 'HD_1d971ac1-9645-49b0-bd09-e80dd84cd694',\n",
       " 'target': 'cpu-cluster',\n",
       " 'status': 'Completed',\n",
       " 'startTimeUtc': '2020-11-10T14:48:32.114213Z',\n",
       " 'endTimeUtc': '2020-11-10T14:52:45.439012Z',\n",
       " 'properties': {'primary_metric_config': '{\"name\": \"Accuracy\", \"goal\": \"maximize\"}',\n",
       "  'resume_from': 'null',\n",
       "  'runTemplate': 'HyperDrive',\n",
       "  'azureml.runsource': 'hyperdrive',\n",
       "  'platform': 'AML',\n",
       "  'ContentSnapshotId': '85945b44-aa5e-4665-8bb6-b0ee6279d871',\n",
       "  'score': '0.9072837632776934',\n",
       "  'best_child_run_id': 'HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_0',\n",
       "  'best_metric_status': 'Succeeded'},\n",
       " 'inputDatasets': [],\n",
       " 'outputDatasets': [],\n",
       " 'logFiles': {'azureml-logs/hyperdrive.txt': 'https://mlstrg125527.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_1d971ac1-9645-49b0-bd09-e80dd84cd694/azureml-logs/hyperdrive.txt?sv=2019-02-02&sr=b&sig=%2FBRbKvYTgz9x1BEr4xnBe%2BOmqh1W%2Bf051AJvunrk%2FuU%3D&st=2020-11-10T14%3A43%3A02Z&se=2020-11-10T22%3A53%3A02Z&sp=r'}}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Submit your hyperdrive run to the experiment and show run details with the widget.\n",
    "\n",
    "### YOUR CODE HERE ###\n",
    "hyperdrive_run=exp.submit(hyperdrive_config,show_output=True )\n",
    "RunDetails(hyperdrive_run).show() #Run history widget\n",
    "hyperdrive_run.wait_for_completion(show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "gather": {
     "logged": 1598276310862
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Run Id:  HD_1d971ac1-9645-49b0-bd09-e80dd84cd694_0\n",
      "{'Regularization Strength:': 0.14994722607487992, 'Max iterations:': 80, 'Accuracy': 0.9072837632776934}\n"
     ]
    }
   ],
   "source": [
    "import joblib\n",
    "# Get your best run and save the model from that run.\n",
    "### YOUR CODE HERE ###\n",
    "best_run = hyperdrive_run.get_best_run_by_primary_metric()\n",
    "print('Best Run Id: ', best_run.id)\n",
    "best_run_metrics = best_run.get_metrics()\n",
    "print(best_run_metrics)\n",
    "\n",
    "best_run.download_file(\"outputs/hyperdrive_model.joblib\",\"./outputs/hyperdrive_model.joblib\")\n",
    "model=best_run.register_model(model_name='hyperdrive_model',model_path='outputs/hyperdrive_model.joblib',tags={'Training context':'Hyperdrive'},\n",
    "                        properties={'Accuracy': best_run_metrics['Accuracy']})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.data.dataset_factory import TabularDatasetFactory\n",
    "\n",
    "# Create TabularDataset using TabularDatasetFactory\n",
    "# Data is available at: \n",
    "# \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/bankmarketing_train.csv\"\n",
    "\n",
    "### YOUR CODE HERE ###\n",
    "src= \"https://automlsamplenotebookdata.blob.core.windows.net/automl-sample-notebook-data/bankmarketing_train.csv\"\n",
    "ds=TabularDatasetFactory.from_delimited_files(path=src)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "gather": {
     "logged": 1598275726969
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "outputs": [],
   "source": [
    "from train import clean_data\n",
    "\n",
    "# Use the clean_data function to clean your data.\n",
    "x, y = clean_data(ds) ### YOUR DATA OBJECT HERE ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#splitting the data\n",
    "from sklearn.model_selection import train_test_split  \n",
    "import pandas as pd\n",
    "x_train,x_test,y_train,y_test= train_test_split(x,y, train_size=0.8, test_size=0.2, random_state=42) \n",
    "#Concating the x_train & y_train together\n",
    "train_data=pd.concat((x_train,y_train),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "gather": {
     "logged": 1598275665403
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "outputs": [],
   "source": [
    "from azureml.train.automl import AutoMLConfig\n",
    "\n",
    "# Set parameters for AutoMLConfig\n",
    "# NOTE: DO NOT CHANGE THE experiment_timeout_minutes PARAMETER OR YOUR INSTANCE WILL TIME OUT.\n",
    "# If you wish to run the experiment longer, you will need to run this notebook in your own\n",
    "# Azure tenant, which will incur personal costs.\n",
    "automl_config = AutoMLConfig(\n",
    "    experiment_timeout_minutes=30,\n",
    "    task=\"classification\",\n",
    "    primary_metric=\"accuracy\",\n",
    "    training_data=train_data,\n",
    "    label_column_name=\"y\",\n",
    "    n_cross_validations=2,\n",
    "    max_concurrent_iterations=4,\n",
    "    featurization='auto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on local machine\n",
      "Parent Run ID: AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\n",
      "\n",
      "Current status: DatasetEvaluation. Gathering dataset statistics.\n",
      "Current status: FeaturesGeneration. Generating features for the dataset.\n",
      "Current status: DatasetFeaturization. Beginning to fit featurizers and featurize the dataset.\n",
      "Current status: DatasetFeaturizationCompleted. Completed fit featurizers and featurizing the dataset.\n",
      "Current status: DatasetBalancing. Performing class balancing sweeping\n",
      "Current status: DatasetCrossValidationSplit. Generating individually featurized CV splits.\n",
      "\n",
      "****************************************************************************************************\n",
      "DATA GUARDRAILS: \n",
      "\n",
      "TYPE:         Class balancing detection\n",
      "STATUS:       ALERTED\n",
      "DESCRIPTION:  To decrease model bias, please cancel the current run and fix balancing problem.\n",
      "              Learn more about imbalanced data: https://aka.ms/AutomatedMLImbalancedData\n",
      "DETAILS:      Imbalanced data can lead to a falsely perceived positive effect of a model's accuracy because the input data has bias towards one class.\n",
      "+---------------------------------+---------------------------------+--------------------------------------+\n",
      "|Size of the smallest class       |Name/Label of the smallest class |Number of samples in the training data|\n",
      "+=================================+=================================+======================================+\n",
      "|2951                             |1                                |26360                                 |\n",
      "+---------------------------------+---------------------------------+--------------------------------------+\n",
      "\n",
      "****************************************************************************************************\n",
      "\n",
      "TYPE:         Missing feature values imputation\n",
      "STATUS:       PASSED\n",
      "DESCRIPTION:  No feature missing values were detected in the training data.\n",
      "              Learn more about missing value imputation: https://aka.ms/AutomatedMLFeaturization\n",
      "\n",
      "****************************************************************************************************\n",
      "\n",
      "TYPE:         High cardinality feature detection\n",
      "STATUS:       PASSED\n",
      "DESCRIPTION:  Your inputs were analyzed, and no high cardinality features were detected.\n",
      "              Learn more about high cardinality feature handling: https://aka.ms/AutomatedMLFeaturization\n",
      "\n",
      "****************************************************************************************************\n",
      "Current status: ModelSelection. Beginning model selection.\n",
      "\n",
      "****************************************************************************************************\n",
      "ITERATION: The iteration being evaluated.\n",
      "PIPELINE: A summary description of the pipeline being evaluated.\n",
      "DURATION: Time taken for the current iteration.\n",
      "METRIC: The result of computing score on the fitted pipeline.\n",
      "BEST: The best observed score thus far.\n",
      "****************************************************************************************************\n",
      "\n",
      " ITERATION   PIPELINE                                       DURATION      METRIC      BEST\n",
      "         0   MaxAbsScaler LightGBM                          0:00:28       0.9148    0.9148\n",
      "         1   MaxAbsScaler XGBoostClassifier                 0:00:27       0.9154    0.9154\n",
      "         2   MaxAbsScaler RandomForest                      0:00:23       0.8926    0.9154\n",
      "         3   MaxAbsScaler RandomForest                      0:00:23       0.8881    0.9154\n",
      "         4   MaxAbsScaler SGD                               0:00:28       0.8437    0.9154\n",
      "         5   MaxAbsScaler SGD                               0:00:28       0.9066    0.9154\n",
      "         6   MaxAbsScaler ExtremeRandomTrees                0:00:30       0.9001    0.9154\n",
      "         7   MaxAbsScaler ExtremeRandomTrees                0:00:24       0.8999    0.9154\n",
      "         8   MaxAbsScaler ExtremeRandomTrees                0:00:25       0.9000    0.9154\n",
      "         9   MaxAbsScaler ExtremeRandomTrees                0:00:23       0.7548    0.9154\n",
      "        10   MaxAbsScaler SGD                               0:00:22       0.9016    0.9154\n",
      "        11   MaxAbsScaler SGD                               0:00:30       0.9014    0.9154\n",
      "        12   MaxAbsScaler RandomForest                      0:00:30       0.8903    0.9154\n",
      "        13   StandardScalerWrapper ExtremeRandomTrees       0:00:30       0.8881    0.9154\n",
      "        14   MaxAbsScaler RandomForest                      0:00:28       0.7730    0.9154\n",
      "        15   MaxAbsScaler SGD                               0:00:24       0.8738    0.9154\n",
      "        16   MaxAbsScaler RandomForest                      0:00:23       0.8881    0.9154\n",
      "        17   MaxAbsScaler ExtremeRandomTrees                0:00:24       0.9006    0.9154\n",
      "        18   SparseNormalizer ExtremeRandomTrees            0:00:36       0.7282    0.9154\n",
      "        19   MaxAbsScaler SGD                               0:00:27       0.9075    0.9154\n",
      "        20   MaxAbsScaler ExtremeRandomTrees                0:00:25       0.7365    0.9154\n",
      "        21   MaxAbsScaler RandomForest                      0:00:24       0.7526    0.9154\n",
      "        22   MaxAbsScaler LightGBM                          0:00:30       0.9068    0.9154\n",
      "        23   MaxAbsScaler RandomForest                      0:00:30       0.8881    0.9154\n",
      "        24   MaxAbsScaler LightGBM                          0:00:24       0.9104    0.9154\n",
      "        25   SparseNormalizer XGBoostClassifier             0:00:22       0.8881    0.9154\n",
      "        26   StandardScalerWrapper LightGBM                 0:00:23       0.9074    0.9154\n",
      "        27   StandardScalerWrapper LightGBM                 0:00:26       0.8984    0.9154\n",
      "        28   MaxAbsScaler ExtremeRandomTrees                0:00:27       0.9006    0.9154\n",
      "        29   SparseNormalizer XGBoostClassifier             0:00:40       0.9129    0.9154\n",
      "        30   StandardScalerWrapper XGBoostClassifier        0:00:24       0.9018    0.9154\n",
      "        31   MaxAbsScaler ExtremeRandomTrees                0:00:30       0.8881    0.9154\n",
      "        32   StandardScalerWrapper XGBoostClassifier        0:00:42       0.9146    0.9154\n",
      "        33   MaxAbsScaler LightGBM                          0:00:32       0.9120    0.9154\n",
      "        34   StandardScalerWrapper LightGBM                 0:00:25       0.9041    0.9154\n",
      "        35   StandardScalerWrapper XGBoostClassifier        0:00:30       0.8881    0.9154\n",
      "        36   StandardScalerWrapper ExtremeRandomTrees       0:00:30       0.8997    0.9154\n",
      "        37   MaxAbsScaler LightGBM                          0:00:22       0.9050    0.9154\n",
      "        38   MaxAbsScaler LightGBM                          0:00:22       0.8932    0.9154\n",
      "        39   SparseNormalizer ExtremeRandomTrees            0:00:36       0.8881    0.9154\n",
      "        40   SparseNormalizer XGBoostClassifier             0:00:42       0.9115    0.9154\n",
      "        41   StandardScalerWrapper LightGBM                 0:00:24       0.9072    0.9154\n",
      "        42   StandardScalerWrapper ExtremeRandomTrees       0:00:27       0.9000    0.9154\n",
      "        43   MaxAbsScaler LightGBM                          0:01:53       0.8972    0.9154\n",
      "        44   MaxAbsScaler LightGBM                          0:00:24       0.9071    0.9154\n",
      "        45   SparseNormalizer XGBoostClassifier             0:00:24       0.9059    0.9154\n",
      "        46   MaxAbsScaler LightGBM                          0:00:24       0.9063    0.9154\n",
      "        47   MaxAbsScaler ExtremeRandomTrees                0:00:28       0.8881    0.9154\n",
      "        48   SparseNormalizer XGBoostClassifier             0:00:26       0.9137    0.9154\n",
      "        49   MaxAbsScaler ExtremeRandomTrees                0:00:30       0.8999    0.9154\n",
      "        50   VotingEnsemble                                 0:00:55       0.9178    0.9178\n",
      "        51   StackEnsemble                                  0:01:18       0.9159    0.9178\n",
      "Stopping criteria reached at iteration 52. Ending experiment.\n",
      "****************************************************************************************************\n",
      "Current status: BestRunExplainModel. Best run model explanations started\n",
      "Current status: ModelExplanationDataSetSetup. Model explanations data setup completed\n",
      "Current status: PickSurrogateModel. Choosing LightGBM as the surrogate model for explanations\n",
      "Current status: EngineeredFeatureExplanations. Computation of engineered features started\n",
      "Current status: EngineeredFeatureExplanations. Computation of engineered features completed\n",
      "Current status: BestRunExplainModel. Best run model explanations completed\n",
      "****************************************************************************************************\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "478ce49ad2e24c26b140d4d95d3b64a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "_AutoMLWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO', 's…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/aml.mini.widget.v1": "{\"status\": \"Completed\", \"workbench_run_details_uri\": \"https://ml.azure.com/experiments/quick-starts-ws-125463/runs/AutoML_eb130077-f46f-4f09-9157-1552c2b503ab?wsid=/subscriptions/94e14ad4-bf97-47e8-aae0-f9b85a7befa8/resourcegroups/aml-quickstarts-125527/workspaces/quick-starts-ws-125527\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"run_properties\": {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"created_utc\": \"2020-11-10T14:02:43.926969Z\", \"properties\": {\"num_iterations\": \"1000\", \"training_type\": \"TrainFull\", \"acquisition_function\": \"EI\", \"primary_metric\": \"accuracy\", \"train_split\": \"0\", \"acquisition_parameter\": \"0\", \"num_cross_validation\": \"2\", \"target\": \"local\", \"AMLSettingsJsonString\": \"{\\\"path\\\":null,\\\"name\\\":\\\"quick-starts-ws-125463\\\",\\\"subscription_id\\\":\\\"94e14ad4-bf97-47e8-aae0-f9b85a7befa8\\\",\\\"resource_group\\\":\\\"aml-quickstarts-125527\\\",\\\"workspace_name\\\":\\\"quick-starts-ws-125527\\\",\\\"region\\\":\\\"southcentralus\\\",\\\"compute_target\\\":\\\"local\\\",\\\"spark_service\\\":null,\\\"azure_service\\\":\\\"Microsoft.AzureNotebookVM\\\",\\\"_local_managed_run_id\\\":null,\\\"many_models\\\":false,\\\"pipeline_fetch_max_batch_size\\\":1,\\\"iterations\\\":1000,\\\"primary_metric\\\":\\\"accuracy\\\",\\\"task_type\\\":\\\"classification\\\",\\\"data_script\\\":null,\\\"validation_size\\\":0.0,\\\"n_cross_validations\\\":2,\\\"y_min\\\":null,\\\"y_max\\\":null,\\\"num_classes\\\":2,\\\"featurization\\\":\\\"auto\\\",\\\"_ignore_package_version_incompatibilities\\\":false,\\\"is_timeseries\\\":false,\\\"max_cores_per_iteration\\\":1,\\\"max_concurrent_iterations\\\":4,\\\"iteration_timeout_minutes\\\":null,\\\"mem_in_mb\\\":null,\\\"enforce_time_on_windows\\\":false,\\\"experiment_timeout_minutes\\\":30,\\\"experiment_exit_score\\\":null,\\\"whitelist_models\\\":null,\\\"blacklist_algos\\\":[\\\"TensorFlowLinearClassifier\\\",\\\"TensorFlowDNN\\\"],\\\"supported_models\\\":[\\\"XGBoostClassifier\\\",\\\"TensorFlowLinearClassifier\\\",\\\"RandomForest\\\",\\\"LinearSVM\\\",\\\"MultinomialNaiveBayes\\\",\\\"KNN\\\",\\\"AveragedPerceptronClassifier\\\",\\\"DecisionTree\\\",\\\"SGD\\\",\\\"BernoulliNaiveBayes\\\",\\\"LogisticRegression\\\",\\\"ExtremeRandomTrees\\\",\\\"LightGBM\\\",\\\"SVM\\\",\\\"TensorFlowDNN\\\",\\\"GradientBoosting\\\"],\\\"auto_blacklist\\\":true,\\\"blacklist_samples_reached\\\":false,\\\"exclude_nan_labels\\\":true,\\\"verbosity\\\":20,\\\"_debug_log\\\":\\\"automl.log\\\",\\\"show_warnings\\\":false,\\\"model_explainability\\\":true,\\\"service_url\\\":null,\\\"sdk_url\\\":null,\\\"sdk_packages\\\":null,\\\"enable_onnx_compatible_models\\\":false,\\\"enable_split_onnx_featurizer_estimator_models\\\":false,\\\"vm_type\\\":null,\\\"telemetry_verbosity\\\":20,\\\"send_telemetry\\\":true,\\\"enable_dnn\\\":false,\\\"force_text_dnn\\\":false,\\\"enable_feature_sweeping\\\":true,\\\"enable_early_stopping\\\":false,\\\"early_stopping_n_iters\\\":10,\\\"metrics\\\":null,\\\"enable_ensembling\\\":true,\\\"enable_stack_ensembling\\\":true,\\\"ensemble_iterations\\\":15,\\\"enable_tf\\\":false,\\\"enable_subsampling\\\":null,\\\"subsample_seed\\\":null,\\\"enable_nimbusml\\\":false,\\\"enable_streaming\\\":false,\\\"force_streaming\\\":false,\\\"track_child_runs\\\":true,\\\"allowed_private_models\\\":[],\\\"label_column_name\\\":\\\"y\\\",\\\"weight_column_name\\\":null,\\\"cv_split_column_names\\\":null,\\\"enable_local_managed\\\":false,\\\"cost_mode\\\":1,\\\"lag_length\\\":0,\\\"metric_operation\\\":\\\"maximize\\\",\\\"preprocess\\\":true,\\\"scenario\\\":\\\"SDK-1.13.0\\\"}\", \"DataPrepJsonString\": null, \"EnableSubsampling\": null, \"runTemplate\": \"AutoML\", \"azureml.runsource\": \"automl\", \"display_task_type\": \"classification\", \"dependencies_versions\": \"{\\\"azureml-widgets\\\": \\\"1.17.0\\\", \\\"azureml-train\\\": \\\"1.17.0\\\", \\\"azureml-train-restclients-hyperdrive\\\": \\\"1.17.0\\\", \\\"azureml-train-core\\\": \\\"1.17.0\\\", \\\"azureml-train-automl\\\": \\\"1.17.0\\\", \\\"azureml-train-automl-runtime\\\": \\\"1.17.0\\\", \\\"azureml-train-automl-client\\\": \\\"1.17.0\\\", \\\"azureml-tensorboard\\\": \\\"1.17.0\\\", \\\"azureml-telemetry\\\": \\\"1.17.0\\\", \\\"azureml-sdk\\\": \\\"1.17.0\\\", \\\"azureml-samples\\\": \\\"0+unknown\\\", \\\"azureml-pipeline\\\": \\\"1.17.0\\\", \\\"azureml-pipeline-steps\\\": \\\"1.17.0\\\", \\\"azureml-pipeline-core\\\": \\\"1.17.0\\\", \\\"azureml-opendatasets\\\": \\\"1.17.0\\\", \\\"azureml-model-management-sdk\\\": \\\"1.0.1b6.post1\\\", \\\"azureml-mlflow\\\": \\\"1.17.0.post1\\\", \\\"azureml-interpret\\\": \\\"1.17.0\\\", \\\"azureml-explain-model\\\": \\\"1.17.0\\\", \\\"azureml-defaults\\\": \\\"1.17.0\\\", \\\"azureml-dataset-runtime\\\": \\\"1.17.0\\\", \\\"azureml-dataprep\\\": \\\"2.4.0\\\", \\\"azureml-dataprep-rslex\\\": \\\"1.2.0\\\", \\\"azureml-dataprep-native\\\": \\\"24.0.0\\\", \\\"azureml-datadrift\\\": \\\"1.17.0\\\", \\\"azureml-core\\\": \\\"1.17.0\\\", \\\"azureml-contrib-services\\\": \\\"1.17.0\\\", \\\"azureml-contrib-server\\\": \\\"1.17.0\\\", \\\"azureml-contrib-reinforcementlearning\\\": \\\"1.17.0\\\", \\\"azureml-contrib-pipeline-steps\\\": \\\"1.17.0\\\", \\\"azureml-contrib-notebook\\\": \\\"1.17.0\\\", \\\"azureml-contrib-interpret\\\": \\\"1.17.0\\\", \\\"azureml-contrib-gbdt\\\": \\\"1.17.0\\\", \\\"azureml-contrib-fairness\\\": \\\"1.17.0\\\", \\\"azureml-contrib-dataset\\\": \\\"1.17.0\\\", \\\"azureml-cli-common\\\": \\\"1.17.0\\\", \\\"azureml-automl-runtime\\\": \\\"1.17.0\\\", \\\"azureml-automl-core\\\": \\\"1.17.0\\\", \\\"azureml-accel-models\\\": \\\"1.17.0\\\"}\", \"_aml_system_scenario_identification\": \"Local.Parent\", \"ClientSdkVersion\": \"1.17.0\", \"ClientType\": \"SDK\", \"environment_cpu_name\": \"AzureML-AutoML\", \"environment_cpu_version\": \"44\", \"environment_gpu_name\": \"AzureML-AutoML-GPU\", \"environment_gpu_version\": \"32\", \"root_attribution\": \"automl\", \"attribution\": \"AutoML\", \"Orchestrator\": \"AutoML\", \"_azureml.ComputeTargetType\": \"local\", \"ProblemInfoJsonString\": \"{\\\"dataset_num_categorical\\\": 0, \\\"is_sparse\\\": true, \\\"subsampling\\\": false, \\\"dataset_classes\\\": 2, \\\"dataset_features\\\": 121, \\\"dataset_samples\\\": 26360, \\\"single_frequency_class_detected\\\": false}\", \"feature_skus\": \"automatedml_sdk_guardrails\"}, \"tags\": {\"model_explain_run\": \"best_run\", \"best_score\": \"0.9177921092564492\", \"best_pipeline\": \"VotingEnsemble\", \"automl_best_child_run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_50\", \"model_explain_best_run_child_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_50\"}, \"end_time_utc\": \"2020-11-10T14:35:26.067633Z\", \"status\": \"Completed\", \"log_files\": {}, \"log_groups\": [], \"run_duration\": \"0:32:42\"}, \"child_runs\": [{\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_0\", \"run_number\": 10, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:06:43.96004Z\", \"end_time\": \"2020-11-10T14:07:12.345884Z\", \"created_time\": \"2020-11-10T14:06:43.821281Z\", \"created_time_dt\": \"2020-11-10T14:06:43.821281Z\", \"duration\": \"0:00:28\", \"iteration\": \"0\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, LightGBM\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.91483308, \"best_metric\": 0.91483308}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_1\", \"run_number\": 11, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:07:12.918354Z\", \"end_time\": \"2020-11-10T14:07:39.933533Z\", \"created_time\": \"2020-11-10T14:07:12.796203Z\", \"created_time_dt\": \"2020-11-10T14:07:12.796203Z\", \"duration\": \"0:00:27\", \"iteration\": \"1\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, XGBoostClassifier\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.91540212, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_2\", \"run_number\": 12, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:07:40.728976Z\", \"end_time\": \"2020-11-10T14:08:04.667509Z\", \"created_time\": \"2020-11-10T14:07:40.596054Z\", \"created_time_dt\": \"2020-11-10T14:07:40.596054Z\", \"duration\": \"0:00:24\", \"iteration\": \"2\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, RandomForest\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.89264036, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_3\", \"run_number\": 13, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:08:06.821875Z\", \"end_time\": \"2020-11-10T14:08:30.095704Z\", \"created_time\": \"2020-11-10T14:08:06.682138Z\", \"created_time_dt\": \"2020-11-10T14:08:06.682138Z\", \"duration\": \"0:00:23\", \"iteration\": \"3\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, RandomForest\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.88805008, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_4\", \"run_number\": 14, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:08:31.684847Z\", \"end_time\": \"2020-11-10T14:09:00.597993Z\", \"created_time\": \"2020-11-10T14:08:31.555182Z\", \"created_time_dt\": \"2020-11-10T14:08:31.555182Z\", \"duration\": \"0:00:29\", \"iteration\": \"4\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, SGD\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.84374052, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_5\", \"run_number\": 15, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:09:02.629487Z\", \"end_time\": \"2020-11-10T14:09:30.707197Z\", \"created_time\": \"2020-11-10T14:09:02.519239Z\", \"created_time_dt\": \"2020-11-10T14:09:02.519239Z\", \"duration\": \"0:00:28\", \"iteration\": \"5\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, SGD\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.90663885, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_6\", \"run_number\": 16, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:09:32.473164Z\", \"end_time\": \"2020-11-10T14:10:02.5832Z\", \"created_time\": \"2020-11-10T14:09:32.302788Z\", \"created_time_dt\": \"2020-11-10T14:09:32.302788Z\", \"duration\": \"0:00:30\", \"iteration\": \"6\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, ExtremeRandomTrees\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.90007587, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_7\", \"run_number\": 17, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:10:04.808122Z\", \"end_time\": \"2020-11-10T14:10:29.604179Z\", \"created_time\": \"2020-11-10T14:10:04.680318Z\", \"created_time_dt\": \"2020-11-10T14:10:04.680318Z\", \"duration\": \"0:00:24\", \"iteration\": \"7\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, ExtremeRandomTrees\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.89988619, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_8\", \"run_number\": 18, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:10:31.182913Z\", \"end_time\": \"2020-11-10T14:10:56.824159Z\", \"created_time\": \"2020-11-10T14:10:31.05422Z\", \"created_time_dt\": \"2020-11-10T14:10:31.05422Z\", \"duration\": \"0:00:25\", \"iteration\": \"8\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, ExtremeRandomTrees\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.89996206, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_9\", \"run_number\": 19, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:10:58.363755Z\", \"end_time\": \"2020-11-10T14:11:22.346812Z\", \"created_time\": \"2020-11-10T14:10:58.259883Z\", \"created_time_dt\": \"2020-11-10T14:10:58.259883Z\", \"duration\": \"0:00:24\", \"iteration\": \"9\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, ExtremeRandomTrees\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.75477997, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_10\", \"run_number\": 20, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:11:23.822779Z\", \"end_time\": \"2020-11-10T14:11:46.651994Z\", \"created_time\": \"2020-11-10T14:11:23.703094Z\", \"created_time_dt\": \"2020-11-10T14:11:23.703094Z\", \"duration\": \"0:00:22\", \"iteration\": \"10\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, SGD\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.90163126, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_11\", \"run_number\": 21, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:11:48.313784Z\", \"end_time\": \"2020-11-10T14:12:18.639763Z\", \"created_time\": \"2020-11-10T14:11:48.193762Z\", \"created_time_dt\": \"2020-11-10T14:11:48.193762Z\", \"duration\": \"0:00:30\", \"iteration\": \"11\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, SGD\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.90136571, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_12\", \"run_number\": 22, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:12:21.661737Z\", \"end_time\": \"2020-11-10T14:12:52.649251Z\", \"created_time\": \"2020-11-10T14:12:21.264549Z\", \"created_time_dt\": \"2020-11-10T14:12:21.264549Z\", \"duration\": \"0:00:31\", \"iteration\": \"12\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, RandomForest\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.89028832, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_13\", \"run_number\": 23, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:12:55.514461Z\", \"end_time\": \"2020-11-10T14:13:25.956316Z\", \"created_time\": \"2020-11-10T14:12:55.119821Z\", \"created_time_dt\": \"2020-11-10T14:12:55.119821Z\", \"duration\": \"0:00:30\", \"iteration\": \"13\", \"goal\": \"accuracy_max\", \"run_name\": \"StandardScalerWrapper, ExtremeRandomTrees\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7fc49585f048\", \"primary_metric\": 0.88805008, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_14\", \"run_number\": 24, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:13:27.555537Z\", \"end_time\": \"2020-11-10T14:13:56.316785Z\", \"created_time\": \"2020-11-10T14:13:27.437725Z\", \"created_time_dt\": \"2020-11-10T14:13:27.437725Z\", \"duration\": \"0:00:28\", \"iteration\": \"14\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, RandomForest\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.77298938, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_15\", \"run_number\": 25, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:13:57.824504Z\", \"end_time\": \"2020-11-10T14:14:22.13404Z\", \"created_time\": \"2020-11-10T14:13:57.692865Z\", \"created_time_dt\": \"2020-11-10T14:13:57.692865Z\", \"duration\": \"0:00:24\", \"iteration\": \"15\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, SGD\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.87378604, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_16\", \"run_number\": 26, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:14:23.737573Z\", \"end_time\": \"2020-11-10T14:14:47.625176Z\", \"created_time\": \"2020-11-10T14:14:23.623315Z\", \"created_time_dt\": \"2020-11-10T14:14:23.623315Z\", \"duration\": \"0:00:24\", \"iteration\": \"16\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, RandomForest\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.88805008, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_17\", \"run_number\": 27, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:14:49.28357Z\", \"end_time\": \"2020-11-10T14:15:13.594217Z\", \"created_time\": \"2020-11-10T14:14:49.168231Z\", \"created_time_dt\": \"2020-11-10T14:14:49.168231Z\", \"duration\": \"0:00:24\", \"iteration\": \"17\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, ExtremeRandomTrees\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.90064492, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_18\", \"run_number\": 28, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:15:16.18877Z\", \"end_time\": \"2020-11-10T14:15:52.197394Z\", \"created_time\": \"2020-11-10T14:15:15.437854Z\", \"created_time_dt\": \"2020-11-10T14:15:15.437854Z\", \"duration\": \"0:00:36\", \"iteration\": \"18\", \"goal\": \"accuracy_max\", \"run_name\": \"SparseNormalizer, ExtremeRandomTrees\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.SparseNormalizer object at 0x7fc4cd18df28\", \"primary_metric\": 0.72818665, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_19\", \"run_number\": 29, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:15:54.904225Z\", \"end_time\": \"2020-11-10T14:16:22.440448Z\", \"created_time\": \"2020-11-10T14:15:54.517616Z\", \"created_time_dt\": \"2020-11-10T14:15:54.517616Z\", \"duration\": \"0:00:27\", \"iteration\": \"19\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, SGD\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.90754932, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_20\", \"run_number\": 30, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:16:24.022145Z\", \"end_time\": \"2020-11-10T14:16:50.000781Z\", \"created_time\": \"2020-11-10T14:16:23.909288Z\", \"created_time_dt\": \"2020-11-10T14:16:23.909288Z\", \"duration\": \"0:00:26\", \"iteration\": \"20\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, ExtremeRandomTrees\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.73649469, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_21\", \"run_number\": 31, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:16:52.019367Z\", \"end_time\": \"2020-11-10T14:17:17.000924Z\", \"created_time\": \"2020-11-10T14:16:51.588487Z\", \"created_time_dt\": \"2020-11-10T14:16:51.588487Z\", \"duration\": \"0:00:25\", \"iteration\": \"21\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, RandomForest\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.7526176, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_22\", \"run_number\": 32, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:17:19.766951Z\", \"end_time\": \"2020-11-10T14:17:50.381797Z\", \"created_time\": \"2020-11-10T14:17:19.340506Z\", \"created_time_dt\": \"2020-11-10T14:17:19.340506Z\", \"duration\": \"0:00:31\", \"iteration\": \"22\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, LightGBM\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.90682853, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_23\", \"run_number\": 33, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:17:53.63533Z\", \"end_time\": \"2020-11-10T14:18:24.129177Z\", \"created_time\": \"2020-11-10T14:17:52.998055Z\", \"created_time_dt\": \"2020-11-10T14:17:52.998055Z\", \"duration\": \"0:00:31\", \"iteration\": \"23\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, RandomForest\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.88805008, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_24\", \"run_number\": 34, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:18:25.711418Z\", \"end_time\": \"2020-11-10T14:18:50.556846Z\", \"created_time\": \"2020-11-10T14:18:25.564603Z\", \"created_time_dt\": \"2020-11-10T14:18:25.564603Z\", \"duration\": \"0:00:24\", \"iteration\": \"24\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, LightGBM\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.9103566, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_25\", \"run_number\": 35, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:18:53.158458Z\", \"end_time\": \"2020-11-10T14:19:16.121275Z\", \"created_time\": \"2020-11-10T14:18:53.033358Z\", \"created_time_dt\": \"2020-11-10T14:18:53.033358Z\", \"duration\": \"0:00:23\", \"iteration\": \"25\", \"goal\": \"accuracy_max\", \"run_name\": \"SparseNormalizer, XGBoostClassifier\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.SparseNormalizer object at 0x7fc4cd011c18\", \"primary_metric\": 0.88805008, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_26\", \"run_number\": 36, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:19:17.64519Z\", \"end_time\": \"2020-11-10T14:19:41.32242Z\", \"created_time\": \"2020-11-10T14:19:17.503103Z\", \"created_time_dt\": \"2020-11-10T14:19:17.503103Z\", \"duration\": \"0:00:23\", \"iteration\": \"26\", \"goal\": \"accuracy_max\", \"run_name\": \"StandardScalerWrapper, LightGBM\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7fc4cd0b8ac8\", \"primary_metric\": 0.90739757, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_27\", \"run_number\": 37, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:19:42.682696Z\", \"end_time\": \"2020-11-10T14:20:08.827648Z\", \"created_time\": \"2020-11-10T14:19:42.555898Z\", \"created_time_dt\": \"2020-11-10T14:19:42.555898Z\", \"duration\": \"0:00:26\", \"iteration\": \"27\", \"goal\": \"accuracy_max\", \"run_name\": \"StandardScalerWrapper, LightGBM\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7fc4ccf86748\", \"primary_metric\": 0.89844461, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_28\", \"run_number\": 38, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:20:11.516462Z\", \"end_time\": \"2020-11-10T14:20:39.436334Z\", \"created_time\": \"2020-11-10T14:20:10.608149Z\", \"created_time_dt\": \"2020-11-10T14:20:10.608149Z\", \"duration\": \"0:00:28\", \"iteration\": \"28\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, ExtremeRandomTrees\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.90056904, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_29\", \"run_number\": 39, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:20:41.191894Z\", \"end_time\": \"2020-11-10T14:21:21.872104Z\", \"created_time\": \"2020-11-10T14:20:40.964896Z\", \"created_time_dt\": \"2020-11-10T14:20:40.964896Z\", \"duration\": \"0:00:40\", \"iteration\": \"29\", \"goal\": \"accuracy_max\", \"run_name\": \"SparseNormalizer, XGBoostClassifier\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.SparseNormalizer object at 0x7fc4cceec6d8\", \"primary_metric\": 0.91293627, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_30\", \"run_number\": 40, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:21:24.633778Z\", \"end_time\": \"2020-11-10T14:21:48.681874Z\", \"created_time\": \"2020-11-10T14:21:24.447902Z\", \"created_time_dt\": \"2020-11-10T14:21:24.447902Z\", \"duration\": \"0:00:24\", \"iteration\": \"30\", \"goal\": \"accuracy_max\", \"run_name\": \"StandardScalerWrapper, XGBoostClassifier\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7fc4ccf860b8\", \"primary_metric\": 0.901783, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_31\", \"run_number\": 41, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:21:51.892254Z\", \"end_time\": \"2020-11-10T14:22:22.784562Z\", \"created_time\": \"2020-11-10T14:21:51.532293Z\", \"created_time_dt\": \"2020-11-10T14:21:51.532293Z\", \"duration\": \"0:00:31\", \"iteration\": \"31\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, ExtremeRandomTrees\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.88805008, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_32\", \"run_number\": 42, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:22:24.513631Z\", \"end_time\": \"2020-11-10T14:23:06.91667Z\", \"created_time\": \"2020-11-10T14:22:24.319161Z\", \"created_time_dt\": \"2020-11-10T14:22:24.319161Z\", \"duration\": \"0:00:42\", \"iteration\": \"32\", \"goal\": \"accuracy_max\", \"run_name\": \"StandardScalerWrapper, XGBoostClassifier\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7fc4cce98978\", \"primary_metric\": 0.91456753, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_33\", \"run_number\": 43, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:23:10.852614Z\", \"end_time\": \"2020-11-10T14:23:43.245407Z\", \"created_time\": \"2020-11-10T14:23:09.408237Z\", \"created_time_dt\": \"2020-11-10T14:23:09.408237Z\", \"duration\": \"0:00:33\", \"iteration\": \"33\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, LightGBM\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.9120258, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_34\", \"run_number\": 44, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:23:46.003044Z\", \"end_time\": \"2020-11-10T14:24:11.871056Z\", \"created_time\": \"2020-11-10T14:23:45.560278Z\", \"created_time_dt\": \"2020-11-10T14:23:45.560278Z\", \"duration\": \"0:00:26\", \"iteration\": \"34\", \"goal\": \"accuracy_max\", \"run_name\": \"StandardScalerWrapper, LightGBM\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7fc4ccead2e8\", \"primary_metric\": 0.90413505, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_35\", \"run_number\": 45, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:24:15.20899Z\", \"end_time\": \"2020-11-10T14:24:45.778135Z\", \"created_time\": \"2020-11-10T14:24:15.02815Z\", \"created_time_dt\": \"2020-11-10T14:24:15.02815Z\", \"duration\": \"0:00:30\", \"iteration\": \"35\", \"goal\": \"accuracy_max\", \"run_name\": \"StandardScalerWrapper, XGBoostClassifier\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7fc4ccdbb7b8\", \"primary_metric\": 0.88805008, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_36\", \"run_number\": 46, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:24:47.570537Z\", \"end_time\": \"2020-11-10T14:25:18.220661Z\", \"created_time\": \"2020-11-10T14:24:47.399818Z\", \"created_time_dt\": \"2020-11-10T14:24:47.399818Z\", \"duration\": \"0:00:30\", \"iteration\": \"36\", \"goal\": \"accuracy_max\", \"run_name\": \"StandardScalerWrapper, ExtremeRandomTrees\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7fc4cce36198\", \"primary_metric\": 0.89969651, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_37\", \"run_number\": 47, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:25:19.853569Z\", \"end_time\": \"2020-11-10T14:25:42.585448Z\", \"created_time\": \"2020-11-10T14:25:19.742511Z\", \"created_time_dt\": \"2020-11-10T14:25:19.742511Z\", \"duration\": \"0:00:22\", \"iteration\": \"37\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, LightGBM\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.90496965, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_38\", \"run_number\": 48, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:25:44.265353Z\", \"end_time\": \"2020-11-10T14:26:06.655094Z\", \"created_time\": \"2020-11-10T14:25:44.144966Z\", \"created_time_dt\": \"2020-11-10T14:25:44.144966Z\", \"duration\": \"0:00:22\", \"iteration\": \"38\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, LightGBM\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.89324734, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_39\", \"run_number\": 49, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:26:08.108064Z\", \"end_time\": \"2020-11-10T14:26:44.267698Z\", \"created_time\": \"2020-11-10T14:26:07.998913Z\", \"created_time_dt\": \"2020-11-10T14:26:07.998913Z\", \"duration\": \"0:00:36\", \"iteration\": \"39\", \"goal\": \"accuracy_max\", \"run_name\": \"SparseNormalizer, ExtremeRandomTrees\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.SparseNormalizer object at 0x7fc4cccfa940\", \"primary_metric\": 0.88805008, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_40\", \"run_number\": 50, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:26:46.67913Z\", \"end_time\": \"2020-11-10T14:27:28.973162Z\", \"created_time\": \"2020-11-10T14:26:46.547377Z\", \"created_time_dt\": \"2020-11-10T14:26:46.547377Z\", \"duration\": \"0:00:42\", \"iteration\": \"40\", \"goal\": \"accuracy_max\", \"run_name\": \"SparseNormalizer, XGBoostClassifier\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.SparseNormalizer object at 0x7fc49567b160\", \"primary_metric\": 0.91145675, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_41\", \"run_number\": 51, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:27:30.620215Z\", \"end_time\": \"2020-11-10T14:27:54.81208Z\", \"created_time\": \"2020-11-10T14:27:30.490572Z\", \"created_time_dt\": \"2020-11-10T14:27:30.490572Z\", \"duration\": \"0:00:24\", \"iteration\": \"41\", \"goal\": \"accuracy_max\", \"run_name\": \"StandardScalerWrapper, LightGBM\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7fc495751550\", \"primary_metric\": 0.90720789, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_42\", \"run_number\": 52, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:27:56.421741Z\", \"end_time\": \"2020-11-10T14:28:24.249707Z\", \"created_time\": \"2020-11-10T14:27:56.310926Z\", \"created_time_dt\": \"2020-11-10T14:27:56.310926Z\", \"duration\": \"0:00:27\", \"iteration\": \"42\", \"goal\": \"accuracy_max\", \"run_name\": \"StandardScalerWrapper, ExtremeRandomTrees\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7fc4956136d8\", \"primary_metric\": 0.89996206, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_43\", \"run_number\": 53, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:28:25.730678Z\", \"end_time\": \"2020-11-10T14:30:19.437299Z\", \"created_time\": \"2020-11-10T14:28:25.623427Z\", \"created_time_dt\": \"2020-11-10T14:28:25.623427Z\", \"duration\": \"0:01:53\", \"iteration\": \"43\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, LightGBM\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.89723065, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_44\", \"run_number\": 54, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:30:21.311555Z\", \"end_time\": \"2020-11-10T14:30:45.749067Z\", \"created_time\": \"2020-11-10T14:30:21.184762Z\", \"created_time_dt\": \"2020-11-10T14:30:21.184762Z\", \"duration\": \"0:00:24\", \"iteration\": \"44\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, LightGBM\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.90709408, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_45\", \"run_number\": 55, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:30:48.491034Z\", \"end_time\": \"2020-11-10T14:31:12.846379Z\", \"created_time\": \"2020-11-10T14:30:48.296644Z\", \"created_time_dt\": \"2020-11-10T14:30:48.296644Z\", \"duration\": \"0:00:24\", \"iteration\": \"45\", \"goal\": \"accuracy_max\", \"run_name\": \"SparseNormalizer, XGBoostClassifier\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.SparseNormalizer object at 0x7fc4cce6f5c0\", \"primary_metric\": 0.90588012, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_46\", \"run_number\": 56, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:31:14.286381Z\", \"end_time\": \"2020-11-10T14:31:38.832283Z\", \"created_time\": \"2020-11-10T14:31:14.161622Z\", \"created_time_dt\": \"2020-11-10T14:31:14.161622Z\", \"duration\": \"0:00:24\", \"iteration\": \"46\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, LightGBM\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.90633536, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_47\", \"run_number\": 57, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:31:40.887383Z\", \"end_time\": \"2020-11-10T14:32:09.03648Z\", \"created_time\": \"2020-11-10T14:31:40.502337Z\", \"created_time_dt\": \"2020-11-10T14:31:40.502337Z\", \"duration\": \"0:00:28\", \"iteration\": \"47\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, ExtremeRandomTrees\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.88805008, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_48\", \"run_number\": 58, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:32:10.449949Z\", \"end_time\": \"2020-11-10T14:32:37.132906Z\", \"created_time\": \"2020-11-10T14:32:10.340701Z\", \"created_time_dt\": \"2020-11-10T14:32:10.340701Z\", \"duration\": \"0:00:26\", \"iteration\": \"48\", \"goal\": \"accuracy_max\", \"run_name\": \"SparseNormalizer, XGBoostClassifier\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.SparseNormalizer object at 0x7fc4954ac2e8\", \"primary_metric\": 0.91369499, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_49\", \"run_number\": 59, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:32:38.767293Z\", \"end_time\": \"2020-11-10T14:33:09.417812Z\", \"created_time\": \"2020-11-10T14:32:38.621543Z\", \"created_time_dt\": \"2020-11-10T14:32:38.621543Z\", \"duration\": \"0:00:30\", \"iteration\": \"49\", \"goal\": \"accuracy_max\", \"run_name\": \"MaxAbsScaler, ExtremeRandomTrees\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.89992413, \"best_metric\": 0.91540212}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_50\", \"run_number\": 60, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:33:09.90377Z\", \"end_time\": \"2020-11-10T14:34:05.214744Z\", \"created_time\": \"2020-11-10T14:33:09.797873Z\", \"created_time_dt\": \"2020-11-10T14:33:09.797873Z\", \"duration\": \"0:00:55\", \"iteration\": \"50\", \"goal\": \"accuracy_max\", \"run_name\": \"VotingEnsemble\", \"run_properties\": \"classification_labels=None,\\n                              estimators=[('1',\\n                                           Pipeline(memory=None,\\n                                                    steps=[('maxabsscaler',\\n                                                            MaxAbsScaler(copy=True\", \"primary_metric\": 0.91779211, \"best_metric\": 0.91779211}, {\"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_51\", \"run_number\": 61, \"metric\": null, \"status\": \"Completed\", \"run_type\": null, \"training_percent\": \"100\", \"start_time\": \"2020-11-10T14:34:05.633982Z\", \"end_time\": \"2020-11-10T14:35:24.067303Z\", \"created_time\": \"2020-11-10T14:34:05.531914Z\", \"created_time_dt\": \"2020-11-10T14:34:05.531914Z\", \"duration\": \"0:01:18\", \"iteration\": \"51\", \"goal\": \"accuracy_max\", \"run_name\": \"StackEnsemble\", \"run_properties\": \"base_learners=[('1',\\n                                        Pipeline(memory=None,\\n                                                 steps=[('maxabsscaler',\\n                                                         MaxAbsScaler(copy=True\", \"primary_metric\": 0.91593323, \"best_metric\": 0.91779211}], \"children_metrics\": {\"categories\": [0], \"series\": {\"f1_score_micro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"f1_score_micro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9148330804248862, 0.915402124430956, 0.892640364188164, 0.888050075872534, 0.8437405159332321, 0.9066388467374811, 0.9000758725341427, 0.8998861911987861, 0.8999620637329286, 0.7547799696509864, 0.9016312594840667, 0.9013657056145674, 0.8902883156297421, 0.888050075872534, 0.77298937784522, 0.8737860394537178, 0.888050075872534, 0.9006449165402124, 0.728186646433991, 0.9075493171471927, 0.73649468892261, 0.7526176024279211, 0.9068285280728376, 0.888050075872534, 0.9103566009104704, 0.888050075872534, 0.9073975720789074, 0.8984446130500758, 0.9005690440060699, 0.9129362670713201, 0.901783004552352, 0.888050075872534, 0.914567526555387, 0.9120257966616084, 0.9041350531107739, 0.888050075872534, 0.8996965098634295, 0.904969650986343, 0.8932473444613049, 0.888050075872534, 0.9114567526555386, 0.9072078907435508, 0.8999620637329286, 0.8972306525037936, 0.9070940819423368, 0.9058801213960546, 0.9063353566009105, 0.888050075872534, 0.9136949924127467, 0.8999241274658574, 0.9177921092564492, 0.9159332321699545]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"f1_score_micro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9148330804248862, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.9177921092564492, 0.9177921092564492]}], \"precision_score_weighted\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"precision_score_weighted\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9086981287213591, 0.9076551709448089, 0.8773218384365955, 0.7886329732362227, 0.9162853463726108, 0.8946444248584307, 0.8831765486892631, 0.883526497401607, 0.8836896599871029, 0.878058880543168, 0.8851101916175221, 0.8850690436816807, 0.877706173186591, 0.7886329732362227, 0.8858796302014591, 0.9187620712876152, 0.7886329732362227, 0.8851925370652617, 0.8826037116501069, 0.8954561887085528, 0.8737799001627704, 0.8919602966063103, 0.8947692550927965, 0.7886329732362227, 0.9034497228863666, 0.7886329732362227, 0.8959457852420512, 0.881910465072729, 0.8839116387161989, 0.9067359000177446, 0.8922414309344243, 0.7886329732362227, 0.9048895760968425, 0.9044848157249663, 0.8884114707485, 0.7886329732362227, 0.8835120266712657, 0.8901853764752852, 0.8830880933893526, 0.7886329732362227, 0.9056371638981058, 0.8959531123438642, 0.8830238319059602, 0.8787015344391071, 0.897174347958356, 0.8925246275338716, 0.893819519360162, 0.7886329732362227, 0.9033770284028093, 0.8843461680195612, 0.9136292425548103, 0.9094081950036011]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"precision_score_weighted_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9086981287213591, 0.9086981287213591, 0.9086981287213591, 0.9086981287213591, 0.9162853463726108, 0.9162853463726108, 0.9162853463726108, 0.9162853463726108, 0.9162853463726108, 0.9162853463726108, 0.9162853463726108, 0.9162853463726108, 0.9162853463726108, 0.9162853463726108, 0.9162853463726108, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152, 0.9187620712876152]}], \"log_loss\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"log_loss\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.1772159701065366, 0.17869375331906398, 0.22810442120182564, 0.2551379359156587, 0.4141539476386997, 0.21326455204259298, 0.2301978166183244, 0.24723859950142152, 0.24669383165279074, 0.5207866257262166, 0.24353930661787565, 0.26935837827401515, 0.2626600368365731, 0.24818304781985834, 0.6019264793123983, 0.7382765762339876, 0.2750102545448703, 0.2446802973386395, 0.503609652918948, 0.2810545867634827, 0.5819613015106231, 0.5182253950735196, 0.19363000353816656, 0.27461027910779107, 0.19001008855944201, 0.41019742028452094, 0.19396405844018655, 0.2278530320581581, 0.23361534988067195, 0.18191776919225433, 0.360166980952096, 0.2848992626582031, 0.20853447705659722, 0.18181752075948948, 0.2048989044110079, 0.3505741059190245, 0.25652106289030674, 0.20402956795995242, 0.23827629565511807, 0.2665988317381782, 0.18646587822556523, 0.19135864869654157, 0.22778092595720512, 0.22408838062209718, 0.1893599588971112, 0.3365355385094684, 0.19761559483270424, 0.2560161370225239, 0.18441350089909367, 0.23561265672229825, 0.1862627194925323, 0.19112948855534323]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"log_loss_min\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366, 0.1772159701065366]}], \"precision_score_micro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"precision_score_micro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9148330804248862, 0.915402124430956, 0.892640364188164, 0.8880500758725342, 0.8437405159332321, 0.9066388467374811, 0.9000758725341427, 0.8998861911987861, 0.8999620637329286, 0.7547799696509864, 0.9016312594840667, 0.9013657056145675, 0.8902883156297421, 0.8880500758725342, 0.77298937784522, 0.8737860394537178, 0.8880500758725342, 0.9006449165402124, 0.7281866464339909, 0.9075493171471927, 0.73649468892261, 0.7526176024279211, 0.9068285280728376, 0.8880500758725342, 0.9103566009104704, 0.8880500758725342, 0.9073975720789074, 0.8984446130500758, 0.9005690440060699, 0.9129362670713201, 0.901783004552352, 0.8880500758725342, 0.914567526555387, 0.9120257966616085, 0.9041350531107739, 0.8880500758725342, 0.8996965098634295, 0.904969650986343, 0.8932473444613049, 0.8880500758725342, 0.9114567526555387, 0.9072078907435508, 0.8999620637329286, 0.8972306525037936, 0.9070940819423369, 0.9058801213960546, 0.9063353566009105, 0.8880500758725342, 0.9136949924127467, 0.8999241274658574, 0.9177921092564492, 0.9159332321699545]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"precision_score_micro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9148330804248862, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.9177921092564492, 0.9177921092564492]}], \"balanced_accuracy\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"balanced_accuracy\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.745186989948512, 0.7320329539941353, 0.5302852647396163, 0.5, 0.8539722865563668, 0.686671757909148, 0.5891018208976706, 0.5857246411131214, 0.5850354501271965, 0.7350076648537951, 0.6024055265530548, 0.5979731615754194, 0.5134185161202154, 0.5, 0.7585145572196286, 0.8515898377208626, 0.5, 0.588675103579345, 0.7455741038503926, 0.6791265251043257, 0.7216303540538453, 0.7747342873541627, 0.6856003638553869, 0.5, 0.7314107935734074, 0.5, 0.6919833864464595, 0.5735173344340911, 0.5924836271531011, 0.7413145532566341, 0.5828516353535431, 0.5, 0.7106821940285101, 0.7296867517374098, 0.6440912950414573, 0.5, 0.5825184303360106, 0.6556741634263652, 0.5304746002514811, 0.5, 0.7413618943441206, 0.6932160070387305, 0.5884413999686305, 0.5698781731834563, 0.7044044602063044, 0.6202103197331259, 0.6819027852574531, 0.5, 0.7022024435520068, 0.5829340148272167, 0.765791667482807, 0.742647737856283]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"balanced_accuracy_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.745186989948512, 0.745186989948512, 0.745186989948512, 0.745186989948512, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668]}], \"average_precision_score_weighted\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"average_precision_score_weighted\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9537164221427894, 0.9546638994935461, 0.9370749258712259, 0.9275596945489988, 0.9421218499206684, 0.9442903463214709, 0.9368564737899836, 0.9325863797412535, 0.9315557597009021, 0.9125544180784265, 0.9386036706719034, 0.937341897491153, 0.9281600864623809, 0.9240394201292983, 0.9133975758401041, 0.9441687637553333, 0.9337700501291488, 0.9312812266818251, 0.918587772398656, 0.9446784441519265, 0.8992616170436223, 0.921730414185082, 0.9461893228625232, 0.930128631178317, 0.9487389970416066, 0.8011657947273769, 0.9465935462632706, 0.9411089010204785, 0.9377320328292362, 0.9529733637620943, 0.9226517261908507, 0.9174688263433919, 0.9539431530857376, 0.9507675599985215, 0.9445756230570107, 0.8011657947273769, 0.9289770733095195, 0.9452107201518212, 0.9461391028802346, 0.9422532771926374, 0.9518889173766207, 0.9472000646553563, 0.9367430070683278, 0.9411056195604657, 0.947412978317864, 0.9402399765734164, 0.9451904316528059, 0.9279823373224647, 0.9529634490360793, 0.9371001402832347, 0.9552482332641143, 0.9545605510922774]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"average_precision_score_weighted_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9537164221427894, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9546638994935461, 0.9552482332641143, 0.9552482332641143]}], \"accuracy\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"accuracy\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9148330804248862, 0.915402124430956, 0.892640364188164, 0.8880500758725342, 0.8437405159332321, 0.9066388467374811, 0.9000758725341427, 0.8998861911987861, 0.8999620637329286, 0.7547799696509864, 0.9016312594840667, 0.9013657056145675, 0.8902883156297421, 0.8880500758725342, 0.77298937784522, 0.8737860394537178, 0.8880500758725342, 0.9006449165402124, 0.7281866464339909, 0.9075493171471927, 0.73649468892261, 0.7526176024279211, 0.9068285280728376, 0.8880500758725342, 0.9103566009104704, 0.8880500758725342, 0.9073975720789074, 0.8984446130500758, 0.9005690440060699, 0.9129362670713201, 0.901783004552352, 0.8880500758725342, 0.914567526555387, 0.9120257966616085, 0.9041350531107739, 0.8880500758725342, 0.8996965098634295, 0.904969650986343, 0.8932473444613049, 0.8880500758725342, 0.9114567526555387, 0.9072078907435508, 0.8999620637329286, 0.8972306525037936, 0.9070940819423369, 0.9058801213960546, 0.9063353566009105, 0.8880500758725342, 0.9136949924127467, 0.8999241274658574, 0.9177921092564492, 0.9159332321699545]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"accuracy_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9148330804248862, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.9177921092564492, 0.9177921092564492]}], \"AUC_macro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"AUC_macro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9470045425173352, 0.945847504854574, 0.9148884570080293, 0.9023956233096497, 0.923047894541215, 0.9306764068616, 0.9134128374818961, 0.8980085552786405, 0.8985872620967681, 0.8381268652623393, 0.9141286543786914, 0.9106787548786519, 0.8895017345723936, 0.8953244045429432, 0.8437361591636614, 0.9323629419021225, 0.9038038030416702, 0.8976289714623461, 0.8634379764079372, 0.9276904360231448, 0.7934756173954942, 0.8735646566563322, 0.9344944106322695, 0.8945803222226768, 0.9366796898196788, 0.5, 0.9352384617506759, 0.9206052747757435, 0.9123834855136997, 0.9440415950760115, 0.8795311016689497, 0.8441068316067989, 0.943251936584553, 0.943075664350136, 0.9305319540179708, 0.5, 0.8854391271787336, 0.9322192531440308, 0.9271268412636664, 0.9225931473469982, 0.9417593181496331, 0.9367668923087973, 0.9159234494193433, 0.9220184756598566, 0.936606442553418, 0.9161628467073248, 0.9326753539374175, 0.892051707908956, 0.9434452809808499, 0.9150177242346215, 0.9469727974295878, 0.9458039011371836]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"AUC_macro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352]}], \"average_precision_score_micro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"average_precision_score_micro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9812391954572518, 0.9810964483932146, 0.9714175197595747, 0.9690880372176942, 0.8697693441889807, 0.9711299053543277, 0.9716041924650941, 0.9687626142941843, 0.9689397140456051, 0.7926005987061051, 0.9718299972369078, 0.9706751447768319, 0.9652086208073133, 0.9673390019586043, 0.8171970972158389, 0.9239102208505596, 0.9691567922050098, 0.9685307823222367, 0.8577376816988873, 0.9728161134027871, 0.7931318544300029, 0.8575425552016197, 0.9776467254048785, 0.96761896591171, 0.9785330474864069, 0.8446079352999556, 0.9778936055249865, 0.9734405163713661, 0.972088224319206, 0.9804797284157923, 0.9594425184176966, 0.9563182184414051, 0.980201998967928, 0.9800433497196666, 0.9761997181184286, 0.8446079352999556, 0.966280265978848, 0.9766527637727135, 0.973744425561589, 0.9732604851063532, 0.97983754706315, 0.9782054656263826, 0.9723165322940684, 0.9737540804540962, 0.978205909187292, 0.9707511830102705, 0.9771802333871767, 0.9674148265361604, 0.9802550693110944, 0.9721673806351797, 0.9813575506552559, 0.9810739537957073]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"average_precision_score_micro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9812391954572518, 0.9813575506552559, 0.9813575506552559]}], \"norm_macro_recall\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"norm_macro_recall\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.490373979897024, 0.4640659079882705, 0.06057052947923269, 0.0, 0.7079445731127335, 0.37334351581829606, 0.1782036417953412, 0.1714492822262429, 0.17007090025439298, 0.4700153297075903, 0.2048110531061097, 0.1959463231508386, 0.02683703224043066, 0.0, 0.5170291144392571, 0.7031796754417251, 0.0, 0.1773502071586901, 0.49114820770078516, 0.3582530502086515, 0.4432607081076907, 0.5494685747083254, 0.37120072771077395, 0.0, 0.4628215871468149, 0.0, 0.3839667728929189, 0.14703466886818228, 0.1849672543062022, 0.4826291065132683, 0.16570327070708613, 0.0, 0.42136438805702014, 0.45937350347481964, 0.2881825900829147, 0.0, 0.16503686067202117, 0.3113483268527303, 0.06094920050296215, 0.0, 0.4827237886882412, 0.3864320140774612, 0.17688279993726108, 0.1397563463669128, 0.40880892041260897, 0.2404206394662517, 0.36380557051490603, 0.0, 0.40440488710401357, 0.16586802965443348, 0.531583334965614, 0.4852954757125659]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"norm_macro_recall_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.490373979897024, 0.490373979897024, 0.490373979897024, 0.490373979897024, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335, 0.7079445731127335]}], \"recall_score_weighted\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"recall_score_weighted\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9148330804248862, 0.915402124430956, 0.892640364188164, 0.8880500758725342, 0.8437405159332321, 0.9066388467374811, 0.9000758725341427, 0.8998861911987861, 0.8999620637329286, 0.7547799696509864, 0.9016312594840667, 0.9013657056145675, 0.8902883156297421, 0.8880500758725342, 0.77298937784522, 0.8737860394537178, 0.8880500758725342, 0.9006449165402124, 0.7281866464339909, 0.9075493171471927, 0.73649468892261, 0.7526176024279211, 0.9068285280728376, 0.8880500758725342, 0.9103566009104704, 0.8880500758725342, 0.9073975720789074, 0.8984446130500758, 0.9005690440060699, 0.9129362670713201, 0.901783004552352, 0.8880500758725342, 0.914567526555387, 0.9120257966616085, 0.9041350531107739, 0.8880500758725342, 0.8996965098634295, 0.904969650986343, 0.8932473444613049, 0.8880500758725342, 0.9114567526555387, 0.9072078907435508, 0.8999620637329286, 0.8972306525037936, 0.9070940819423369, 0.9058801213960546, 0.9063353566009105, 0.8880500758725342, 0.9136949924127467, 0.8999241274658574, 0.9177921092564492, 0.9159332321699545]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"recall_score_weighted_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9148330804248862, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.9177921092564492, 0.9177921092564492]}], \"recall_score_micro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"recall_score_micro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9148330804248862, 0.915402124430956, 0.892640364188164, 0.8880500758725342, 0.8437405159332321, 0.9066388467374811, 0.9000758725341427, 0.8998861911987861, 0.8999620637329286, 0.7547799696509864, 0.9016312594840667, 0.9013657056145675, 0.8902883156297421, 0.8880500758725342, 0.77298937784522, 0.8737860394537178, 0.8880500758725342, 0.9006449165402124, 0.7281866464339909, 0.9075493171471927, 0.73649468892261, 0.7526176024279211, 0.9068285280728376, 0.8880500758725342, 0.9103566009104704, 0.8880500758725342, 0.9073975720789074, 0.8984446130500758, 0.9005690440060699, 0.9129362670713201, 0.901783004552352, 0.8880500758725342, 0.914567526555387, 0.9120257966616085, 0.9041350531107739, 0.8880500758725342, 0.8996965098634295, 0.904969650986343, 0.8932473444613049, 0.8880500758725342, 0.9114567526555387, 0.9072078907435508, 0.8999620637329286, 0.8972306525037936, 0.9070940819423369, 0.9058801213960546, 0.9063353566009105, 0.8880500758725342, 0.9136949924127467, 0.8999241274658574, 0.9177921092564492, 0.9159332321699545]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"recall_score_micro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9148330804248862, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.915402124430956, 0.9177921092564492, 0.9177921092564492]}], \"AUC_weighted\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"AUC_weighted\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9470045425173352, 0.945847504854574, 0.9148884570080293, 0.9023956233096497, 0.923047894541215, 0.9306764068616, 0.913412837481896, 0.8980085552786405, 0.8985872620967681, 0.8381268652623393, 0.9141286543786913, 0.9106787548786519, 0.8895017345723936, 0.8953244045429434, 0.8437361591636614, 0.9323629419021227, 0.9038038030416702, 0.8976289714623462, 0.8634379764079371, 0.9276904360231448, 0.7934756173954942, 0.8735646566563322, 0.9344944106322692, 0.8945803222226769, 0.9366796898196786, 0.5, 0.9352384617506758, 0.9206052747757436, 0.9123834855136996, 0.9440416006823706, 0.8795311016689498, 0.8441068316067989, 0.943251942190912, 0.9430756643501359, 0.9305319540179706, 0.5, 0.8854391271787335, 0.9322192531440308, 0.9271268412636664, 0.9225931473469982, 0.941759318149633, 0.9367668923087973, 0.9159234494193433, 0.9220184756598566, 0.936606442553418, 0.916162846707325, 0.9326753539374175, 0.892051707908956, 0.9434452697681316, 0.9150177242346214, 0.9469727974295878, 0.9458039011371835]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"AUC_weighted_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352, 0.9470045425173352]}], \"matthews_correlation\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"matthews_correlation\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.5372778581687214, 0.5286009628391766, 0.19532446780624244, 0.0, 0.523823475429154, 0.45633571271883744, 0.32860045192596476, 0.3240654435008961, 0.3237917310031342, 0.32760068940148945, 0.3514637595116914, 0.34565228892567323, 0.12463591158770063, 0.0, 0.36539119355783656, 0.5579609022283558, 0.0, 0.3325937556257864, 0.3287361341275603, 0.45202663572972596, 0.30263824553989166, 0.3743242960979441, 0.45616967748782833, 0.0, 0.5105058357177655, 0.0, 0.46411294362208505, 0.3012109145735789, 0.3351766014422696, 0.5276007351151015, 0.3342555283640353, 0.0, 0.5082128247840798, 0.5144766511106492, 0.40511874689069194, 0.0, 0.31951081641588863, 0.4202598219340813, 0.20440058416244472, 0.0, 0.5227810554877872, 0.4647066221858003, 0.32721981873182626, 0.28808656044282954, 0.4746945214558837, 0.3933916804478367, 0.45058782740071757, 0.0, 0.49792256366792276, 0.32174139937563034, 0.5639792569659967, 0.5385257153445588]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"matthews_correlation_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.5372778581687214, 0.5372778581687214, 0.5372778581687214, 0.5372778581687214, 0.5372778581687214, 0.5372778581687214, 0.5372778581687214, 0.5372778581687214, 0.5372778581687214, 0.5372778581687214, 0.5372778581687214, 0.5372778581687214, 0.5372778581687214, 0.5372778581687214, 0.5372778581687214, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5579609022283558, 0.5639792569659967, 0.5639792569659967]}], \"average_precision_score_macro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"average_precision_score_macro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.8169601188326454, 0.8218637458359002, 0.7606914241086925, 0.7236582890609391, 0.7778344015255038, 0.7832306581059462, 0.7591491771835758, 0.7477502747502685, 0.7427058261789012, 0.6987404905894969, 0.7683563553636577, 0.7666173568662159, 0.7394815564324271, 0.7109264420339021, 0.6957164517287107, 0.7829044572653845, 0.7517629161487338, 0.7425679855363114, 0.702842551567221, 0.78990510938062, 0.6840458503542014, 0.7134618839321815, 0.789549044173065, 0.7391991150688368, 0.8003736843100285, 0.5, 0.791010466824617, 0.7748749951306837, 0.7626886631321715, 0.815201611955916, 0.7283705125531265, 0.7163821124757972, 0.8211250006941924, 0.8056333878748645, 0.7848396823131332, 0.5, 0.7382998135181484, 0.786591651101229, 0.7957972173986039, 0.7782641187160726, 0.8114623313426077, 0.7928011089803882, 0.7563533739756696, 0.7739072222770744, 0.7937066941515102, 0.7831459289508156, 0.7861183256689077, 0.729257338787394, 0.815524091705748, 0.7585734258763017, 0.8241572826007662, 0.8218137541812073]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"average_precision_score_macro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.8169601188326454, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8218637458359002, 0.8241572826007662, 0.8241572826007662]}], \"f1_score_weighted\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"f1_score_weighted\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9109600232365221, 0.9099503595753192, 0.8503208232607722, 0.8353941004300992, 0.8659584677985018, 0.8974454350122908, 0.8735210072367556, 0.872452823888549, 0.8723356723235829, 0.7953860723305608, 0.877890468061655, 0.8766228436023837, 0.8422541172435443, 0.8353941004300992, 0.8094981525176332, 0.8881071784036576, 0.8353941004300992, 0.8737288263121462, 0.7762692449838425, 0.8964738838302984, 0.7817037721724116, 0.7951545252194893, 0.8974256592915841, 0.8353941004300992, 0.906004792846641, 0.8353941004300992, 0.8987648589651883, 0.8681473309527883, 0.8747132534296629, 0.909067675167178, 0.872251620968032, 0.8353941004300992, 0.9066882412240894, 0.9070741607178714, 0.8885798132630215, 0.8353941004300992, 0.8714770764393318, 0.8912814604671475, 0.8506561617532656, 0.8353941004300992, 0.9079395112019881, 0.8988018143563168, 0.8732729253397759, 0.866398293945861, 0.9002376856905274, 0.8848247528589375, 0.8965134156748054, 0.8353941004300992, 0.9048895764445426, 0.8716988519909439, 0.9153446680856965, 0.9113483641215134]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"f1_score_weighted_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9109600232365221, 0.9153446680856965, 0.9153446680856965]}], \"f1_score_macro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"f1_score_macro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.7666078500137502, 0.7603252170036062, 0.5296160307229362, 0.4703530256574846, 0.7296819548691473, 0.720126198029852, 0.6215416234638458, 0.6169607685219229, 0.6162590047517995, 0.6203133334845045, 0.6386187410054681, 0.6332738200316198, 0.4973339500638291, 0.4703530256574846, 0.6409124072763985, 0.7597552300730582, 0.4703530256574846, 0.6213091572752809, 0.6061985865620265, 0.7135053691738658, 0.6035220088574975, 0.631993132878745, 0.7196047696810337, 0.4703530256574846, 0.7529875522688252, 0.4703530256574846, 0.7247390635212596, 0.599976982352234, 0.6260489768608046, 0.7618631247654462, 0.612183325133884, 0.4703530256574846, 0.7464661191931949, 0.7542597958559485, 0.683447126012699, 0.4703530256574846, 0.6128267222976, 0.6944224230599564, 0.5299159075437316, 0.4703530256574846, 0.7598585215306174, 0.7253593301316134, 0.6206222778928183, 0.5943988931264147, 0.7325657087593431, 0.6620024022384878, 0.7163322337118958, 0.4703530256574846, 0.7398117597869989, 0.6133760089570424, 0.7810667351818813, 0.7659301505692095]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"f1_score_macro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7666078500137502, 0.7810667351818813, 0.7810667351818813]}], \"AUC_micro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"AUC_micro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9803985869747929, 0.9802600346895214, 0.9706231967320698, 0.9680575019399882, 0.8981751532993615, 0.9738621796716871, 0.970729167865967, 0.9675904368369788, 0.9677517695915777, 0.8225356767853073, 0.9713534813864757, 0.9706493146142705, 0.9655306788692115, 0.9666549509419016, 0.8409430990533779, 0.9341958921758033, 0.9683406481057195, 0.9675152867613366, 0.8550254472334733, 0.9736108619764623, 0.8173328904557188, 0.8640594022073267, 0.9766661648794215, 0.9665093510883507, 0.9776877874003237, 0.888050075872534, 0.9769301788473361, 0.9724849319910381, 0.9708853110083102, 0.9796213079780143, 0.9640169239501613, 0.9564683235048275, 0.9795968062728971, 0.9791339967670701, 0.9752387710952126, 0.888050075872534, 0.9650582082798925, 0.9756935457687534, 0.9731726779435436, 0.9720759629134132, 0.9789462410282741, 0.977205199513679, 0.9711405720029198, 0.9727510125932288, 0.9771849735539893, 0.9730534521427371, 0.9762093984079432, 0.9660030832571538, 0.9793929592821238, 0.9709527932375581, 0.9805878308054002, 0.9803232239034174]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"AUC_micro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9803985869747929, 0.9805878308054002, 0.9805878308054002]}], \"precision_score_macro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"precision_score_macro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.7943360374183552, 0.801080693446058, 0.8190372016167253, 0.4440250379362671, 0.6937953499937036, 0.7788956577339636, 0.8029787258738209, 0.8070177302971979, 0.8082649283925956, 0.61429885433761, 0.8017739697444568, 0.8048994574862441, 0.8325647278410132, 0.4440250379362671, 0.6294183092267782, 0.7219657774725365, 0.4440250379362671, 0.8123556515903276, 0.6100236839076809, 0.7883865453346797, 0.6033211938466583, 0.62764615752432, 0.7803642778526241, 0.4440250379362671, 0.7815810890128919, 0.4440250379362671, 0.7804948965670562, 0.8086986108132102, 0.8037710186714695, 0.7884150551253062, 0.8480451327558153, 0.4440250379362671, 0.8064842025596757, 0.7880963422211399, 0.7847623534888132, 0.4440250379362671, 0.8093216994295334, 0.7836740564000964, 0.8446745296183902, 0.4440250379362671, 0.7830839019316966, 0.7794738945662536, 0.8027740756315547, 0.7969446670414044, 0.7756281813348297, 0.8218595296126469, 0.7790370233007364, 0.4440250379362671, 0.806558865180211, 0.8127538125854059, 0.7991967953605927, 0.7996916253482804]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"precision_score_macro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.7943360374183552, 0.801080693446058, 0.8190372016167253, 0.8190372016167253, 0.8190372016167253, 0.8190372016167253, 0.8190372016167253, 0.8190372016167253, 0.8190372016167253, 0.8190372016167253, 0.8190372016167253, 0.8190372016167253, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8325647278410132, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153, 0.8480451327558153]}], \"recall_score_macro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"recall_score_macro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.745186989948512, 0.7320329539941353, 0.5302852647396163, 0.5, 0.8539722865563668, 0.686671757909148, 0.5891018208976706, 0.5857246411131214, 0.5850354501271965, 0.7350076648537951, 0.6024055265530548, 0.5979731615754194, 0.5134185161202154, 0.5, 0.7585145572196286, 0.8515898377208626, 0.5, 0.588675103579345, 0.7455741038503926, 0.6791265251043257, 0.7216303540538453, 0.7747342873541627, 0.6856003638553869, 0.5, 0.7314107935734074, 0.5, 0.6919833864464595, 0.5735173344340911, 0.5924836271531011, 0.7413145532566341, 0.5828516353535431, 0.5, 0.7106821940285101, 0.7296867517374098, 0.6440912950414573, 0.5, 0.5825184303360106, 0.6556741634263652, 0.5304746002514811, 0.5, 0.7413618943441206, 0.6932160070387305, 0.5884413999686305, 0.5698781731834563, 0.7044044602063044, 0.6202103197331259, 0.6819027852574531, 0.5, 0.7022024435520068, 0.5829340148272167, 0.765791667482807, 0.742647737856283]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"recall_score_macro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.745186989948512, 0.745186989948512, 0.745186989948512, 0.745186989948512, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668, 0.8539722865563668]}], \"weighted_accuracy\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"markers\", \"name\": \"weighted_accuracy\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9569359408997471, 0.9609110724767057, 0.9825724327129988, 0.9843567475377293, 0.8412007871077982, 0.9612300271079458, 0.9772539047348807, 0.9778524841861588, 0.9781202154236222, 0.7596762327492094, 0.9758913959757021, 0.976662543214351, 0.9838237548836356, 0.9843567475377293, 0.7765988608202222, 0.8792799881504065, 0.9843567475377293, 0.978068477121661, 0.7238816816309654, 0.9642246987398535, 0.740185610378769, 0.7471447051539285, 0.961734169326925, 0.9843567475377293, 0.9547673493979063, 0.9843567475377293, 0.960858336094375, 0.9790842047166786, 0.9770290431179094, 0.9555314065196427, 0.9809458002588147, 0.9843567475377293, 0.9651673728207657, 0.9572786775702418, 0.9686711316632105, 0.9843567475377293, 0.9784137549782213, 0.9668399638272897, 0.9832821064253963, 0.9843567475377293, 0.9536711683008637, 0.9603172827137103, 0.9772748718212999, 0.9784735045869593, 0.9573981647072143, 0.9767787494709828, 0.9620331305393046, 0.9843567475377293, 0.9661845584413118, 0.9785927309398821, 0.9555124394883072, 0.9589281697776313]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\", \"27\", \"28\", \"29\", \"30\", \"31\", \"32\", \"33\", \"34\", \"35\", \"36\", \"37\", \"38\", \"39\", \"40\", \"41\", \"42\", \"43\", \"44\", \"45\", \"46\", \"47\", \"48\", \"49\", \"50\", \"51\"], \"mode\": \"lines\", \"name\": \"weighted_accuracy_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9569359408997471, 0.9609110724767057, 0.9825724327129988, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293, 0.9843567475377293]}]}, \"metricName\": null, \"primaryMetricName\": \"accuracy\", \"showLegend\": false}, \"run_metrics\": [{\"name\": \"experiment_status\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], \"series\": [{\"data\": [\"DatasetEvaluation\", \"FeaturesGeneration\", \"DatasetFeaturization\", \"DatasetFeaturizationCompleted\", \"DatasetBalancing\", \"DatasetCrossValidationSplit\", \"ModelSelection\", \"BestRunExplainModel\", \"ModelExplanationDataSetSetup\", \"PickSurrogateModel\", \"EngineeredFeatureExplanations\", \"EngineeredFeatureExplanations\", \"BestRunExplainModel\"]}]}, {\"name\": \"experiment_status_description\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], \"series\": [{\"data\": [\"Gathering dataset statistics.\", \"Generating features for the dataset.\", \"Beginning to fit featurizers and featurize the dataset.\", \"Completed fit featurizers and featurizing the dataset.\", \"Performing class balancing sweeping\", \"Generating individually featurized CV splits.\", \"Beginning model selection.\", \"Best run model explanations started\", \"Model explanations data setup completed\", \"Choosing LightGBM as the surrogate model for explanations\", \"Computation of engineered features started\", \"Computation of engineered features completed\", \"Best run model explanations completed\"]}]}, {\"name\": \"f1_score_micro\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.9177921092564492]}]}, {\"name\": \"precision_score_weighted\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.9136292425548103]}]}, {\"name\": \"log_loss\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.1862627194925323]}]}, {\"name\": \"precision_score_micro\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.9177921092564492]}]}, {\"name\": \"balanced_accuracy\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.765791667482807]}]}, {\"name\": \"average_precision_score_weighted\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.9552482332641143]}]}, {\"name\": \"accuracy\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.9177921092564492]}]}, {\"name\": \"AUC_macro\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.9469727974295878]}]}, {\"name\": \"average_precision_score_micro\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.9813575506552559]}]}, {\"name\": \"norm_macro_recall\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.531583334965614]}]}, {\"name\": \"recall_score_weighted\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.9177921092564492]}]}, {\"name\": \"recall_score_micro\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.9177921092564492]}]}, {\"name\": \"AUC_weighted\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.9469727974295878]}]}, {\"name\": \"matthews_correlation\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.5639792569659967]}]}, {\"name\": \"average_precision_score_macro\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.8241572826007662]}]}, {\"name\": \"f1_score_weighted\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.9153446680856965]}]}, {\"name\": \"f1_score_macro\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.7810667351818813]}]}, {\"name\": \"AUC_micro\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.9805878308054002]}]}, {\"name\": \"precision_score_macro\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.7991967953605927]}]}, {\"name\": \"recall_score_macro\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.765791667482807]}]}, {\"name\": \"weighted_accuracy\", \"run_id\": \"AutoML_eb130077-f46f-4f09-9157-1552c2b503ab\", \"categories\": [0], \"series\": [{\"data\": [0.9555124394883072]}]}], \"run_logs\": \"\\nRun is completed.\", \"graph\": {}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": false, \"log_level\": \"INFO\", \"sdk_version\": \"1.17.0\"}, \"loading\": false}"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "****************************************************************************************************\n",
      "DATA GUARDRAILS: \n",
      "\n",
      "TYPE:         Class balancing detection\n",
      "STATUS:       ALERTED\n",
      "DESCRIPTION:  To decrease model bias, please cancel the current run and fix balancing problem.\n",
      "              Learn more about imbalanced data: https://aka.ms/AutomatedMLImbalancedData\n",
      "DETAILS:      Imbalanced data can lead to a falsely perceived positive effect of a model's accuracy because the input data has bias towards one class.\n",
      "+---------------------------------+---------------------------------+--------------------------------------+\n",
      "|Size of the smallest class       |Name/Label of the smallest class |Number of samples in the training data|\n",
      "+=================================+=================================+======================================+\n",
      "|2951                             |1                                |26360                                 |\n",
      "+---------------------------------+---------------------------------+--------------------------------------+\n",
      "\n",
      "****************************************************************************************************\n",
      "\n",
      "TYPE:         Missing feature values imputation\n",
      "STATUS:       PASSED\n",
      "DESCRIPTION:  No feature missing values were detected in the training data.\n",
      "              Learn more about missing value imputation: https://aka.ms/AutomatedMLFeaturization\n",
      "\n",
      "****************************************************************************************************\n",
      "\n",
      "TYPE:         High cardinality feature detection\n",
      "STATUS:       PASSED\n",
      "DESCRIPTION:  Your inputs were analyzed, and no high cardinality features were detected.\n",
      "              Learn more about high cardinality feature handling: https://aka.ms/AutomatedMLFeaturization\n",
      "\n",
      "****************************************************************************************************\n",
      "\n",
      "****************************************************************************************************\n",
      "ITERATION: The iteration being evaluated.\n",
      "PIPELINE: A summary description of the pipeline being evaluated.\n",
      "DURATION: Time taken for the current iteration.\n",
      "METRIC: The result of computing score on the fitted pipeline.\n",
      "BEST: The best observed score thus far.\n",
      "****************************************************************************************************\n",
      "\n",
      " ITERATION   PIPELINE                                       DURATION      METRIC      BEST\n",
      "         0   MaxAbsScaler LightGBM                          0:00:28       0.9148    0.9148\n",
      "         1   MaxAbsScaler XGBoostClassifier                 0:00:27       0.9154    0.9154\n",
      "         2   MaxAbsScaler RandomForest                      0:00:24       0.8926    0.9154\n",
      "         3   MaxAbsScaler RandomForest                      0:00:23       0.8881    0.9154\n",
      "         4   MaxAbsScaler SGD                               0:00:29       0.8437    0.9154\n",
      "         5   MaxAbsScaler SGD                               0:00:28       0.9066    0.9154\n",
      "         6   MaxAbsScaler ExtremeRandomTrees                0:00:30       0.9001    0.9154\n",
      "         7   MaxAbsScaler ExtremeRandomTrees                0:00:25       0.8999    0.9154\n",
      "        10   MaxAbsScaler SGD                               0:00:23       0.9016    0.9154\n",
      "        11   MaxAbsScaler SGD                               0:00:30       0.9014    0.9154\n",
      "         8   MaxAbsScaler ExtremeRandomTrees                0:00:26       0.9000    0.9154\n",
      "         9   MaxAbsScaler ExtremeRandomTrees                0:00:24       0.7548    0.9154\n",
      "        12   MaxAbsScaler RandomForest                      0:00:31       0.8903    0.9154\n",
      "        13   StandardScalerWrapper ExtremeRandomTrees       0:00:30       0.8881    0.9154\n",
      "        14   MaxAbsScaler RandomForest                      0:00:29       0.7730    0.9154\n",
      "        15   MaxAbsScaler SGD                               0:00:24       0.8738    0.9154\n",
      "        16   MaxAbsScaler RandomForest                      0:00:24       0.8881    0.9154\n",
      "        17   MaxAbsScaler ExtremeRandomTrees                0:00:24       0.9006    0.9154\n",
      "        18   SparseNormalizer ExtremeRandomTrees            0:00:36       0.7282    0.9154\n",
      "        19   MaxAbsScaler SGD                               0:00:28       0.9075    0.9154\n",
      "        20   MaxAbsScaler ExtremeRandomTrees                0:00:26       0.7365    0.9154\n",
      "        21   MaxAbsScaler RandomForest                      0:00:25       0.7526    0.9154\n",
      "        22   MaxAbsScaler LightGBM                          0:00:31       0.9068    0.9154\n",
      "        23   MaxAbsScaler RandomForest                      0:00:30       0.8881    0.9154\n",
      "        24   MaxAbsScaler LightGBM                          0:00:25       0.9104    0.9154\n",
      "        25   SparseNormalizer XGBoostClassifier             0:00:23       0.8881    0.9154\n",
      "        26   StandardScalerWrapper LightGBM                 0:00:24       0.9074    0.9154\n",
      "        27   StandardScalerWrapper LightGBM                 0:00:26       0.8984    0.9154\n",
      "        28   MaxAbsScaler ExtremeRandomTrees                0:00:28       0.9006    0.9154\n",
      "        29   SparseNormalizer XGBoostClassifier             0:00:41       0.9129    0.9154\n",
      "        30   StandardScalerWrapper XGBoostClassifier        0:00:24       0.9018    0.9154\n",
      "        31   MaxAbsScaler ExtremeRandomTrees                0:00:31       0.8881    0.9154\n",
      "        32   StandardScalerWrapper XGBoostClassifier        0:00:42       0.9146    0.9154\n",
      "        33   MaxAbsScaler LightGBM                          0:00:32       0.9120    0.9154\n",
      "        34   StandardScalerWrapper LightGBM                 0:00:26       0.9041    0.9154\n",
      "        35   StandardScalerWrapper XGBoostClassifier        0:00:31       0.8881    0.9154\n",
      "        36   StandardScalerWrapper ExtremeRandomTrees       0:00:31       0.8997    0.9154\n",
      "        37   MaxAbsScaler LightGBM                          0:00:23       0.9050    0.9154\n",
      "        38   MaxAbsScaler LightGBM                          0:00:22       0.8932    0.9154\n",
      "        39   SparseNormalizer ExtremeRandomTrees            0:00:36       0.8881    0.9154\n",
      "        40   SparseNormalizer XGBoostClassifier             0:00:42       0.9115    0.9154\n",
      "        41   StandardScalerWrapper LightGBM                 0:00:24       0.9072    0.9154\n",
      "        42   StandardScalerWrapper ExtremeRandomTrees       0:00:28       0.9000    0.9154\n",
      "        43   MaxAbsScaler LightGBM                          0:01:54       0.8972    0.9154\n",
      "        44   MaxAbsScaler LightGBM                          0:00:24       0.9071    0.9154\n",
      "        45   SparseNormalizer XGBoostClassifier             0:00:24       0.9059    0.9154\n",
      "        46   MaxAbsScaler LightGBM                          0:00:25       0.9063    0.9154\n",
      "        47   MaxAbsScaler ExtremeRandomTrees                0:00:28       0.8881    0.9154\n",
      "        48   SparseNormalizer XGBoostClassifier             0:00:27       0.9137    0.9154\n",
      "        49   MaxAbsScaler ExtremeRandomTrees                0:00:31       0.8999    0.9154\n",
      "        50    VotingEnsemble                                0:00:55       0.9178    0.9178\n",
      "        51    StackEnsemble                                 0:01:18       0.9159    0.9178\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'runId': 'AutoML_eb130077-f46f-4f09-9157-1552c2b503ab',\n",
       " 'target': 'local',\n",
       " 'status': 'Completed',\n",
       " 'startTimeUtc': '2020-11-10T14:02:46.353986Z',\n",
       " 'endTimeUtc': '2020-11-10T14:35:26.067633Z',\n",
       " 'properties': {'num_iterations': '1000',\n",
       "  'training_type': 'TrainFull',\n",
       "  'acquisition_function': 'EI',\n",
       "  'primary_metric': 'accuracy',\n",
       "  'train_split': '0',\n",
       "  'acquisition_parameter': '0',\n",
       "  'num_cross_validation': '2',\n",
       "  'target': 'local',\n",
       "  'AMLSettingsJsonString': '{\"path\":null,\"name\":\"quick-starts-ws-125463\",\"subscription_id\":\"94e14ad4-bf97-47e8-aae0-f9b85a7befa8\",\"resource_group\":\"aml-quickstarts-125527\",\"workspace_name\":\"quick-starts-ws-125527\",\"region\":\"southcentralus\",\"compute_target\":\"local\",\"spark_service\":null,\"azure_service\":\"Microsoft.AzureNotebookVM\",\"_local_managed_run_id\":null,\"many_models\":false,\"pipeline_fetch_max_batch_size\":1,\"iterations\":1000,\"primary_metric\":\"accuracy\",\"task_type\":\"classification\",\"data_script\":null,\"validation_size\":0.0,\"n_cross_validations\":2,\"y_min\":null,\"y_max\":null,\"num_classes\":2,\"featurization\":\"auto\",\"_ignore_package_version_incompatibilities\":false,\"is_timeseries\":false,\"max_cores_per_iteration\":1,\"max_concurrent_iterations\":4,\"iteration_timeout_minutes\":null,\"mem_in_mb\":null,\"enforce_time_on_windows\":false,\"experiment_timeout_minutes\":30,\"experiment_exit_score\":null,\"whitelist_models\":null,\"blacklist_algos\":[\"TensorFlowLinearClassifier\",\"TensorFlowDNN\"],\"supported_models\":[\"XGBoostClassifier\",\"TensorFlowLinearClassifier\",\"RandomForest\",\"LinearSVM\",\"MultinomialNaiveBayes\",\"KNN\",\"AveragedPerceptronClassifier\",\"DecisionTree\",\"SGD\",\"BernoulliNaiveBayes\",\"LogisticRegression\",\"ExtremeRandomTrees\",\"LightGBM\",\"SVM\",\"TensorFlowDNN\",\"GradientBoosting\"],\"auto_blacklist\":true,\"blacklist_samples_reached\":false,\"exclude_nan_labels\":true,\"verbosity\":20,\"_debug_log\":\"automl.log\",\"show_warnings\":false,\"model_explainability\":true,\"service_url\":null,\"sdk_url\":null,\"sdk_packages\":null,\"enable_onnx_compatible_models\":false,\"enable_split_onnx_featurizer_estimator_models\":false,\"vm_type\":null,\"telemetry_verbosity\":20,\"send_telemetry\":true,\"enable_dnn\":false,\"force_text_dnn\":false,\"enable_feature_sweeping\":true,\"enable_early_stopping\":false,\"early_stopping_n_iters\":10,\"metrics\":null,\"enable_ensembling\":true,\"enable_stack_ensembling\":true,\"ensemble_iterations\":15,\"enable_tf\":false,\"enable_subsampling\":null,\"subsample_seed\":null,\"enable_nimbusml\":false,\"enable_streaming\":false,\"force_streaming\":false,\"track_child_runs\":true,\"allowed_private_models\":[],\"label_column_name\":\"y\",\"weight_column_name\":null,\"cv_split_column_names\":null,\"enable_local_managed\":false,\"cost_mode\":1,\"lag_length\":0,\"metric_operation\":\"maximize\",\"preprocess\":true,\"scenario\":\"SDK-1.13.0\"}',\n",
       "  'DataPrepJsonString': None,\n",
       "  'EnableSubsampling': None,\n",
       "  'runTemplate': 'AutoML',\n",
       "  'azureml.runsource': 'automl',\n",
       "  'display_task_type': 'classification',\n",
       "  'dependencies_versions': '{\"azureml-widgets\": \"1.17.0\", \"azureml-train\": \"1.17.0\", \"azureml-train-restclients-hyperdrive\": \"1.17.0\", \"azureml-train-core\": \"1.17.0\", \"azureml-train-automl\": \"1.17.0\", \"azureml-train-automl-runtime\": \"1.17.0\", \"azureml-train-automl-client\": \"1.17.0\", \"azureml-tensorboard\": \"1.17.0\", \"azureml-telemetry\": \"1.17.0\", \"azureml-sdk\": \"1.17.0\", \"azureml-samples\": \"0+unknown\", \"azureml-pipeline\": \"1.17.0\", \"azureml-pipeline-steps\": \"1.17.0\", \"azureml-pipeline-core\": \"1.17.0\", \"azureml-opendatasets\": \"1.17.0\", \"azureml-model-management-sdk\": \"1.0.1b6.post1\", \"azureml-mlflow\": \"1.17.0.post1\", \"azureml-interpret\": \"1.17.0\", \"azureml-explain-model\": \"1.17.0\", \"azureml-defaults\": \"1.17.0\", \"azureml-dataset-runtime\": \"1.17.0\", \"azureml-dataprep\": \"2.4.0\", \"azureml-dataprep-rslex\": \"1.2.0\", \"azureml-dataprep-native\": \"24.0.0\", \"azureml-datadrift\": \"1.17.0\", \"azureml-core\": \"1.17.0\", \"azureml-contrib-services\": \"1.17.0\", \"azureml-contrib-server\": \"1.17.0\", \"azureml-contrib-reinforcementlearning\": \"1.17.0\", \"azureml-contrib-pipeline-steps\": \"1.17.0\", \"azureml-contrib-notebook\": \"1.17.0\", \"azureml-contrib-interpret\": \"1.17.0\", \"azureml-contrib-gbdt\": \"1.17.0\", \"azureml-contrib-fairness\": \"1.17.0\", \"azureml-contrib-dataset\": \"1.17.0\", \"azureml-cli-common\": \"1.17.0\", \"azureml-automl-runtime\": \"1.17.0\", \"azureml-automl-core\": \"1.17.0\", \"azureml-accel-models\": \"1.17.0\"}',\n",
       "  '_aml_system_scenario_identification': 'Local.Parent',\n",
       "  'ClientSdkVersion': '1.17.0',\n",
       "  'ClientType': 'SDK',\n",
       "  'environment_cpu_name': 'AzureML-AutoML',\n",
       "  'environment_cpu_version': '44',\n",
       "  'environment_gpu_name': 'AzureML-AutoML-GPU',\n",
       "  'environment_gpu_version': '32',\n",
       "  'root_attribution': 'automl',\n",
       "  'attribution': 'AutoML',\n",
       "  'Orchestrator': 'AutoML',\n",
       "  '_azureml.ComputeTargetType': 'local',\n",
       "  'ProblemInfoJsonString': '{\"dataset_num_categorical\": 0, \"is_sparse\": true, \"subsampling\": false, \"dataset_classes\": 2, \"dataset_features\": 121, \"dataset_samples\": 26360, \"single_frequency_class_detected\": false}',\n",
       "  'feature_skus': 'automatedml_sdk_guardrails'},\n",
       " 'inputDatasets': [],\n",
       " 'outputDatasets': [],\n",
       " 'logFiles': {}}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Submit your automl run\n",
    "### YOUR CODE HERE ###\n",
    "automl_run = exp.submit(automl_config, show_output=True)\n",
    "RunDetails(automl_run).show()\n",
    "automl_run.wait_for_completion(show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run(Experiment: quick-starts-ws-125463,\n",
      "Id: AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_50,\n",
      "Type: None,\n",
      "Status: Completed)\n",
      "Pipeline(memory=None,\n",
      "         steps=[('datatransformer',\n",
      "                 DataTransformer(allow_chargram=None, enable_dnn=None,\n",
      "                                 enable_feature_sweeping=None,\n",
      "                                 feature_sweeping_config=None,\n",
      "                                 feature_sweeping_timeout=None,\n",
      "                                 featurization_config=None, force_text_dnn=None,\n",
      "                                 is_cross_validation=None,\n",
      "                                 is_onnx_compatible=None, logger=None,\n",
      "                                 observer=None, task=None, working_dir=None)),\n",
      "                ('prefittedso...\n",
      "                                                                                                  l1_ratio=0.836734693877551,\n",
      "                                                                                                  learning_rate='constant',\n",
      "                                                                                                  loss='modified_huber',\n",
      "                                                                                                  max_iter=1000,\n",
      "                                                                                                  n_jobs=1,\n",
      "                                                                                                  penalty='l2',\n",
      "                                                                                                  power_t=0.2222222222222222,\n",
      "                                                                                                  random_state=None,\n",
      "                                                                                                  tol=0.0001))],\n",
      "                                                                     verbose=False))],\n",
      "                                               flatten_transform=None,\n",
      "                                               weights=[0.14285714285714285,\n",
      "                                                        0.14285714285714285,\n",
      "                                                        0.14285714285714285,\n",
      "                                                        0.14285714285714285,\n",
      "                                                        0.2857142857142857,\n",
      "                                                        0.14285714285714285]))],\n",
      "         verbose=False)\n",
      "f1_score_micro 0.9177921092564492\n",
      "precision_score_weighted 0.9136292425548103\n",
      "log_loss 0.1862627194925323\n",
      "precision_score_micro 0.9177921092564492\n",
      "balanced_accuracy 0.765791667482807\n",
      "average_precision_score_weighted 0.9552482332641143\n",
      "accuracy 0.9177921092564492\n",
      "AUC_macro 0.9469727974295878\n",
      "average_precision_score_micro 0.9813575506552559\n",
      "norm_macro_recall 0.531583334965614\n",
      "recall_score_weighted 0.9177921092564492\n",
      "recall_score_micro 0.9177921092564492\n",
      "AUC_weighted 0.9469727974295878\n",
      "matthews_correlation 0.5639792569659967\n",
      "average_precision_score_macro 0.8241572826007662\n",
      "f1_score_weighted 0.9153446680856965\n",
      "f1_score_macro 0.7810667351818813\n",
      "AUC_micro 0.9805878308054002\n",
      "precision_score_macro 0.7991967953605927\n",
      "recall_score_macro 0.765791667482807\n",
      "weighted_accuracy 0.9555124394883072\n",
      "accuracy_table aml://artifactId/ExperimentRun/dcid.AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_50/accuracy_table\n",
      "confusion_matrix aml://artifactId/ExperimentRun/dcid.AutoML_eb130077-f46f-4f09-9157-1552c2b503ab_50/confusion_matrix\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Model(workspace=Workspace.create(name='quick-starts-ws-125527', subscription_id='94e14ad4-bf97-47e8-aae0-f9b85a7befa8', resource_group='aml-quickstarts-125527'), name=automl_model, id=automl_model:1, version=1, tags={'Training context': 'Auto ML'}, properties={'Accuracy': '0.9177921092564492'})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Retrieve and save your best automl model.\n",
    "### YOUR CODE HERE ###\n",
    "#Getting the best run and the model it produced\n",
    "best_run, fitted_model = automl_run.get_output()\n",
    "print(best_run) \n",
    "print(fitted_model) \n",
    "#Printing the metric details of the best run\n",
    "best_run_metrics = best_run.get_metrics()\n",
    "for metric_name in best_run_metrics:\n",
    "    metric = best_run_metrics[metric_name]\n",
    "    print(metric_name, metric)\n",
    "\n",
    "# Register model\n",
    "from azureml.core import Model\n",
    "best_run.register_model(model_path='outputs/model.pkl', model_name='automl_model', #try changing pkl to joblib\n",
    "                        tags={'Training context':'Auto ML'},\n",
    "                        properties={'Accuracy': best_run_metrics['accuracy']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "outputs": [],
   "source": [
    "#To fix ValidationException Error (if it arises when submitting th automl run) \n",
    "#Run this block then restart the whole notebook\n",
    "!pip install -r /anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/automl/core/validated_linux_requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cleaning up the compute cluster once the training process is complete\n",
    "train_cluster.delete()"
   ]
  }
 ],
 "metadata": {
  "kernel_info": {
   "name": "python3"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "nteract": {
   "version": "nteract-front-end@1.0.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}