diff --git a/tutorials/automl/java_plugin/java_plugin.ipynb b/tutorials/automl/java_plugin/java_plugin.ipynb index aa6ab676f..8d2e02cf0 100644 --- a/tutorials/automl/java_plugin/java_plugin.ipynb +++ b/tutorials/automl/java_plugin/java_plugin.ipynb @@ -6,7 +6,9 @@ "source": [ "# H2OAutoML Plugin\n", "\n", - "Since H2O-3 `3.28.0.1`, users have the possibility to customize the `H2OAutoML` model selection engine by writing their own training steps as a Java plugin." + "Since H2O-3 `3.28.0.1`, users have the possibility to customize the `H2OAutoML` model selection engine by writing their own training steps as a Java plugin.\n", + "\n", + "This tutorial was updated for H2O-3 `3.46.0.1`." ] }, { @@ -59,60 +61,65 @@ "\n", "public class MyDRFStepsProvider implements ModelingStepsProvider {\n", "\n", - " public static class DRFSteps extends ModelingSteps {\n", - "\n", - " static abstract class DRFGridStep extends ModelingStep.GridStep {\n", - "\n", - " DRFGridStep(String id, int weight, AutoML autoML) {\n", - " super(Algo.DRF, id, weight, autoML);\n", - " }\n", - "\n", - " DRFParameters prepareModelParameters() {\n", - " DRFParameters drfParameters = new DRFParameters();\n", - " drfParameters._sample_rate = 0.8;\n", - " drfParameters._col_sample_rate_per_tree = 0.8;\n", - " drfParameters._col_sample_rate_change_per_level = 0.9;\n", - " return drfParameters;\n", - " }\n", - " }\n", - "\n", - " private ModelingStep[] grids = new ModelingStep[] {\n", - " new DRFGridStep(\"grid_1\", 10*DEFAULT_MODEL_TRAINING_WEIGHT, aml()) {\n", - " @Override\n", - " protected Job startJob() {\n", - " DRFParameters drfParameters = prepareModelParameters();\n", - "\n", - " Map searchParams = new HashMap<>();\n", - " searchParams.put(\"_ntrees\", IntStream.rangeClosed(5, 1000).filter(i -> i % 50 == 0).boxed().toArray());\n", - " searchParams.put(\"_nbins\", IntStream.of(5, 10, 15, 20, 30).boxed().toArray());\n", - " searchParams.put(\"_max_depth\", IntStream.rangeClosed(3, 20).boxed().toArray());\n", - " searchParams.put(\"_min_rows\", IntStream.of(3, 5, 10, 20, 50, 80, 100).boxed().toArray());\n", - "\n", - " return hyperparameterSearch(makeKey(\"MyDRF\", false), drfParameters, searchParams);\n", - " }\n", - " },\n", - " };\n", - "\n", - " public DRFSteps(AutoML autoML) {\n", - " super(autoML);\n", - " }\n", - "\n", - " @Override\n", - " protected ModelingStep[] getGrids() {\n", - " return grids;\n", - " }\n", + " public static class DRFSteps extends ModelingSteps {\n", + "\n", + " static final String NAME = Algo.DRF.name();\n", + " static abstract class DRFGridStep extends ModelingStep.GridStep {\n", + "\n", + " DRFGridStep(String id, AutoML autoML) {\n", + " super(NAME, Algo.DRF, id, autoML);\n", + " }\n", + "\n", + " public DRFParameters prepareModelParameters() {\n", + " DRFParameters drfParameters = new DRFParameters();\n", + " drfParameters._sample_rate = 0.8;\n", + " drfParameters._col_sample_rate_per_tree = 0.8;\n", + " drfParameters._col_sample_rate_change_per_level = 0.9;\n", + " return drfParameters;\n", + " }\n", + " }\n", + "\n", + " private ModelingStep[] grids = new ModelingStep[]{\n", + " new DRFGridStep(\"grid_1\", aml()) {\n", + " @Override\n", + " public Map prepareSearchParameters() {\n", + " Map searchParams = new HashMap<>();\n", + " searchParams.put(\"_ntrees\", IntStream.rangeClosed(5, 1000).filter(i -> i % 50 == 0).boxed().toArray());\n", + " searchParams.put(\"_nbins\", IntStream.of(5, 10, 15, 20, 30).boxed().toArray());\n", + " searchParams.put(\"_max_depth\", IntStream.rangeClosed(3, 20).boxed().toArray());\n", + " searchParams.put(\"_min_rows\", IntStream.of(3, 5, 10, 20, 50, 80, 100).boxed().toArray());\n", + " return searchParams;\n", + " }\n", + " },\n", + " };\n", + "\n", + " public DRFSteps(AutoML autoML) {\n", + " super(autoML);\n", " }\n", "\n", " @Override\n", - " public String getName() {\n", - " return \"MyDRF\";\n", + " protected ModelingStep[] getGrids() {\n", + " return grids;\n", " }\n", "\n", " @Override\n", - " public DRFSteps newInstance(AutoML aml) {\n", - " return new DRFSteps(aml);\n", + " public String getProvider() {\n", + " return NAME;\n", " }\n", + " }\n", + "\n", + " @Override\n", + " public String getName() {\n", + " return \"MyDRF\";\n", + " }\n", + "\n", + " @Override\n", + " public DRFSteps newInstance(AutoML aml) {\n", + " return new DRFSteps(aml);\n", + " }\n", "}\n", + "\n", + "\n", "```\n", "\n", "As shown above, writing a `ModelingStepsProvider` simply requires to implement 2 methods:\n", @@ -162,19 +169,18 @@ "output_type": "stream", "text": [ "Collecting h2o\n", - " Downloading h2o-3.28.0.2.tar.gz (126.2 MB)\n", - "\u001b[K |████████████████████████████████| 126.2 MB 1.6 MB/s eta 0:00:01\n", - "\u001b[?25hRequirement already satisfied: requests in /Users/seb/.pyenv/versions/3.7.5/lib/python3.7/site-packages (from h2o) (2.22.0)\n", - "Requirement already satisfied: tabulate in /Users/seb/.pyenv/versions/3.7.5/lib/python3.7/site-packages (from h2o) (0.8.6)\n", - "Requirement already satisfied: future in /Users/seb/.pyenv/versions/3.7.5/lib/python3.7/site-packages (from h2o) (0.18.2)\n", - "Requirement already satisfied: colorama>=0.3.8 in /Users/seb/.pyenv/versions/3.7.5/lib/python3.7/site-packages (from h2o) (0.4.3)\n", - "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /Users/seb/.pyenv/versions/3.7.5/lib/python3.7/site-packages (from requests->h2o) (3.0.4)\n", - "Requirement already satisfied: idna<2.9,>=2.5 in /Users/seb/.pyenv/versions/3.7.5/lib/python3.7/site-packages (from requests->h2o) (2.8)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /Users/seb/.pyenv/versions/3.7.5/lib/python3.7/site-packages (from requests->h2o) (2019.11.28)\n", - "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /Users/seb/.pyenv/versions/3.7.5/lib/python3.7/site-packages (from requests->h2o) (1.24.3)\n", - "Installing collected packages: h2o\n", - " Running setup.py install for h2o ... \u001b[?25ldone\n", - "\u001b[?25hSuccessfully installed h2o-3.28.0.2\n" + " Using cached h2o-3.46.0.1-py2.py3-none-any.whl.metadata (2.1 kB)\n", + "Requirement already satisfied: requests in /Users/tomasfryda/sources/h2o-tutorials/tutorials/automl/venv/lib/python3.10/site-packages (from h2o) (2.31.0)\n", + "Collecting tabulate (from h2o)\n", + " Using cached tabulate-0.9.0-py3-none-any.whl.metadata (34 kB)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/tomasfryda/sources/h2o-tutorials/tutorials/automl/venv/lib/python3.10/site-packages (from requests->h2o) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /Users/tomasfryda/sources/h2o-tutorials/tutorials/automl/venv/lib/python3.10/site-packages (from requests->h2o) (3.6)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/tomasfryda/sources/h2o-tutorials/tutorials/automl/venv/lib/python3.10/site-packages (from requests->h2o) (2.2.1)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /Users/tomasfryda/sources/h2o-tutorials/tutorials/automl/venv/lib/python3.10/site-packages (from requests->h2o) (2024.2.2)\n", + "Using cached h2o-3.46.0.1-py2.py3-none-any.whl (265.6 MB)\n", + "Using cached tabulate-0.9.0-py3-none-any.whl (35 kB)\n", + "Installing collected packages: tabulate, h2o\n", + "Successfully installed h2o-3.46.0.1 tabulate-0.9.0\n" ] } ], @@ -195,7 +201,7 @@ "rm -Rf ./build ./dist\n", "sources = ./src/my/automl/MyGLMStepsProvider.java ./src/my/automl/MyDRFStepsProvider.java\n", "mkdir -p build\n", - "javac ./src/my/automl/MyGLMStepsProvider.java ./src/my/automl/MyDRFStepsProvider.java -cp \"/Users/seb/.pyenv/versions/3.7.5/lib/python3.7/site-packages/h2o/backend/bin/h2o.jar\" -d ./build\n", + "javac ./src/my/automl/MyGLMStepsProvider.java ./src/my/automl/MyDRFStepsProvider.java -cp \"/Users/tomasfryda/sources/h2o-tutorials/tutorials/automl/venv/lib/python3.10/site-packages/h2o/backend/bin/h2o.jar\" -d ./build\n", "cp -R ./src/META-INF ./build\n", "mkdir -p dist\n", "jar cvf ./dist/h2oautoml_plugin.jar -C ./build .\n", @@ -205,14 +211,14 @@ "adding: META-INF/services/ai.h2o.automl.ModelingStepsProvider(in = 59) (out= 40)(deflated 32%)\n", "adding: my/(in = 0) (out= 0)(stored 0%)\n", "adding: my/automl/(in = 0) (out= 0)(stored 0%)\n", - "adding: my/automl/MyGLMStepsProvider$GLMSteps$1.class(in = 2770) (out= 1282)(deflated 53%)\n", - "adding: my/automl/MyDRFStepsProvider$DRFSteps.class(in = 899) (out= 452)(deflated 49%)\n", - "adding: my/automl/MyDRFStepsProvider$DRFSteps$DRFGridStep.class(in = 1109) (out= 571)(deflated 48%)\n", - "adding: my/automl/MyDRFStepsProvider.class(in = 827) (out= 411)(deflated 50%)\n", - "adding: my/automl/MyGLMStepsProvider.class(in = 827) (out= 415)(deflated 49%)\n", - "adding: my/automl/MyDRFStepsProvider$DRFSteps$1.class(in = 2500) (out= 1191)(deflated 52%)\n", - "adding: my/automl/MyGLMStepsProvider$GLMSteps.class(in = 900) (out= 452)(deflated 49%)\n", - "adding: my/automl/MyGLMStepsProvider$GLMSteps$GLMGridStep.class(in = 1510) (out= 779)(deflated 48%)\n" + "adding: my/automl/MyGLMStepsProvider$GLMSteps$1.class(in = 2720) (out= 1267)(deflated 53%)\n", + "adding: my/automl/MyDRFStepsProvider$DRFSteps.class(in = 1104) (out= 545)(deflated 50%)\n", + "adding: my/automl/MyDRFStepsProvider$DRFSteps$DRFGridStep.class(in = 1308) (out= 644)(deflated 50%)\n", + "adding: my/automl/MyDRFStepsProvider.class(in = 827) (out= 415)(deflated 49%)\n", + "adding: my/automl/MyGLMStepsProvider.class(in = 827) (out= 413)(deflated 50%)\n", + "adding: my/automl/MyDRFStepsProvider$DRFSteps$1.class(in = 2141) (out= 1006)(deflated 53%)\n", + "adding: my/automl/MyGLMStepsProvider$GLMSteps.class(in = 1105) (out= 544)(deflated 50%)\n", + "adding: my/automl/MyGLMStepsProvider$GLMSteps$GLMGridStep.class(in = 1709) (out= 869)(deflated 49%)\n" ] } ], @@ -230,13 +236,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Checking whether there is an H2O instance running at http://localhost:54321 ..... not found.\n", + "Checking whether there is an H2O instance running at http://localhost:54321..... not found.\n", "Attempting to start a local H2O server...\n", - " Java Version: java version \"1.8.0_202\"; Java(TM) SE Runtime Environment (build 1.8.0_202-b08); Java HotSpot(TM) 64-Bit Server VM (build 25.202-b08, mixed mode)\n", - " Starting server from /Users/seb/.pyenv/versions/3.7.5/envs/ve37-h2o/lib/python3.7/site-packages/h2o/backend/bin/h2o.jar\n", - " Ice root: /var/folders/8j/1spy0dnn4pj3f018plmmbf200000gn/T/tmpaufn47qd\n", - " JVM stdout: /var/folders/8j/1spy0dnn4pj3f018plmmbf200000gn/T/tmpaufn47qd/h2o_seb_started_from_python.out\n", - " JVM stderr: /var/folders/8j/1spy0dnn4pj3f018plmmbf200000gn/T/tmpaufn47qd/h2o_seb_started_from_python.err\n", + " Java Version: java version \"1.8.0_241\"; Java(TM) SE Runtime Environment (build 1.8.0_241-b07); Java HotSpot(TM) 64-Bit Server VM (build 25.241-b07, mixed mode)\n", + " Starting server from /Users/tomasfryda/sources/h2o-tutorials/tutorials/automl/venv/lib/python3.10/site-packages/h2o/backend/bin/h2o.jar\n", + " Ice root: /var/folders/49/kh67vvnj633ftt08t8zfwsvh0000gn/T/tmpaxfdhqe1\n", + " JVM stdout: /var/folders/49/kh67vvnj633ftt08t8zfwsvh0000gn/T/tmpaxfdhqe1/h2o_tomasfryda_started_from_python.out\n", + " JVM stderr: /var/folders/49/kh67vvnj633ftt08t8zfwsvh0000gn/T/tmpaxfdhqe1/h2o_tomasfryda_started_from_python.err\n", " Server is running at http://127.0.0.1:54321\n", "Connecting to H2O server at http://127.0.0.1:54321 ... successful.\n" ] @@ -244,58 +250,99 @@ { "data": { "text/html": [ - "
\n", - "\n", - "\n", + "\n", + " \n", + "
\n", + "
H2O cluster uptime:01 secs
H2O cluster timezone:
\n", + " \n", + " \n", + " \n", + "\n", + "\n", "\n", - "\n", + "\n", "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", "\n", - "\n", - "\n", - "\n", + "\n", + "\n", + "\n", "\n", - "\n", - "\n", - "\n", - "
H2O_cluster_uptime:03 secs
H2O_cluster_timezone:Europe/Prague
H2O data parsing timezone:
H2O_data_parsing_timezone:UTC
H2O cluster version:3.28.0.2
H2O cluster version age:7 days, 23 hours and 16 minutes
H2O cluster name:H2O_from_python_seb_cy6gp3
H2O cluster total nodes:
H2O_cluster_version:3.46.0.1
H2O_cluster_version_age:26 days
H2O_cluster_name:H2O_from_python_tomasfryda_1jugg5
H2O_cluster_total_nodes:1
H2O cluster free memory:3.556 Gb
H2O cluster total cores:8
H2O cluster allowed cores:8
H2O cluster status:accepting new members, healthy
H2O connection url:
H2O_cluster_free_memory:3.547 Gb
H2O_cluster_total_cores:16
H2O_cluster_allowed_cores:16
H2O_cluster_status:locked, healthy
H2O_connection_url:http://127.0.0.1:54321
H2O connection proxy:{'http': None, 'https': None}
H2O internal security:
H2O_connection_proxy:{\"http\": null, \"https\": null}
H2O_internal_security:False
H2O API Extensions:Amazon S3, XGBoost, Algos, AutoML, Core V3, TargetEncoder, Core V4
Python version:3.7.5 final
" + "Python_version:\n", + "3.10.0 final\n", + " \n", + "\n" ], "text/plain": [ - "-------------------------- ------------------------------------------------------------------\n", - "H2O cluster uptime: 01 secs\n", - "H2O cluster timezone: Europe/Prague\n", - "H2O data parsing timezone: UTC\n", - "H2O cluster version: 3.28.0.2\n", - "H2O cluster version age: 7 days, 23 hours and 16 minutes\n", - "H2O cluster name: H2O_from_python_seb_cy6gp3\n", - "H2O cluster total nodes: 1\n", - "H2O cluster free memory: 3.556 Gb\n", - "H2O cluster total cores: 8\n", - "H2O cluster allowed cores: 8\n", - "H2O cluster status: accepting new members, healthy\n", - "H2O connection url: http://127.0.0.1:54321\n", - "H2O connection proxy: {'http': None, 'https': None}\n", - "H2O internal security: False\n", - "H2O API Extensions: Amazon S3, XGBoost, Algos, AutoML, Core V3, TargetEncoder, Core V4\n", - "Python version: 3.7.5 final\n", - "-------------------------- ------------------------------------------------------------------" + "-------------------------- ---------------------------------\n", + "H2O_cluster_uptime: 03 secs\n", + "H2O_cluster_timezone: Europe/Prague\n", + "H2O_data_parsing_timezone: UTC\n", + "H2O_cluster_version: 3.46.0.1\n", + "H2O_cluster_version_age: 26 days\n", + "H2O_cluster_name: H2O_from_python_tomasfryda_1jugg5\n", + "H2O_cluster_total_nodes: 1\n", + "H2O_cluster_free_memory: 3.547 Gb\n", + "H2O_cluster_total_cores: 16\n", + "H2O_cluster_allowed_cores: 16\n", + "H2O_cluster_status: locked, healthy\n", + "H2O_connection_url: http://127.0.0.1:54321\n", + "H2O_connection_proxy: {\"http\": null, \"https\": null}\n", + "H2O_internal_security: False\n", + "Python_version: 3.10.0 final\n", + "-------------------------- ---------------------------------" ] }, "metadata": {}, @@ -332,7 +379,7 @@ "source": [ "from h2o.automl import H2OAutoML\n", "\n", - "aml = H2OAutoML(project_name=\"without_plugin\", max_models=15, seed=42)" + "aml = H2OAutoML(project_name=\"without_plugin\", max_models=20, seed=42)" ] }, { @@ -344,7 +391,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Parse progress: |█████████████████████████████████████████████████████████| 100%\n" + "Parse progress: |████████████████████████████████████████████████████████████████| (done) 100%\n" ] } ], @@ -372,8 +419,1460 @@ "name": "stdout", "output_type": "stream", "text": [ - "AutoML progress: |████████████████████████████████████████████████████████| 100%\n" + "AutoML progress: |███████████████████████████████████████████████████████████████| (done) 100%\n" ] + }, + { + "data": { + "text/html": [ + "
Model Details\n",
+       "=============\n",
+       "H2OStackedEnsembleEstimator : Stacked Ensemble\n",
+       "Model Key: StackedEnsemble_BestOfFamily_1_AutoML_1_20240409_111346\n",
+       "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Model Summary for Stacked Ensemble:
keyvalue
Stacking strategycross_validation
Number of base models (used / total)4/6
# GBM base models (used / total)0/1
# XGBoost base models (used / total)1/1
# GLM base models (used / total)1/1
# DRF base models (used / total)2/2
# DeepLearning base models (used / total)0/1
Metalearner algorithmGLM
Metalearner fold assignment schemeRandom
Metalearner nfolds5
Metalearner fold_columnNone
Custom metalearner hyperparametersNone
\n", + "
\n", + "
\n", + "
ModelMetricsBinomialGLM: stackedensemble\n",
+       "** Reported on train data. **\n",
+       "\n",
+       "MSE: 0.10290879720448431\n",
+       "RMSE: 0.3207940105495804\n",
+       "LogLoss: 0.342063416354005\n",
+       "AUC: 0.9456681350954478\n",
+       "AUCPR: 0.9214480152822256\n",
+       "Gini: 0.8913362701908956\n",
+       "Null degrees of freedom: 379\n",
+       "Residual degrees of freedom: 375\n",
+       "Null deviance: 512.2888401848891\n",
+       "Residual deviance: 259.9681964290438\n",
+       "AIC: 269.9681964290438
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.47830442786395266
01ErrorRate
0206.021.00.0925 (21.0/227.0)
126.0127.00.1699 (26.0/153.0)
Total232.0148.00.1237 (47.0/380.0)
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Maximum Metrics: Maximum metrics at their respective thresholds
metricthresholdvalueidx
max f10.47830440.8438538147.0
max f20.27845640.9124088209.0
max f0point50.56305800.8628659123.0
max accuracy0.47830440.8763158147.0
max precision0.99374991.00.0
max recall0.16901891.0262.0
max specificity0.99374991.00.0
max absolute_mcc0.47830440.7417846147.0
max min_per_class_accuracy0.41188990.8546256163.0
max mean_per_class_accuracy0.47830440.8687772147.0
max tns0.9937499227.00.0
max fns0.9937499152.00.0
max fps0.0031108227.0379.0
max tps0.1690189153.0262.0
max tnr0.99374991.00.0
max fnr0.99374990.99346410.0
max fpr0.00311081.0379.0
max tpr0.16901891.0262.0
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Gains/Lift Table: Avg response rate: 40.26 %, avg score: 40.19 %
groupcumulative_data_fractionlower_thresholdliftcumulative_liftresponse_ratescorecumulative_response_ratecumulative_scorecapture_ratecumulative_capture_rategaincumulative_gainkolmogorov_smirnov
10.01052630.97409262.48366012.48366011.00.98348701.00.98348700.02614380.0261438148.3660131148.36601310.0261438
20.02105260.96074352.48366012.48366011.00.96943451.00.97646070.02614380.0522876148.3660131148.36601310.0522876
30.03157890.95213382.48366012.48366011.00.95545051.00.96945730.02614380.0784314148.3660131148.36601310.0784314
40.04210530.93080372.48366012.48366011.00.94175081.00.96253070.02614380.1045752148.3660131148.36601310.1045752
50.050.91356912.48366012.48366011.00.92265371.00.95623430.01960780.1241830148.3660131148.36601310.1241830
60.10.83446942.48366012.48366011.00.87771121.00.91697280.12418300.2483660148.3660131148.36601310.2483660
70.150.76929702.35294122.44008710.94736840.80384760.98245610.87926440.11764710.3660131135.2941176144.00871460.3616078
80.20.72446282.22222222.38562090.89473680.74918700.96052630.84674510.11111110.4771242122.2222222138.56209150.4639083
90.30.59862282.02614382.26579520.81578950.66841140.91228070.78730050.20261440.6797386102.6143791126.57952070.6356857
100.40.45667221.50326802.07516340.60526320.52836600.83552630.72256690.15032680.830065450.3267974107.51633990.7199332
110.50.34325231.04575161.86928100.42105260.39241970.75263160.65653740.10457520.93464054.575163486.92810460.7275921
120.60.23324130.45751631.63398690.18421050.27907370.65789470.59362680.04575160.9803922-54.248366063.39869280.6367798
130.70.16608150.19607841.42857140.07894740.19558620.57518800.53676390.01960781.0-80.392156942.85714290.5022026
140.80.11732610.01.250.00.14008150.50328950.48717860.01.0-100.025.00.3348018
150.90.06617630.01.11111110.00.08893760.44736840.44292960.01.0-100.011.11111110.1674009
161.00.00311080.01.00.00.03283460.40263160.40192010.01.0-100.00.00.0
\n", + "
\n", + "
\n", + "
ModelMetricsBinomialGLM: stackedensemble\n",
+       "** Reported on cross-validation data. **\n",
+       "\n",
+       "MSE: 0.17465666065170454\n",
+       "RMSE: 0.41791944277779725\n",
+       "LogLoss: 0.5287043166690218\n",
+       "AUC: 0.8107454435518701\n",
+       "AUCPR: 0.7077792265050035\n",
+       "Gini: 0.6214908871037401\n",
+       "Null degrees of freedom: 379\n",
+       "Residual degrees of freedom: 374\n",
+       "Null deviance: 515.5918625851845\n",
+       "Residual deviance: 401.8152806684566\n",
+       "AIC: 413.8152806684566
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.34546184637575167
01ErrorRate
0160.067.00.2952 (67.0/227.0)
127.0126.00.1765 (27.0/153.0)
Total187.0193.00.2474 (94.0/380.0)
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Maximum Metrics: Maximum metrics at their respective thresholds
metricthresholdvalueidx
max f10.34546180.7283237192.0
max f20.17400430.8082497284.0
max f0point50.50532200.7132244129.0
max accuracy0.45073470.7631579152.0
max precision0.98485741.00.0
max recall0.06517141.0358.0
max specificity0.98485741.00.0
max absolute_mcc0.34546180.5183244192.0
max min_per_class_accuracy0.41052550.7533040171.0
max mean_per_class_accuracy0.34546180.7641876192.0
max tns0.9848574227.00.0
max fns0.9848574152.00.0
max fps0.0012083227.0379.0
max tps0.0651714153.0358.0
max tnr0.98485741.00.0
max fnr0.98485740.99346410.0
max fpr0.00120831.0379.0
max tpr0.06517141.0358.0
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Gains/Lift Table: Avg response rate: 40.26 %, avg score: 40.34 %
groupcumulative_data_fractionlower_thresholdliftcumulative_liftresponse_ratescorecumulative_response_ratecumulative_scorecapture_ratecumulative_capture_rategaincumulative_gainkolmogorov_smirnov
10.01052630.94818612.48366012.48366011.00.96469421.00.96469420.02614380.0261438148.3660131148.36601310.0261438
20.02105260.93449321.24183011.86274510.50.94181780.750.95325600.01307190.039215724.183006586.27450980.0304051
30.03157890.91302651.86274511.86274510.750.92594080.750.94415090.01960780.058823586.274509886.27450980.0456077
40.04210530.90499682.48366012.01797391.00.90765270.81250.93502640.02614380.0849673148.3660131101.79738560.0717515
50.050.88793472.48366012.09150331.00.89616600.84210530.92889050.01960780.1045752148.3660131109.15032680.0913593
60.10.81059371.56862751.83006540.63157890.85730700.73684210.89309880.07843140.183006556.862745183.00653590.1389537
70.150.72376051.96078431.87363830.78947370.76986540.75438600.85202100.09803920.281045896.078431487.36383440.2193717
80.20.67283911.83006541.86274510.73684210.69776020.750.81345580.09150330.372549083.006535986.27450980.2888486
90.30.54072941.76470591.83006540.71052630.60324890.73684210.74338680.17647060.549019676.470588283.00653590.4168610
100.40.45160461.50326801.74836600.60526320.49579170.70394740.68148810.15032680.699346450.326797474.83660130.5011085
110.50.36176071.11111111.62091500.44736840.41314470.65263160.62781940.11111110.810457511.111111162.09150330.5197086
120.60.27157590.52287581.43790850.21052630.30820380.57894740.57455010.05228760.8627451-47.712418343.79084970.4398376
130.70.20052100.52287581.30718950.21052630.23270810.52631580.52571550.05228760.9150327-47.712418330.71895420.3599666
140.80.14705440.45751631.20098040.18421050.17403990.48355260.48175610.04575160.9607843-54.248366020.09803920.2691544
150.90.09347670.19607841.08932460.07894740.12090210.43859650.44166120.01960780.9803922-80.39215698.93246190.1345772
161.00.00120830.19607841.00.07894740.05881500.40263160.40337660.01960781.0-80.39215690.00.0
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Cross-Validation Metrics Summary:
meansdcv_1_validcv_2_validcv_3_validcv_4_validcv_5_valid
accuracy0.77620730.03930270.73239430.73809520.80555560.78571430.8192771
aic90.7312511.92466390.25044108.7973777.8161182.7186494.07366
auc0.81348150.05421030.780.74806890.86467350.8733660.8012987
err0.22379270.03930270.26760560.26190480.19444440.21428570.1807229
err_count17.03.39116519.022.014.015.015.0
f0point50.70933070.06345700.6211180.66298340.74866310.76388890.75
f10.73687550.06539800.67796610.68571430.80.81481480.7058824
f20.77098120.09130140.74626860.71005920.85889570.87301590.6666667
lift_top_group2.01426221.19807912.840.02.32258061.94444442.9642856
loglikelihood0.00.00.00.00.00.00.0
------------------------
mean_per_class_error0.22813710.03180900.25217390.26381460.18253340.21813730.2240260
mse0.1742950.01840490.18403940.19818890.15026470.17521270.1637693
null_deviance103.118377.682760293.271225112.6165798.773834102.38067108.5495760
pr_auc0.71722910.13144930.63553930.56397950.82673860.87766390.6822243
precision0.69415490.07618230.58823530.64864860.71794870.73333330.7826087
r20.26309030.08713790.19326730.16909030.38711850.29857660.2673985
recall0.79800440.11643660.80.72727280.90322580.91666670.6428571
residual_deviance79.93124412.52172278.2504498.7973765.8161172.7186484.07366
rmse0.41701810.02210490.42899810.44518410.38764000.41858410.4046841
specificity0.74572150.09890350.69565220.74509800.73170730.64705880.9090909
\n", + "
\n", + "
[24 rows x 8 columns]
\n",
+       "\n",
+       "[tips]\n",
+       "Use `model.explain()` to inspect the model.\n",
+       "--\n",
+       "Use `h2o.display.toggle_user_tips()` to switch on/off this section.
" + ], + "text/plain": [ + "Model Details\n", + "=============\n", + "H2OStackedEnsembleEstimator : Stacked Ensemble\n", + "Model Key: StackedEnsemble_BestOfFamily_1_AutoML_1_20240409_111346\n", + "\n", + "\n", + "Model Summary for Stacked Ensemble: \n", + "key value\n", + "----------------------------------------- ----------------\n", + "Stacking strategy cross_validation\n", + "Number of base models (used / total) 4/6\n", + "# GBM base models (used / total) 0/1\n", + "# XGBoost base models (used / total) 1/1\n", + "# GLM base models (used / total) 1/1\n", + "# DRF base models (used / total) 2/2\n", + "# DeepLearning base models (used / total) 0/1\n", + "Metalearner algorithm GLM\n", + "Metalearner fold assignment scheme Random\n", + "Metalearner nfolds 5\n", + "Metalearner fold_column\n", + "Custom metalearner hyperparameters None\n", + "\n", + "ModelMetricsBinomialGLM: stackedensemble\n", + "** Reported on train data. **\n", + "\n", + "MSE: 0.10290879720448431\n", + "RMSE: 0.3207940105495804\n", + "LogLoss: 0.342063416354005\n", + "AUC: 0.9456681350954478\n", + "AUCPR: 0.9214480152822256\n", + "Gini: 0.8913362701908956\n", + "Null degrees of freedom: 379\n", + "Residual degrees of freedom: 375\n", + "Null deviance: 512.2888401848891\n", + "Residual deviance: 259.9681964290438\n", + "AIC: 269.9681964290438\n", + "\n", + "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.47830442786395266\n", + " 0 1 Error Rate\n", + "----- --- --- ------- ------------\n", + "0 206 21 0.0925 (21.0/227.0)\n", + "1 26 127 0.1699 (26.0/153.0)\n", + "Total 232 148 0.1237 (47.0/380.0)\n", + "\n", + "Maximum Metrics: Maximum metrics at their respective thresholds\n", + "metric threshold value idx\n", + "--------------------------- ----------- -------- -----\n", + "max f1 0.478304 0.843854 147\n", + "max f2 0.278456 0.912409 209\n", + "max f0point5 0.563058 0.862866 123\n", + "max accuracy 0.478304 0.876316 147\n", + "max precision 0.99375 1 0\n", + "max recall 0.169019 1 262\n", + "max specificity 0.99375 1 0\n", + "max absolute_mcc 0.478304 0.741785 147\n", + "max min_per_class_accuracy 0.41189 0.854626 163\n", + "max mean_per_class_accuracy 0.478304 0.868777 147\n", + "max tns 0.99375 227 0\n", + "max fns 0.99375 152 0\n", + "max fps 0.00311081 227 379\n", + "max tps 0.169019 153 262\n", + "max tnr 0.99375 1 0\n", + "max fnr 0.99375 0.993464 0\n", + "max fpr 0.00311081 1 379\n", + "max tpr 0.169019 1 262\n", + "\n", + "Gains/Lift Table: Avg response rate: 40.26 %, avg score: 40.19 %\n", + "group cumulative_data_fraction lower_threshold lift cumulative_lift response_rate score cumulative_response_rate cumulative_score capture_rate cumulative_capture_rate gain cumulative_gain kolmogorov_smirnov\n", + "------- -------------------------- ----------------- -------- ----------------- --------------- --------- -------------------------- ------------------ -------------- ------------------------- -------- ----------------- --------------------\n", + "1 0.0105263 0.974093 2.48366 2.48366 1 0.983487 1 0.983487 0.0261438 0.0261438 148.366 148.366 0.0261438\n", + "2 0.0210526 0.960744 2.48366 2.48366 1 0.969435 1 0.976461 0.0261438 0.0522876 148.366 148.366 0.0522876\n", + "3 0.0315789 0.952134 2.48366 2.48366 1 0.955451 1 0.969457 0.0261438 0.0784314 148.366 148.366 0.0784314\n", + "4 0.0421053 0.930804 2.48366 2.48366 1 0.941751 1 0.962531 0.0261438 0.104575 148.366 148.366 0.104575\n", + "5 0.05 0.913569 2.48366 2.48366 1 0.922654 1 0.956234 0.0196078 0.124183 148.366 148.366 0.124183\n", + "6 0.1 0.834469 2.48366 2.48366 1 0.877711 1 0.916973 0.124183 0.248366 148.366 148.366 0.248366\n", + "7 0.15 0.769297 2.35294 2.44009 0.947368 0.803848 0.982456 0.879264 0.117647 0.366013 135.294 144.009 0.361608\n", + "8 0.2 0.724463 2.22222 2.38562 0.894737 0.749187 0.960526 0.846745 0.111111 0.477124 122.222 138.562 0.463908\n", + "9 0.3 0.598623 2.02614 2.2658 0.815789 0.668411 0.912281 0.787301 0.202614 0.679739 102.614 126.58 0.635686\n", + "10 0.4 0.456672 1.50327 2.07516 0.605263 0.528366 0.835526 0.722567 0.150327 0.830065 50.3268 107.516 0.719933\n", + "11 0.5 0.343252 1.04575 1.86928 0.421053 0.39242 0.752632 0.656537 0.104575 0.934641 4.57516 86.9281 0.727592\n", + "12 0.6 0.233241 0.457516 1.63399 0.184211 0.279074 0.657895 0.593627 0.0457516 0.980392 -54.2484 63.3987 0.63678\n", + "13 0.7 0.166082 0.196078 1.42857 0.0789474 0.195586 0.575188 0.536764 0.0196078 1 -80.3922 42.8571 0.502203\n", + "14 0.8 0.117326 0 1.25 0 0.140082 0.503289 0.487179 0 1 -100 25 0.334802\n", + "15 0.9 0.0661763 0 1.11111 0 0.0889376 0.447368 0.44293 0 1 -100 11.1111 0.167401\n", + "16 1 0.00311081 0 1 0 0.0328346 0.402632 0.40192 0 1 -100 0 0\n", + "\n", + "ModelMetricsBinomialGLM: stackedensemble\n", + "** Reported on cross-validation data. **\n", + "\n", + "MSE: 0.17465666065170454\n", + "RMSE: 0.41791944277779725\n", + "LogLoss: 0.5287043166690218\n", + "AUC: 0.8107454435518701\n", + "AUCPR: 0.7077792265050035\n", + "Gini: 0.6214908871037401\n", + "Null degrees of freedom: 379\n", + "Residual degrees of freedom: 374\n", + "Null deviance: 515.5918625851845\n", + "Residual deviance: 401.8152806684566\n", + "AIC: 413.8152806684566\n", + "\n", + "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.34546184637575167\n", + " 0 1 Error Rate\n", + "----- --- --- ------- ------------\n", + "0 160 67 0.2952 (67.0/227.0)\n", + "1 27 126 0.1765 (27.0/153.0)\n", + "Total 187 193 0.2474 (94.0/380.0)\n", + "\n", + "Maximum Metrics: Maximum metrics at their respective thresholds\n", + "metric threshold value idx\n", + "--------------------------- ----------- -------- -----\n", + "max f1 0.345462 0.728324 192\n", + "max f2 0.174004 0.80825 284\n", + "max f0point5 0.505322 0.713224 129\n", + "max accuracy 0.450735 0.763158 152\n", + "max precision 0.984857 1 0\n", + "max recall 0.0651714 1 358\n", + "max specificity 0.984857 1 0\n", + "max absolute_mcc 0.345462 0.518324 192\n", + "max min_per_class_accuracy 0.410526 0.753304 171\n", + "max mean_per_class_accuracy 0.345462 0.764188 192\n", + "max tns 0.984857 227 0\n", + "max fns 0.984857 152 0\n", + "max fps 0.00120828 227 379\n", + "max tps 0.0651714 153 358\n", + "max tnr 0.984857 1 0\n", + "max fnr 0.984857 0.993464 0\n", + "max fpr 0.00120828 1 379\n", + "max tpr 0.0651714 1 358\n", + "\n", + "Gains/Lift Table: Avg response rate: 40.26 %, avg score: 40.34 %\n", + "group cumulative_data_fraction lower_threshold lift cumulative_lift response_rate score cumulative_response_rate cumulative_score capture_rate cumulative_capture_rate gain cumulative_gain kolmogorov_smirnov\n", + "------- -------------------------- ----------------- -------- ----------------- --------------- -------- -------------------------- ------------------ -------------- ------------------------- -------- ----------------- --------------------\n", + "1 0.0105263 0.948186 2.48366 2.48366 1 0.964694 1 0.964694 0.0261438 0.0261438 148.366 148.366 0.0261438\n", + "2 0.0210526 0.934493 1.24183 1.86275 0.5 0.941818 0.75 0.953256 0.0130719 0.0392157 24.183 86.2745 0.0304051\n", + "3 0.0315789 0.913026 1.86275 1.86275 0.75 0.925941 0.75 0.944151 0.0196078 0.0588235 86.2745 86.2745 0.0456077\n", + "4 0.0421053 0.904997 2.48366 2.01797 1 0.907653 0.8125 0.935026 0.0261438 0.0849673 148.366 101.797 0.0717515\n", + "5 0.05 0.887935 2.48366 2.0915 1 0.896166 0.842105 0.928891 0.0196078 0.104575 148.366 109.15 0.0913593\n", + "6 0.1 0.810594 1.56863 1.83007 0.631579 0.857307 0.736842 0.893099 0.0784314 0.183007 56.8627 83.0065 0.138954\n", + "7 0.15 0.723761 1.96078 1.87364 0.789474 0.769865 0.754386 0.852021 0.0980392 0.281046 96.0784 87.3638 0.219372\n", + "8 0.2 0.672839 1.83007 1.86275 0.736842 0.69776 0.75 0.813456 0.0915033 0.372549 83.0065 86.2745 0.288849\n", + "9 0.3 0.540729 1.76471 1.83007 0.710526 0.603249 0.736842 0.743387 0.176471 0.54902 76.4706 83.0065 0.416861\n", + "10 0.4 0.451605 1.50327 1.74837 0.605263 0.495792 0.703947 0.681488 0.150327 0.699346 50.3268 74.8366 0.501109\n", + "11 0.5 0.361761 1.11111 1.62092 0.447368 0.413145 0.652632 0.627819 0.111111 0.810458 11.1111 62.0915 0.519709\n", + "12 0.6 0.271576 0.522876 1.43791 0.210526 0.308204 0.578947 0.57455 0.0522876 0.862745 -47.7124 43.7908 0.439838\n", + "13 0.7 0.200521 0.522876 1.30719 0.210526 0.232708 0.526316 0.525716 0.0522876 0.915033 -47.7124 30.719 0.359967\n", + "14 0.8 0.147054 0.457516 1.20098 0.184211 0.17404 0.483553 0.481756 0.0457516 0.960784 -54.2484 20.098 0.269154\n", + "15 0.9 0.0934767 0.196078 1.08932 0.0789474 0.120902 0.438596 0.441661 0.0196078 0.980392 -80.3922 8.93246 0.134577\n", + "16 1 0.00120828 0.196078 1 0.0789474 0.058815 0.402632 0.403377 0.0196078 1 -80.3922 0 0\n", + "\n", + "Cross-Validation Metrics Summary: \n", + " mean sd cv_1_valid cv_2_valid cv_3_valid cv_4_valid cv_5_valid\n", + "-------------------- ---------- ----------- ------------ ------------ ------------ ------------ ------------\n", + "accuracy 0.7762073 0.039302714 0.73239434 0.7380952 0.8055556 0.78571427 0.8192771\n", + "aic 90.73125 11.924663 90.25044 108.79737 77.81611 82.71864 94.07366\n", + "auc 0.81348145 0.054210264 0.78 0.7480689 0.8646735 0.873366 0.8012987\n", + "err 0.22379269 0.039302714 0.26760563 0.26190478 0.19444445 0.21428572 0.18072289\n", + "err_count 17.0 3.391165 19.0 22.0 14.0 15.0 15.0\n", + "f0point5 0.7093307 0.06345702 0.621118 0.6629834 0.7486631 0.7638889 0.75\n", + "f1 0.73687553 0.065398015 0.6779661 0.6857143 0.8 0.8148148 0.7058824\n", + "f2 0.7709812 0.09130137 0.74626863 0.71005917 0.8588957 0.8730159 0.6666667\n", + "lift_top_group 2.0142622 1.1980791 2.84 0.0 2.3225806 1.9444444 2.9642856\n", + "loglikelihood 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "--- --- --- --- --- --- --- ---\n", + "mean_per_class_error 0.22813705 0.031808984 0.2521739 0.26381463 0.18253344 0.21813725 0.22402598\n", + "mse 0.174295 0.018404886 0.1840394 0.19818892 0.15026474 0.17521268 0.16376925\n", + "null_deviance 103.11837 7.6827602 93.271225 112.61657 98.773834 102.38067 108.549576\n", + "pr_auc 0.7172291 0.13144928 0.6355393 0.5639795 0.8267386 0.87766385 0.6822243\n", + "precision 0.6941549 0.07618227 0.5882353 0.6486486 0.71794873 0.73333335 0.7826087\n", + "r2 0.26309025 0.08713793 0.19326732 0.16909033 0.38711846 0.29857665 0.26739848\n", + "recall 0.79800445 0.11643657 0.8 0.72727275 0.9032258 0.9166667 0.64285713\n", + "residual_deviance 79.931244 12.521722 78.25044 98.79737 65.81611 72.71864 84.07366\n", + "rmse 0.41701812 0.022104867 0.42899814 0.44518414 0.38763997 0.41858414 0.40468413\n", + "specificity 0.74572146 0.0989035 0.6956522 0.74509805 0.73170733 0.64705884 0.90909094\n", + "[24 rows x 8 columns]\n", + "\n", + "\n", + "[tips]\n", + "Use `model.explain()` to inspect the model.\n", + "--\n", + "Use `h2o.display.toggle_user_tips()` to switch on/off this section." + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -388,24 +1887,29 @@ { "data": { "text/plain": [ - "[{'name': 'XGBoost',\n", - " 'steps': [{'id': 'def_1', 'weight': 10},\n", - " {'id': 'def_2', 'weight': 10},\n", - " {'id': 'def_3', 'weight': 10}]},\n", - " {'name': 'GLM', 'steps': [{'id': 'def_1', 'weight': 10}]},\n", - " {'name': 'DRF', 'steps': [{'id': 'def_1', 'weight': 10}]},\n", + "[{'name': 'XGBoost', 'steps': [{'id': 'def_2', 'group': 1, 'weight': 10}]},\n", + " {'name': 'GLM', 'steps': [{'id': 'def_1', 'group': 1, 'weight': 10}]},\n", + " {'name': 'GBM', 'steps': [{'id': 'def_5', 'group': 1, 'weight': 10}]},\n", + " {'name': 'XGBoost', 'steps': [{'id': 'def_1', 'group': 2, 'weight': 10}]},\n", + " {'name': 'DRF', 'steps': [{'id': 'def_1', 'group': 2, 'weight': 10}]},\n", " {'name': 'GBM',\n", - " 'steps': [{'id': 'def_1', 'weight': 10},\n", - " {'id': 'def_2', 'weight': 10},\n", - " {'id': 'def_3', 'weight': 10},\n", - " {'id': 'def_4', 'weight': 10},\n", - " {'id': 'def_5', 'weight': 10}]},\n", - " {'name': 'DeepLearning', 'steps': [{'id': 'def_1', 'weight': 10}]},\n", - " {'name': 'DRF', 'steps': [{'id': 'XRT', 'weight': 10}]},\n", - " {'name': 'XGBoost', 'steps': [{'id': 'grid_1', 'weight': 100}]},\n", - " {'name': 'GBM', 'steps': [{'id': 'grid_1', 'weight': 60}]},\n", + " 'steps': [{'id': 'def_2', 'group': 2, 'weight': 10},\n", + " {'id': 'def_3', 'group': 2, 'weight': 10},\n", + " {'id': 'def_4', 'group': 2, 'weight': 10}]},\n", + " {'name': 'XGBoost', 'steps': [{'id': 'def_3', 'group': 3, 'weight': 10}]},\n", + " {'name': 'DRF', 'steps': [{'id': 'XRT', 'group': 3, 'weight': 10}]},\n", + " {'name': 'GBM', 'steps': [{'id': 'def_1', 'group': 3, 'weight': 10}]},\n", + " {'name': 'DeepLearning',\n", + " 'steps': [{'id': 'def_1', 'group': 3, 'weight': 10}]},\n", + " {'name': 'XGBoost', 'steps': [{'id': 'grid_1', 'group': 4, 'weight': 90}]},\n", + " {'name': 'GBM', 'steps': [{'id': 'grid_1', 'group': 4, 'weight': 60}]},\n", + " {'name': 'DeepLearning',\n", + " 'steps': [{'id': 'grid_1', 'group': 4, 'weight': 30},\n", + " {'id': 'grid_2', 'group': 5, 'weight': 30},\n", + " {'id': 'grid_3', 'group': 5, 'weight': 30}]},\n", " {'name': 'StackedEnsemble',\n", - " 'steps': [{'id': 'best', 'weight': 10}, {'id': 'all', 'weight': 10}]}]" + " 'steps': [{'id': 'best_of_family_xglm', 'group': 10, 'weight': 10},\n", + " {'id': 'all_xglm', 'group': 10, 'weight': 10}]}]" ] }, "execution_count": 8, @@ -453,16 +1957,15 @@ "\n", "# or even go into further details, \n", "# for example by tweaking the 'weight' property of the `modeling_plan` to produce more models from the `MyGBM` grid, relatively to other grids: \n", - "# this is currently applied only for grids when using `max_runtime_secs` and/or `max_models` constraints.\n", + "# this is currently applied only for grids when using `max_runtime_secs` and/or `max_models` constraints. \n", "yet_another_plan = [\n", " ('XGBoost', 'defaults'),\n", " ('GLM', 'defaults'),\n", " ('DRF', 'defaults'),\n", " ('GBM', 'defaults'),\n", " ('DeepLearning', 'defaults'),\n", - " ('MyGLM', 'grids'),\n", - " dict(name='MyDRF', steps=dict(id='grid_1', weight=100)),\n", - " dict(name='XGBoost', steps=dict(id='grid_1', weight=60)),\n", + " dict(name='MyDRF', steps=[dict(id='grid_1', group=1, weight=100)]),\n", + " dict(name='MyGLM', steps=[dict(id='solvers', group=1, weight=60)]),\n", " ('GBM', 'grids'),\n", " ('DeepLearning', 'grids'),\n", " 'StackedEnsemble'\n", @@ -479,24 +1982,29 @@ "text/plain": [ "['MyGLM',\n", " 'MyDRF',\n", - " {'name': 'XGBoost',\n", - " 'steps': [{'id': 'def_1', 'weight': 10},\n", - " {'id': 'def_2', 'weight': 10},\n", - " {'id': 'def_3', 'weight': 10}]},\n", - " {'name': 'GLM', 'steps': [{'id': 'def_1', 'weight': 10}]},\n", - " {'name': 'DRF', 'steps': [{'id': 'def_1', 'weight': 10}]},\n", + " {'name': 'XGBoost', 'steps': [{'id': 'def_2', 'group': 1, 'weight': 10}]},\n", + " {'name': 'GLM', 'steps': [{'id': 'def_1', 'group': 1, 'weight': 10}]},\n", + " {'name': 'GBM', 'steps': [{'id': 'def_5', 'group': 1, 'weight': 10}]},\n", + " {'name': 'XGBoost', 'steps': [{'id': 'def_1', 'group': 2, 'weight': 10}]},\n", + " {'name': 'DRF', 'steps': [{'id': 'def_1', 'group': 2, 'weight': 10}]},\n", " {'name': 'GBM',\n", - " 'steps': [{'id': 'def_1', 'weight': 10},\n", - " {'id': 'def_2', 'weight': 10},\n", - " {'id': 'def_3', 'weight': 10},\n", - " {'id': 'def_4', 'weight': 10},\n", - " {'id': 'def_5', 'weight': 10}]},\n", - " {'name': 'DeepLearning', 'steps': [{'id': 'def_1', 'weight': 10}]},\n", - " {'name': 'DRF', 'steps': [{'id': 'XRT', 'weight': 10}]},\n", - " {'name': 'XGBoost', 'steps': [{'id': 'grid_1', 'weight': 100}]},\n", - " {'name': 'GBM', 'steps': [{'id': 'grid_1', 'weight': 60}]},\n", + " 'steps': [{'id': 'def_2', 'group': 2, 'weight': 10},\n", + " {'id': 'def_3', 'group': 2, 'weight': 10},\n", + " {'id': 'def_4', 'group': 2, 'weight': 10}]},\n", + " {'name': 'XGBoost', 'steps': [{'id': 'def_3', 'group': 3, 'weight': 10}]},\n", + " {'name': 'DRF', 'steps': [{'id': 'XRT', 'group': 3, 'weight': 10}]},\n", + " {'name': 'GBM', 'steps': [{'id': 'def_1', 'group': 3, 'weight': 10}]},\n", + " {'name': 'DeepLearning',\n", + " 'steps': [{'id': 'def_1', 'group': 3, 'weight': 10}]},\n", + " {'name': 'XGBoost', 'steps': [{'id': 'grid_1', 'group': 4, 'weight': 90}]},\n", + " {'name': 'GBM', 'steps': [{'id': 'grid_1', 'group': 4, 'weight': 60}]},\n", + " {'name': 'DeepLearning',\n", + " 'steps': [{'id': 'grid_1', 'group': 4, 'weight': 30},\n", + " {'id': 'grid_2', 'group': 5, 'weight': 30},\n", + " {'id': 'grid_3', 'group': 5, 'weight': 30}]},\n", " {'name': 'StackedEnsemble',\n", - " 'steps': [{'id': 'best', 'weight': 10}, {'id': 'all', 'weight': 10}]}]" + " 'steps': [{'id': 'best_of_family_xglm', 'group': 10, 'weight': 10},\n", + " {'id': 'all_xglm', 'group': 10, 'weight': 10}]}]" ] }, "execution_count": 10, @@ -514,7 +2022,7 @@ "metadata": {}, "outputs": [], "source": [ - "aml_plugin = H2OAutoML(project_name=\"with_plugin\", max_models=25, modeling_plan=new_plan, seed=42)" + "aml_plugin = H2OAutoML(project_name=\"with_plugin\", max_models=20, modeling_plan=new_plan, seed=42)" ] }, { @@ -526,14 +2034,2945 @@ "name": "stdout", "output_type": "stream", "text": [ - "AutoML progress: |████████████████████████████████████████████████████████| 100%\n" + "AutoML progress: |███████████████████████████████████████████████████████████████| (done) 100%\n" ] + }, + { + "data": { + "text/html": [ + "
Model Details\n",
+       "=============\n",
+       "H2OStackedEnsembleEstimator : Stacked Ensemble\n",
+       "Model Key: StackedEnsemble_BestOfFamily_1_AutoML_2_20240409_111554\n",
+       "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Model Summary for Stacked Ensemble:
keyvalue
Stacking strategycross_validation
Number of base models (used / total)5/6
# GBM base models (used / total)0/1
# XGBoost base models (used / total)1/1
# GLM base models (used / total)1/1
# DeepLearning base models (used / total)1/1
# DRF base models (used / total)2/2
Metalearner algorithmGLM
Metalearner fold assignment schemeRandom
Metalearner nfolds5
Metalearner fold_columnNone
Custom metalearner hyperparametersNone
\n", + "
\n", + "
\n", + "
ModelMetricsBinomialGLM: stackedensemble\n",
+       "** Reported on train data. **\n",
+       "\n",
+       "MSE: 0.08428085752530143\n",
+       "RMSE: 0.29031165585505075\n",
+       "LogLoss: 0.2968382692640996\n",
+       "AUC: 0.9698828136247157\n",
+       "AUCPR: 0.9617000811501878\n",
+       "Gini: 0.9397656272494315\n",
+       "Null degrees of freedom: 379\n",
+       "Residual degrees of freedom: 374\n",
+       "Null deviance: 512.2888401848891\n",
+       "Residual deviance: 225.59708464071568\n",
+       "AIC: 237.59708464071568
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.49521924383835275
01ErrorRate
0215.012.00.0529 (12.0/227.0)
119.0134.00.1242 (19.0/153.0)
Total234.0146.00.0816 (31.0/380.0)
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Maximum Metrics: Maximum metrics at their respective thresholds
metricthresholdvalueidx
max f10.49521920.8963211145.0
max f20.37420250.9228824178.0
max f0point50.51140670.9186536139.0
max accuracy0.51140670.9184211139.0
max precision0.99655231.00.0
max recall0.20427881.0254.0
max specificity0.99655231.00.0
max absolute_mcc0.51140670.8301909139.0
max min_per_class_accuracy0.43246380.9074890159.0
max mean_per_class_accuracy0.45713760.9124701152.0
max tns0.9965523227.00.0
max fns0.9965523152.00.0
max fps0.0003468227.0379.0
max tps0.2042788153.0254.0
max tnr0.99655231.00.0
max fnr0.99655230.99346410.0
max fpr0.00034681.0379.0
max tpr0.20427881.0254.0
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Gains/Lift Table: Avg response rate: 40.26 %, avg score: 41.51 %
groupcumulative_data_fractionlower_thresholdliftcumulative_liftresponse_ratescorecumulative_response_ratecumulative_scorecapture_ratecumulative_capture_rategaincumulative_gainkolmogorov_smirnov
10.01052630.98995512.48366012.48366011.00.99492051.00.99492050.02614380.0261438148.3660131148.36601310.0261438
20.02105260.97705822.48366012.48366011.00.98302831.00.98897440.02614380.0522876148.3660131148.36601310.0522876
30.03157890.96654722.48366012.48366011.00.97252741.00.98349210.02614380.0784314148.3660131148.36601310.0784314
40.04210530.96340502.48366012.48366011.00.96500071.00.97886920.02614380.1045752148.3660131148.36601310.1045752
50.050.95744032.48366012.48366011.00.96126271.00.97608930.01960780.1241830148.3660131148.36601310.1241830
60.10.88385512.48366012.48366011.00.91877911.00.94743420.12418300.2483660148.3660131148.36601310.2483660
70.150.83863182.48366012.48366011.00.86182861.00.91889900.12418300.3725490148.3660131148.36601310.3725490
80.20.77217062.48366012.48366011.00.80372381.00.89010520.12418300.4967320148.3660131148.36601310.4967320
90.30.60844832.28758172.41830070.92105260.69824450.97368420.82615160.22875820.7254902128.7581699141.83006540.7122743
100.40.45948451.63398692.22222220.65789470.53502900.89473680.75337100.16339870.888888963.3986928122.22222220.8184043
110.50.32655400.65359481.90849670.26315790.39561770.76842110.68182030.06535950.9542484-34.640522990.84967320.7604158
120.60.24431210.26143791.63398690.10526320.27835750.65789470.61457650.02614380.9803922-73.856209263.39869280.6367798
130.70.17735040.19607841.42857140.07894740.21520310.57518800.55752320.01960781.0-80.392156942.85714290.5022026
140.80.10871150.01.250.00.13814030.50328950.50510030.01.0-100.025.00.3348018
150.90.05687210.01.11111110.00.08249620.44736840.45814430.01.0-100.011.11111110.1674009
161.00.00034680.01.00.00.02766730.40263160.41509660.01.0-100.00.00.0
\n", + "
\n", + "
\n", + "
ModelMetricsBinomialGLM: stackedensemble\n",
+       "** Reported on cross-validation data. **\n",
+       "\n",
+       "MSE: 0.171169924338239\n",
+       "RMSE: 0.4137268716656425\n",
+       "LogLoss: 0.5177156788424586\n",
+       "AUC: 0.8162160605798855\n",
+       "AUCPR: 0.7160218992973033\n",
+       "Gini: 0.632432121159771\n",
+       "Null degrees of freedom: 379\n",
+       "Residual degrees of freedom: 375\n",
+       "Null deviance: 515.5918625851845\n",
+       "Residual deviance: 393.46391592026856\n",
+       "AIC: 403.46391592026856
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.3921056943800239
01ErrorRate
0170.057.00.2511 (57.0/227.0)
135.0118.00.2288 (35.0/153.0)
Total205.0175.00.2421 (92.0/380.0)
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Maximum Metrics: Maximum metrics at their respective thresholds
metricthresholdvalueidx
max f10.39210570.7195122174.0
max f20.19643750.8182844273.0
max f0point50.48521480.7263752138.0
max accuracy0.48521480.7736842138.0
max precision0.99331001.00.0
max recall0.04990551.0366.0
max specificity0.99331001.00.0
max absolute_mcc0.47250310.5252645142.0
max min_per_class_accuracy0.39991850.7577093170.0
max mean_per_class_accuracy0.45547880.7604302149.0
max tns0.9933100227.00.0
max fns0.9933100152.00.0
max fps0.0033122227.0379.0
max tps0.0499055153.0366.0
max tnr0.99331001.00.0
max fnr0.99331000.99346410.0
max fpr0.00331221.0379.0
max tpr0.04990551.0366.0
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Gains/Lift Table: Avg response rate: 40.26 %, avg score: 40.32 %
groupcumulative_data_fractionlower_thresholdliftcumulative_liftresponse_ratescorecumulative_response_ratecumulative_scorecapture_ratecumulative_capture_rategaincumulative_gainkolmogorov_smirnov
10.01052630.95472032.48366012.48366011.00.97693181.00.97693180.02614380.0261438148.3660131148.36601310.0261438
20.02105260.92295961.86274512.17320260.750.93986560.8750.95839870.01960780.045751686.2745098117.32026140.0413463
30.03157890.90955942.48366012.27668851.00.91838130.91666670.94505960.02614380.0718954148.3660131127.66884530.0674901
40.04210530.87333671.24183012.01797390.50.88572970.81250.93022710.01307190.084967324.1830065101.79738560.0717515
50.050.86881571.65577341.96078430.66666670.87049070.78947370.92079500.01307190.098039265.577342096.07843140.0804181
60.10.78044731.69934641.83006540.68421050.82519420.73684210.87299460.08496730.183006569.934640583.00653590.1389537
70.150.73691021.96078431.87363830.78947370.75628020.75438600.83408980.09803920.281045896.078431487.36383440.2193717
80.20.68483501.96078431.89542480.78947370.71027920.76315790.80313720.09803920.379085096.078431489.54248370.2997898
90.30.54867101.83006541.87363830.73684210.61958900.75438600.74195440.18300650.562091583.006535987.36383440.4387435
100.40.44867231.43790851.76470590.57894740.49885810.71052630.68118030.14379080.705882443.790849776.47058820.5120498
110.50.35320210.84967321.58169930.34210530.39817720.63684210.62457970.08496730.7908497-15.032679758.16993460.4868849
120.60.27761970.71895421.43790850.28947370.31438870.57894740.57288120.07189540.8627451-28.104575243.79084970.4398376
130.70.20755200.58823531.31652660.23684210.23972680.53007520.52528770.05882350.9215686-41.176470631.65266110.3709078
140.80.15203990.52287581.21732030.21052630.17661920.49013160.48170420.05228760.9738562-47.712418321.73202610.2910368
150.90.09427920.13071901.09658680.05263160.12117560.44152050.44164540.01307190.9869281-86.92810469.65867830.1455184
161.00.00331220.13071901.00.05263160.05728520.40263160.40320940.01307191.0-86.92810460.00.0
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Cross-Validation Metrics Summary:
meansdcv_1_validcv_2_validcv_3_validcv_4_validcv_5_valid
accuracy0.76180020.03192440.73239430.750.79166670.80.7349398
aic88.5557410.71225386.2648105.2735577.54880581.9839291.70763
auc0.82134590.05080800.78521740.76232920.86939420.8774510.8123376
err0.23819980.03192440.26760560.250.20833330.20.2650602
err_count18.23.56370619.021.015.014.022.0
f0point50.68289420.06893260.62130180.67567570.73770490.76754390.6122449
f10.73888120.06594850.68852460.70422540.78260870.83333330.6857143
f20.80627310.06844090.77205880.73529410.83333330.91145830.7792208
lift_top_group2.5233530.4094572.842.54545452.32258061.94444442.9642856
loglikelihood0.00.00.00.00.00.00.0
------------------------
mean_per_class_error0.22609990.02275860.24304350.24866310.19866250.20506540.2350649
mse0.16969980.01771270.18045950.19336430.14906480.16820420.1574061
null_deviance103.118377.682760293.271225112.6165798.773834102.38067108.5495760
pr_auc0.74702110.10266870.66752610.64592090.83523610.87559400.7108287
precision0.65046990.07178080.58333330.65789470.71052630.72916670.5714286
r20.28255740.08397980.20895990.18931770.39201280.32663340.2958632
recall0.85958170.07683120.840.75757570.87096780.97222220.8571429
residual_deviance77.7557411.55556676.264895.2735565.54880569.9839281.70763
rmse0.41149960.02144410.42480520.43973200.38608910.41012710.3967444
specificity0.68821850.05136030.67391310.74509800.73170730.61764700.6727273
\n", + "
\n", + "
[24 rows x 8 columns]
\n",
+       "\n",
+       "[tips]\n",
+       "Use `model.explain()` to inspect the model.\n",
+       "--\n",
+       "Use `h2o.display.toggle_user_tips()` to switch on/off this section.
" + ], + "text/plain": [ + "Model Details\n", + "=============\n", + "H2OStackedEnsembleEstimator : Stacked Ensemble\n", + "Model Key: StackedEnsemble_BestOfFamily_1_AutoML_2_20240409_111554\n", + "\n", + "\n", + "Model Summary for Stacked Ensemble: \n", + "key value\n", + "----------------------------------------- ----------------\n", + "Stacking strategy cross_validation\n", + "Number of base models (used / total) 5/6\n", + "# GBM base models (used / total) 0/1\n", + "# XGBoost base models (used / total) 1/1\n", + "# GLM base models (used / total) 1/1\n", + "# DeepLearning base models (used / total) 1/1\n", + "# DRF base models (used / total) 2/2\n", + "Metalearner algorithm GLM\n", + "Metalearner fold assignment scheme Random\n", + "Metalearner nfolds 5\n", + "Metalearner fold_column\n", + "Custom metalearner hyperparameters None\n", + "\n", + "ModelMetricsBinomialGLM: stackedensemble\n", + "** Reported on train data. **\n", + "\n", + "MSE: 0.08428085752530143\n", + "RMSE: 0.29031165585505075\n", + "LogLoss: 0.2968382692640996\n", + "AUC: 0.9698828136247157\n", + "AUCPR: 0.9617000811501878\n", + "Gini: 0.9397656272494315\n", + "Null degrees of freedom: 379\n", + "Residual degrees of freedom: 374\n", + "Null deviance: 512.2888401848891\n", + "Residual deviance: 225.59708464071568\n", + "AIC: 237.59708464071568\n", + "\n", + "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.49521924383835275\n", + " 0 1 Error Rate\n", + "----- --- --- ------- ------------\n", + "0 215 12 0.0529 (12.0/227.0)\n", + "1 19 134 0.1242 (19.0/153.0)\n", + "Total 234 146 0.0816 (31.0/380.0)\n", + "\n", + "Maximum Metrics: Maximum metrics at their respective thresholds\n", + "metric threshold value idx\n", + "--------------------------- ----------- -------- -----\n", + "max f1 0.495219 0.896321 145\n", + "max f2 0.374203 0.922882 178\n", + "max f0point5 0.511407 0.918654 139\n", + "max accuracy 0.511407 0.918421 139\n", + "max precision 0.996552 1 0\n", + "max recall 0.204279 1 254\n", + "max specificity 0.996552 1 0\n", + "max absolute_mcc 0.511407 0.830191 139\n", + "max min_per_class_accuracy 0.432464 0.907489 159\n", + "max mean_per_class_accuracy 0.457138 0.91247 152\n", + "max tns 0.996552 227 0\n", + "max fns 0.996552 152 0\n", + "max fps 0.000346797 227 379\n", + "max tps 0.204279 153 254\n", + "max tnr 0.996552 1 0\n", + "max fnr 0.996552 0.993464 0\n", + "max fpr 0.000346797 1 379\n", + "max tpr 0.204279 1 254\n", + "\n", + "Gains/Lift Table: Avg response rate: 40.26 %, avg score: 41.51 %\n", + "group cumulative_data_fraction lower_threshold lift cumulative_lift response_rate score cumulative_response_rate cumulative_score capture_rate cumulative_capture_rate gain cumulative_gain kolmogorov_smirnov\n", + "------- -------------------------- ----------------- -------- ----------------- --------------- --------- -------------------------- ------------------ -------------- ------------------------- -------- ----------------- --------------------\n", + "1 0.0105263 0.989955 2.48366 2.48366 1 0.994921 1 0.994921 0.0261438 0.0261438 148.366 148.366 0.0261438\n", + "2 0.0210526 0.977058 2.48366 2.48366 1 0.983028 1 0.988974 0.0261438 0.0522876 148.366 148.366 0.0522876\n", + "3 0.0315789 0.966547 2.48366 2.48366 1 0.972527 1 0.983492 0.0261438 0.0784314 148.366 148.366 0.0784314\n", + "4 0.0421053 0.963405 2.48366 2.48366 1 0.965001 1 0.978869 0.0261438 0.104575 148.366 148.366 0.104575\n", + "5 0.05 0.95744 2.48366 2.48366 1 0.961263 1 0.976089 0.0196078 0.124183 148.366 148.366 0.124183\n", + "6 0.1 0.883855 2.48366 2.48366 1 0.918779 1 0.947434 0.124183 0.248366 148.366 148.366 0.248366\n", + "7 0.15 0.838632 2.48366 2.48366 1 0.861829 1 0.918899 0.124183 0.372549 148.366 148.366 0.372549\n", + "8 0.2 0.772171 2.48366 2.48366 1 0.803724 1 0.890105 0.124183 0.496732 148.366 148.366 0.496732\n", + "9 0.3 0.608448 2.28758 2.4183 0.921053 0.698245 0.973684 0.826152 0.228758 0.72549 128.758 141.83 0.712274\n", + "10 0.4 0.459485 1.63399 2.22222 0.657895 0.535029 0.894737 0.753371 0.163399 0.888889 63.3987 122.222 0.818404\n", + "11 0.5 0.326554 0.653595 1.9085 0.263158 0.395618 0.768421 0.68182 0.0653595 0.954248 -34.6405 90.8497 0.760416\n", + "12 0.6 0.244312 0.261438 1.63399 0.105263 0.278357 0.657895 0.614577 0.0261438 0.980392 -73.8562 63.3987 0.63678\n", + "13 0.7 0.17735 0.196078 1.42857 0.0789474 0.215203 0.575188 0.557523 0.0196078 1 -80.3922 42.8571 0.502203\n", + "14 0.8 0.108711 0 1.25 0 0.13814 0.503289 0.5051 0 1 -100 25 0.334802\n", + "15 0.9 0.0568721 0 1.11111 0 0.0824962 0.447368 0.458144 0 1 -100 11.1111 0.167401\n", + "16 1 0.000346797 0 1 0 0.0276673 0.402632 0.415097 0 1 -100 0 0\n", + "\n", + "ModelMetricsBinomialGLM: stackedensemble\n", + "** Reported on cross-validation data. **\n", + "\n", + "MSE: 0.171169924338239\n", + "RMSE: 0.4137268716656425\n", + "LogLoss: 0.5177156788424586\n", + "AUC: 0.8162160605798855\n", + "AUCPR: 0.7160218992973033\n", + "Gini: 0.632432121159771\n", + "Null degrees of freedom: 379\n", + "Residual degrees of freedom: 375\n", + "Null deviance: 515.5918625851845\n", + "Residual deviance: 393.46391592026856\n", + "AIC: 403.46391592026856\n", + "\n", + "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.3921056943800239\n", + " 0 1 Error Rate\n", + "----- --- --- ------- ------------\n", + "0 170 57 0.2511 (57.0/227.0)\n", + "1 35 118 0.2288 (35.0/153.0)\n", + "Total 205 175 0.2421 (92.0/380.0)\n", + "\n", + "Maximum Metrics: Maximum metrics at their respective thresholds\n", + "metric threshold value idx\n", + "--------------------------- ----------- -------- -----\n", + "max f1 0.392106 0.719512 174\n", + "max f2 0.196437 0.818284 273\n", + "max f0point5 0.485215 0.726375 138\n", + "max accuracy 0.485215 0.773684 138\n", + "max precision 0.99331 1 0\n", + "max recall 0.0499055 1 366\n", + "max specificity 0.99331 1 0\n", + "max absolute_mcc 0.472503 0.525265 142\n", + "max min_per_class_accuracy 0.399918 0.757709 170\n", + "max mean_per_class_accuracy 0.455479 0.76043 149\n", + "max tns 0.99331 227 0\n", + "max fns 0.99331 152 0\n", + "max fps 0.00331223 227 379\n", + "max tps 0.0499055 153 366\n", + "max tnr 0.99331 1 0\n", + "max fnr 0.99331 0.993464 0\n", + "max fpr 0.00331223 1 379\n", + "max tpr 0.0499055 1 366\n", + "\n", + "Gains/Lift Table: Avg response rate: 40.26 %, avg score: 40.32 %\n", + "group cumulative_data_fraction lower_threshold lift cumulative_lift response_rate score cumulative_response_rate cumulative_score capture_rate cumulative_capture_rate gain cumulative_gain kolmogorov_smirnov\n", + "------- -------------------------- ----------------- -------- ----------------- --------------- --------- -------------------------- ------------------ -------------- ------------------------- -------- ----------------- --------------------\n", + "1 0.0105263 0.95472 2.48366 2.48366 1 0.976932 1 0.976932 0.0261438 0.0261438 148.366 148.366 0.0261438\n", + "2 0.0210526 0.92296 1.86275 2.1732 0.75 0.939866 0.875 0.958399 0.0196078 0.0457516 86.2745 117.32 0.0413463\n", + "3 0.0315789 0.909559 2.48366 2.27669 1 0.918381 0.916667 0.94506 0.0261438 0.0718954 148.366 127.669 0.0674901\n", + "4 0.0421053 0.873337 1.24183 2.01797 0.5 0.88573 0.8125 0.930227 0.0130719 0.0849673 24.183 101.797 0.0717515\n", + "5 0.05 0.868816 1.65577 1.96078 0.666667 0.870491 0.789474 0.920795 0.0130719 0.0980392 65.5773 96.0784 0.0804181\n", + "6 0.1 0.780447 1.69935 1.83007 0.684211 0.825194 0.736842 0.872995 0.0849673 0.183007 69.9346 83.0065 0.138954\n", + "7 0.15 0.73691 1.96078 1.87364 0.789474 0.75628 0.754386 0.83409 0.0980392 0.281046 96.0784 87.3638 0.219372\n", + "8 0.2 0.684835 1.96078 1.89542 0.789474 0.710279 0.763158 0.803137 0.0980392 0.379085 96.0784 89.5425 0.29979\n", + "9 0.3 0.548671 1.83007 1.87364 0.736842 0.619589 0.754386 0.741954 0.183007 0.562092 83.0065 87.3638 0.438743\n", + "10 0.4 0.448672 1.43791 1.76471 0.578947 0.498858 0.710526 0.68118 0.143791 0.705882 43.7908 76.4706 0.51205\n", + "11 0.5 0.353202 0.849673 1.5817 0.342105 0.398177 0.636842 0.62458 0.0849673 0.79085 -15.0327 58.1699 0.486885\n", + "12 0.6 0.27762 0.718954 1.43791 0.289474 0.314389 0.578947 0.572881 0.0718954 0.862745 -28.1046 43.7908 0.439838\n", + "13 0.7 0.207552 0.588235 1.31653 0.236842 0.239727 0.530075 0.525288 0.0588235 0.921569 -41.1765 31.6527 0.370908\n", + "14 0.8 0.15204 0.522876 1.21732 0.210526 0.176619 0.490132 0.481704 0.0522876 0.973856 -47.7124 21.732 0.291037\n", + "15 0.9 0.0942792 0.130719 1.09659 0.0526316 0.121176 0.44152 0.441645 0.0130719 0.986928 -86.9281 9.65868 0.145518\n", + "16 1 0.00331223 0.130719 1 0.0526316 0.0572852 0.402632 0.403209 0.0130719 1 -86.9281 0 0\n", + "\n", + "Cross-Validation Metrics Summary: \n", + " mean sd cv_1_valid cv_2_valid cv_3_valid cv_4_valid cv_5_valid\n", + "-------------------- ---------- ----------- ------------ ------------ ------------ ------------ ------------\n", + "accuracy 0.76180017 0.03192437 0.73239434 0.75 0.7916667 0.8 0.73493975\n", + "aic 88.55574 10.712253 86.2648 105.27355 77.548805 81.98392 91.70763\n", + "auc 0.82134587 0.050807994 0.7852174 0.76232916 0.8693942 0.877451 0.81233764\n", + "err 0.23819984 0.03192437 0.26760563 0.25 0.20833333 0.2 0.26506025\n", + "err_count 18.2 3.563706 19.0 21.0 15.0 14.0 22.0\n", + "f0point5 0.68289423 0.068932645 0.6213018 0.6756757 0.73770493 0.76754385 0.6122449\n", + "f1 0.73888123 0.06594853 0.6885246 0.70422536 0.7826087 0.8333333 0.6857143\n", + "f2 0.8062731 0.068440944 0.77205884 0.7352941 0.8333333 0.9114583 0.77922076\n", + "lift_top_group 2.523353 0.409457 2.84 2.5454545 2.3225806 1.9444444 2.9642856\n", + "loglikelihood 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "--- --- --- --- --- --- --- ---\n", + "mean_per_class_error 0.22609986 0.02275862 0.24304348 0.2486631 0.19866247 0.20506535 0.23506494\n", + "mse 0.16969977 0.017712666 0.18045947 0.19336428 0.14906476 0.16820423 0.1574061\n", + "null_deviance 103.11837 7.6827602 93.271225 112.61657 98.773834 102.38067 108.549576\n", + "pr_auc 0.74702114 0.1026687 0.6675261 0.64592093 0.8352361 0.87559396 0.71082866\n", + "precision 0.6504699 0.0717808 0.5833333 0.65789473 0.7105263 0.7291667 0.5714286\n", + "r2 0.2825574 0.08397985 0.20895985 0.18931769 0.39201277 0.3266334 0.2958632\n", + "recall 0.8595817 0.0768312 0.84 0.75757575 0.87096775 0.9722222 0.85714287\n", + "residual_deviance 77.75574 11.555566 76.2648 95.27355 65.548805 69.98392 81.70763\n", + "rmse 0.41149956 0.02144415 0.42480522 0.43973204 0.38608906 0.41012707 0.39674437\n", + "specificity 0.68821853 0.051360276 0.67391306 0.74509805 0.73170733 0.61764705 0.6727273\n", + "[24 rows x 8 columns]\n", + "\n", + "\n", + "[tips]\n", + "Use `model.explain()` to inspect the model.\n", + "--\n", + "Use `h2o.display.toggle_user_tips()` to switch on/off this section." + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "aml_plugin.train(y=target, training_frame=train)" ] }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "aml_plugin_with_weight_and_group = H2OAutoML(project_name=\"with_plugin_weight_and_group\",\n", + " max_models=20, modeling_plan=yet_another_plan, seed=42)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "AutoML progress: |███████████████████████████████████████████████████████████████| (done) 100%\n" + ] + }, + { + "data": { + "text/html": [ + "
Model Details\n",
+       "=============\n",
+       "H2OStackedEnsembleEstimator : Stacked Ensemble\n",
+       "Model Key: StackedEnsemble_BestOfFamily_5_AutoML_3_20240409_111636\n",
+       "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Model Summary for Stacked Ensemble:
keyvalue
Stacking strategycross_validation
Number of base models (used / total)6/6
# GBM base models (used / total)1/1
# XGBoost base models (used / total)1/1
# GLM base models (used / total)1/1
# DRF base models (used / total)2/2
# DeepLearning base models (used / total)1/1
Metalearner algorithmGLM
Metalearner fold assignment schemeRandom
Metalearner nfolds5
Metalearner fold_columnNone
Custom metalearner hyperparametersNone
\n", + "
\n", + "
\n", + "
ModelMetricsBinomialGLM: stackedensemble\n",
+       "** Reported on train data. **\n",
+       "\n",
+       "MSE: 0.09184265410860146\n",
+       "RMSE: 0.3030555297443052\n",
+       "LogLoss: 0.31372343953161186\n",
+       "AUC: 0.9598917393682876\n",
+       "AUCPR: 0.9475549649186908\n",
+       "Gini: 0.9197834787365753\n",
+       "Null degrees of freedom: 379\n",
+       "Residual degrees of freedom: 373\n",
+       "Null deviance: 512.2888401848891\n",
+       "Residual deviance: 238.42981404402497\n",
+       "AIC: 252.42981404402497
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.379784359056072
01ErrorRate
0195.032.00.141 (32.0/227.0)
114.0139.00.0915 (14.0/153.0)
Total209.0171.00.1211 (46.0/380.0)
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Maximum Metrics: Maximum metrics at their respective thresholds
metricthresholdvalueidx
max f10.37978440.8580247170.0
max f20.27238520.9118727204.0
max f0point50.59025070.896117.0
max accuracy0.54735980.8842105130.0
max precision0.98860061.00.0
max recall0.18334531.0247.0
max specificity0.98860061.00.0
max absolute_mcc0.54735980.7593044130.0
max min_per_class_accuracy0.41359860.8758170159.0
max mean_per_class_accuracy0.37978440.8837638170.0
max tns0.9886006227.00.0
max fns0.9886006152.00.0
max fps0.0044334227.0379.0
max tps0.1833453153.0247.0
max tnr0.98860061.00.0
max fnr0.98860060.99346410.0
max fpr0.00443341.0379.0
max tpr0.18334531.0247.0
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Gains/Lift Table: Avg response rate: 40.26 %, avg score: 39.87 %
groupcumulative_data_fractionlower_thresholdliftcumulative_liftresponse_ratescorecumulative_response_ratecumulative_scorecapture_ratecumulative_capture_rategaincumulative_gainkolmogorov_smirnov
10.01052630.96952192.48366012.48366011.00.97776261.00.97776260.02614380.0261438148.3660131148.36601310.0261438
20.02105260.95783252.48366012.48366011.00.96471351.00.97123810.02614380.0522876148.3660131148.36601310.0522876
30.03157890.94855302.48366012.48366011.00.95292171.00.96513260.02614380.0784314148.3660131148.36601310.0784314
40.04210530.94045932.48366012.48366011.00.94513671.00.96013360.02614380.1045752148.3660131148.36601310.1045752
50.050.93504482.48366012.48366011.00.93811471.00.95665690.01960780.1241830148.3660131148.36601310.1241830
60.10.86268082.48366012.48366011.00.89851861.00.92758780.12418300.2483660148.3660131148.36601310.2483660
70.150.79048342.48366012.48366011.00.82513891.00.89343820.12418300.3725490148.3660131148.36601310.3725490
80.20.74085652.48366012.48366011.00.77045381.00.86269210.12418300.4967320148.3660131148.36601310.4967320
90.30.60433532.09150332.35294120.84210530.68232810.94736840.80257080.20915030.7058824109.1503268135.29411760.6794506
100.40.44003831.37254902.10784310.55263160.52872860.84868420.73411020.13725490.843137337.2549020110.78431370.7418157
110.50.31424681.04575161.89542480.42105260.37608560.76315790.66250530.10457520.94771244.575163489.54248370.7494745
120.60.21700780.32679741.63398690.13157890.25837620.65789470.59515040.03267970.9803922-67.320261463.39869280.6367798
130.70.14615940.19607841.42857140.07894740.18307590.57518800.53628260.01960781.0-80.392156942.85714290.5022026
140.80.09845960.01.250.00.12284090.50328950.48460240.01.0-100.025.00.3348018
150.90.06202970.01.11111110.00.07940720.44736840.43958070.01.0-100.011.11111110.1674009
161.00.00443340.01.00.00.03119410.40263160.39874210.01.0-100.00.00.0
\n", + "
\n", + "
\n", + "
ModelMetricsBinomialGLM: stackedensemble\n",
+       "** Reported on cross-validation data. **\n",
+       "\n",
+       "MSE: 0.1719258765773066\n",
+       "RMSE: 0.41463945371528094\n",
+       "LogLoss: 0.5191224869887493\n",
+       "AUC: 0.8129336903630763\n",
+       "AUCPR: 0.72372477339867\n",
+       "Gini: 0.6258673807261526\n",
+       "Null degrees of freedom: 379\n",
+       "Residual degrees of freedom: 373\n",
+       "Null deviance: 515.3570656721417\n",
+       "Residual deviance: 394.5330901114495\n",
+       "AIC: 408.5330901114495
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.4190514772189537
01ErrorRate
0175.052.00.2291 (52.0/227.0)
138.0115.00.2484 (38.0/153.0)
Total213.0167.00.2368 (90.0/380.0)
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Maximum Metrics: Maximum metrics at their respective thresholds
metricthresholdvalueidx
max f10.41905150.71875166.0
max f20.18458640.8033708277.0
max f0point50.44625970.7106274156.0
max accuracy0.44625970.7684211156.0
max precision0.95256491.00.0
max recall0.07392421.0358.0
max specificity0.95256491.00.0
max absolute_mcc0.44625970.5207521156.0
max min_per_class_accuracy0.40679870.7577093170.0
max mean_per_class_accuracy0.44625970.7614235156.0
max tns0.9525649227.00.0
max fns0.9525649152.00.0
max fps0.0048130227.0379.0
max tps0.0739242153.0358.0
max tnr0.95256491.00.0
max fnr0.95256490.99346410.0
max fpr0.00481301.0379.0
max tpr0.07392421.0358.0
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Gains/Lift Table: Avg response rate: 40.26 %, avg score: 40.24 %
groupcumulative_data_fractionlower_thresholdliftcumulative_liftresponse_ratescorecumulative_response_ratecumulative_scorecapture_ratecumulative_capture_rategaincumulative_gainkolmogorov_smirnov
10.01052630.92123472.48366012.48366011.00.94380151.00.94380150.02614380.0261438148.3660131148.36601310.0261438
20.02105260.90416381.86274512.17320260.750.91283210.8750.92831680.01960780.045751686.2745098117.32026140.0413463
30.03157890.88193901.86274512.06971680.750.89250680.83333330.91638010.01960780.065359586.2745098106.97167760.0565489
40.04210530.86561301.86274512.01797390.750.87314130.81250.90557040.01960780.084967386.2745098101.79738560.0717515
50.050.85809832.48366012.09150331.00.86144130.84210530.89860270.01960780.1045752148.3660131109.15032680.0913593
60.10.81188371.69934641.89542480.68421050.83856580.76315790.86858420.08496730.189542569.934640589.54248370.1498949
70.150.73609182.22222222.00435730.89473680.77578280.80701750.83765040.11111110.3006536122.2222222100.43572980.2521954
80.20.68437801.56862751.89542480.63157890.70844080.76315790.80534800.07843140.379085056.862745189.54248370.2997898
90.30.56697411.76470591.85185190.71052630.60793710.74561400.73954440.17647060.555555676.470588285.18518520.4278023
100.40.46215511.37254901.73202610.55263160.51734560.69736840.68399470.13725490.692810537.254902073.20261440.4901673
110.50.34858481.04575161.59477120.42105260.40290520.64210530.62777680.10457520.79738564.575163459.47712420.4978261
120.60.25669120.71895421.44880170.28947370.29680330.58333330.57261450.07189540.8692810-28.104575244.88017430.4507788
130.70.19526280.45751631.30718950.18421050.22818940.52631580.52341090.04575160.9150327-54.248366030.71895420.3599666
140.80.14867200.45751631.20098040.18421050.17221260.48355260.47951110.04575160.9607843-54.248366020.09803920.2691544
150.90.09982200.26143791.09658680.10526320.12281690.44152050.43987840.02614380.9869281-73.85620929.65867830.1455184
161.00.00481300.13071901.00.05263160.06502250.40263160.40239280.01307191.0-86.92810460.00.0
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Cross-Validation Metrics Summary:
meansdcv_1_validcv_2_validcv_3_validcv_4_validcv_5_valid
accuracy0.76657440.08648450.82857140.61904760.77027030.78571430.8292683
aic91.6785310.63271686.341109.3213290.18603581.1841391.36013
auc0.82046540.04938900.86240790.73978360.81009610.83566430.854375
err0.23342560.08648450.17142860.38095240.22972970.21428570.1707317
err_count18.08.03118912.032.017.015.014.0
f0point50.70232480.10657780.82901560.55147060.6645570.69277110.7738095
f10.74962410.07238530.84210530.65217390.71186440.75409840.7878788
f20.80994360.03348150.85561500.79787240.76642330.82733820.8024691
lift_top_group1.95433991.13881351.89189182.6250.02.69230772.5625
loglikelihood0.00.00.00.00.00.00.0
------------------------
mean_per_class_error0.21645990.06091710.17362820.31971150.22115380.19405590.17375
mse0.16934570.01894000.16903550.20006400.16987420.15636570.1513893
null_deviance103.071418.119151103.67581111.9120597.2038392.78761109.77776
pr_auc0.72394600.12113460.87891810.58561570.61814110.78218500.7548701
precision0.6757450.12415210.82051280.50.63636360.65714290.7647059
r20.28439110.08411370.32164290.15165170.25462250.33025170.3637866
recall0.86143450.05389320.86486490.93750.80769230.88461540.8125
residual_deviance78.0785311.46723572.34197.3213276.18603567.1841377.36013
rmse0.41102020.02258600.41113930.44728520.41215800.39543110.3890877
specificity0.70564570.16362570.78787880.42307690.750.72727280.84
\n", + "
\n", + "
[24 rows x 8 columns]
\n",
+       "\n",
+       "[tips]\n",
+       "Use `model.explain()` to inspect the model.\n",
+       "--\n",
+       "Use `h2o.display.toggle_user_tips()` to switch on/off this section.
" + ], + "text/plain": [ + "Model Details\n", + "=============\n", + "H2OStackedEnsembleEstimator : Stacked Ensemble\n", + "Model Key: StackedEnsemble_BestOfFamily_5_AutoML_3_20240409_111636\n", + "\n", + "\n", + "Model Summary for Stacked Ensemble: \n", + "key value\n", + "----------------------------------------- ----------------\n", + "Stacking strategy cross_validation\n", + "Number of base models (used / total) 6/6\n", + "# GBM base models (used / total) 1/1\n", + "# XGBoost base models (used / total) 1/1\n", + "# GLM base models (used / total) 1/1\n", + "# DRF base models (used / total) 2/2\n", + "# DeepLearning base models (used / total) 1/1\n", + "Metalearner algorithm GLM\n", + "Metalearner fold assignment scheme Random\n", + "Metalearner nfolds 5\n", + "Metalearner fold_column\n", + "Custom metalearner hyperparameters None\n", + "\n", + "ModelMetricsBinomialGLM: stackedensemble\n", + "** Reported on train data. **\n", + "\n", + "MSE: 0.09184265410860146\n", + "RMSE: 0.3030555297443052\n", + "LogLoss: 0.31372343953161186\n", + "AUC: 0.9598917393682876\n", + "AUCPR: 0.9475549649186908\n", + "Gini: 0.9197834787365753\n", + "Null degrees of freedom: 379\n", + "Residual degrees of freedom: 373\n", + "Null deviance: 512.2888401848891\n", + "Residual deviance: 238.42981404402497\n", + "AIC: 252.42981404402497\n", + "\n", + "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.379784359056072\n", + " 0 1 Error Rate\n", + "----- --- --- ------- ------------\n", + "0 195 32 0.141 (32.0/227.0)\n", + "1 14 139 0.0915 (14.0/153.0)\n", + "Total 209 171 0.1211 (46.0/380.0)\n", + "\n", + "Maximum Metrics: Maximum metrics at their respective thresholds\n", + "metric threshold value idx\n", + "--------------------------- ----------- -------- -----\n", + "max f1 0.379784 0.858025 170\n", + "max f2 0.272385 0.911873 204\n", + "max f0point5 0.590251 0.896 117\n", + "max accuracy 0.54736 0.884211 130\n", + "max precision 0.988601 1 0\n", + "max recall 0.183345 1 247\n", + "max specificity 0.988601 1 0\n", + "max absolute_mcc 0.54736 0.759304 130\n", + "max min_per_class_accuracy 0.413599 0.875817 159\n", + "max mean_per_class_accuracy 0.379784 0.883764 170\n", + "max tns 0.988601 227 0\n", + "max fns 0.988601 152 0\n", + "max fps 0.00443344 227 379\n", + "max tps 0.183345 153 247\n", + "max tnr 0.988601 1 0\n", + "max fnr 0.988601 0.993464 0\n", + "max fpr 0.00443344 1 379\n", + "max tpr 0.183345 1 247\n", + "\n", + "Gains/Lift Table: Avg response rate: 40.26 %, avg score: 39.87 %\n", + "group cumulative_data_fraction lower_threshold lift cumulative_lift response_rate score cumulative_response_rate cumulative_score capture_rate cumulative_capture_rate gain cumulative_gain kolmogorov_smirnov\n", + "------- -------------------------- ----------------- -------- ----------------- --------------- --------- -------------------------- ------------------ -------------- ------------------------- -------- ----------------- --------------------\n", + "1 0.0105263 0.969522 2.48366 2.48366 1 0.977763 1 0.977763 0.0261438 0.0261438 148.366 148.366 0.0261438\n", + "2 0.0210526 0.957833 2.48366 2.48366 1 0.964713 1 0.971238 0.0261438 0.0522876 148.366 148.366 0.0522876\n", + "3 0.0315789 0.948553 2.48366 2.48366 1 0.952922 1 0.965133 0.0261438 0.0784314 148.366 148.366 0.0784314\n", + "4 0.0421053 0.940459 2.48366 2.48366 1 0.945137 1 0.960134 0.0261438 0.104575 148.366 148.366 0.104575\n", + "5 0.05 0.935045 2.48366 2.48366 1 0.938115 1 0.956657 0.0196078 0.124183 148.366 148.366 0.124183\n", + "6 0.1 0.862681 2.48366 2.48366 1 0.898519 1 0.927588 0.124183 0.248366 148.366 148.366 0.248366\n", + "7 0.15 0.790483 2.48366 2.48366 1 0.825139 1 0.893438 0.124183 0.372549 148.366 148.366 0.372549\n", + "8 0.2 0.740856 2.48366 2.48366 1 0.770454 1 0.862692 0.124183 0.496732 148.366 148.366 0.496732\n", + "9 0.3 0.604335 2.0915 2.35294 0.842105 0.682328 0.947368 0.802571 0.20915 0.705882 109.15 135.294 0.679451\n", + "10 0.4 0.440038 1.37255 2.10784 0.552632 0.528729 0.848684 0.73411 0.137255 0.843137 37.2549 110.784 0.741816\n", + "11 0.5 0.314247 1.04575 1.89542 0.421053 0.376086 0.763158 0.662505 0.104575 0.947712 4.57516 89.5425 0.749475\n", + "12 0.6 0.217008 0.326797 1.63399 0.131579 0.258376 0.657895 0.59515 0.0326797 0.980392 -67.3203 63.3987 0.63678\n", + "13 0.7 0.146159 0.196078 1.42857 0.0789474 0.183076 0.575188 0.536283 0.0196078 1 -80.3922 42.8571 0.502203\n", + "14 0.8 0.0984596 0 1.25 0 0.122841 0.503289 0.484602 0 1 -100 25 0.334802\n", + "15 0.9 0.0620297 0 1.11111 0 0.0794072 0.447368 0.439581 0 1 -100 11.1111 0.167401\n", + "16 1 0.00443344 0 1 0 0.0311941 0.402632 0.398742 0 1 -100 0 0\n", + "\n", + "ModelMetricsBinomialGLM: stackedensemble\n", + "** Reported on cross-validation data. **\n", + "\n", + "MSE: 0.1719258765773066\n", + "RMSE: 0.41463945371528094\n", + "LogLoss: 0.5191224869887493\n", + "AUC: 0.8129336903630763\n", + "AUCPR: 0.72372477339867\n", + "Gini: 0.6258673807261526\n", + "Null degrees of freedom: 379\n", + "Residual degrees of freedom: 373\n", + "Null deviance: 515.3570656721417\n", + "Residual deviance: 394.5330901114495\n", + "AIC: 408.5330901114495\n", + "\n", + "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.4190514772189537\n", + " 0 1 Error Rate\n", + "----- --- --- ------- ------------\n", + "0 175 52 0.2291 (52.0/227.0)\n", + "1 38 115 0.2484 (38.0/153.0)\n", + "Total 213 167 0.2368 (90.0/380.0)\n", + "\n", + "Maximum Metrics: Maximum metrics at their respective thresholds\n", + "metric threshold value idx\n", + "--------------------------- ----------- -------- -----\n", + "max f1 0.419051 0.71875 166\n", + "max f2 0.184586 0.803371 277\n", + "max f0point5 0.44626 0.710627 156\n", + "max accuracy 0.44626 0.768421 156\n", + "max precision 0.952565 1 0\n", + "max recall 0.0739242 1 358\n", + "max specificity 0.952565 1 0\n", + "max absolute_mcc 0.44626 0.520752 156\n", + "max min_per_class_accuracy 0.406799 0.757709 170\n", + "max mean_per_class_accuracy 0.44626 0.761424 156\n", + "max tns 0.952565 227 0\n", + "max fns 0.952565 152 0\n", + "max fps 0.00481303 227 379\n", + "max tps 0.0739242 153 358\n", + "max tnr 0.952565 1 0\n", + "max fnr 0.952565 0.993464 0\n", + "max fpr 0.00481303 1 379\n", + "max tpr 0.0739242 1 358\n", + "\n", + "Gains/Lift Table: Avg response rate: 40.26 %, avg score: 40.24 %\n", + "group cumulative_data_fraction lower_threshold lift cumulative_lift response_rate score cumulative_response_rate cumulative_score capture_rate cumulative_capture_rate gain cumulative_gain kolmogorov_smirnov\n", + "------- -------------------------- ----------------- -------- ----------------- --------------- --------- -------------------------- ------------------ -------------- ------------------------- -------- ----------------- --------------------\n", + "1 0.0105263 0.921235 2.48366 2.48366 1 0.943802 1 0.943802 0.0261438 0.0261438 148.366 148.366 0.0261438\n", + "2 0.0210526 0.904164 1.86275 2.1732 0.75 0.912832 0.875 0.928317 0.0196078 0.0457516 86.2745 117.32 0.0413463\n", + "3 0.0315789 0.881939 1.86275 2.06972 0.75 0.892507 0.833333 0.91638 0.0196078 0.0653595 86.2745 106.972 0.0565489\n", + "4 0.0421053 0.865613 1.86275 2.01797 0.75 0.873141 0.8125 0.90557 0.0196078 0.0849673 86.2745 101.797 0.0717515\n", + "5 0.05 0.858098 2.48366 2.0915 1 0.861441 0.842105 0.898603 0.0196078 0.104575 148.366 109.15 0.0913593\n", + "6 0.1 0.811884 1.69935 1.89542 0.684211 0.838566 0.763158 0.868584 0.0849673 0.189542 69.9346 89.5425 0.149895\n", + "7 0.15 0.736092 2.22222 2.00436 0.894737 0.775783 0.807018 0.83765 0.111111 0.300654 122.222 100.436 0.252195\n", + "8 0.2 0.684378 1.56863 1.89542 0.631579 0.708441 0.763158 0.805348 0.0784314 0.379085 56.8627 89.5425 0.29979\n", + "9 0.3 0.566974 1.76471 1.85185 0.710526 0.607937 0.745614 0.739544 0.176471 0.555556 76.4706 85.1852 0.427802\n", + "10 0.4 0.462155 1.37255 1.73203 0.552632 0.517346 0.697368 0.683995 0.137255 0.69281 37.2549 73.2026 0.490167\n", + "11 0.5 0.348585 1.04575 1.59477 0.421053 0.402905 0.642105 0.627777 0.104575 0.797386 4.57516 59.4771 0.497826\n", + "12 0.6 0.256691 0.718954 1.4488 0.289474 0.296803 0.583333 0.572615 0.0718954 0.869281 -28.1046 44.8802 0.450779\n", + "13 0.7 0.195263 0.457516 1.30719 0.184211 0.228189 0.526316 0.523411 0.0457516 0.915033 -54.2484 30.719 0.359967\n", + "14 0.8 0.148672 0.457516 1.20098 0.184211 0.172213 0.483553 0.479511 0.0457516 0.960784 -54.2484 20.098 0.269154\n", + "15 0.9 0.099822 0.261438 1.09659 0.105263 0.122817 0.44152 0.439878 0.0261438 0.986928 -73.8562 9.65868 0.145518\n", + "16 1 0.00481303 0.130719 1 0.0526316 0.0650225 0.402632 0.402393 0.0130719 1 -86.9281 0 0\n", + "\n", + "Cross-Validation Metrics Summary: \n", + " mean sd cv_1_valid cv_2_valid cv_3_valid cv_4_valid cv_5_valid\n", + "-------------------- ---------- ----------- ------------ ------------ ------------ ------------ ------------\n", + "accuracy 0.7665744 0.08648454 0.82857144 0.61904764 0.7702703 0.78571427 0.8292683\n", + "aic 91.67853 10.632716 86.341 109.32132 90.186035 81.18413 91.36013\n", + "auc 0.8204654 0.049389027 0.86240786 0.73978364 0.81009614 0.83566433 0.854375\n", + "err 0.23342562 0.08648454 0.17142858 0.3809524 0.22972973 0.21428572 0.17073171\n", + "err_count 18.0 8.031189 12.0 32.0 17.0 15.0 14.0\n", + "f0point5 0.70232475 0.10657778 0.82901555 0.5514706 0.664557 0.6927711 0.77380955\n", + "f1 0.74962413 0.07238532 0.84210527 0.65217394 0.7118644 0.75409836 0.7878788\n", + "f2 0.8099436 0.033481482 0.85561496 0.79787236 0.76642334 0.82733816 0.80246913\n", + "lift_top_group 1.9543399 1.1388135 1.8918918 2.625 0.0 2.6923077 2.5625\n", + "loglikelihood 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "--- --- --- --- --- --- --- ---\n", + "mean_per_class_error 0.2164599 0.060917083 0.17362817 0.31971154 0.22115384 0.19405594 0.17375\n", + "mse 0.16934574 0.018940048 0.16903552 0.20006399 0.16987419 0.15636574 0.15138927\n", + "null_deviance 103.07141 8.119151 103.67581 111.91205 97.20383 92.78761 109.77776\n", + "pr_auc 0.72394603 0.12113462 0.8789181 0.58561575 0.6181411 0.78218496 0.7548701\n", + "precision 0.675745 0.124152124 0.82051283 0.5 0.6363636 0.6571429 0.7647059\n", + "r2 0.28439108 0.08411367 0.32164288 0.15165173 0.25462252 0.33025166 0.36378658\n", + "recall 0.8614345 0.053893156 0.8648649 0.9375 0.8076923 0.88461536 0.8125\n", + "residual_deviance 78.07853 11.467235 72.341 97.32132 76.186035 67.18413 77.36013\n", + "rmse 0.41102025 0.02258599 0.41113928 0.44728515 0.41215798 0.39543107 0.38908774\n", + "specificity 0.7056457 0.16362572 0.7878788 0.42307693 0.75 0.72727275 0.84\n", + "[24 rows x 8 columns]\n", + "\n", + "\n", + "[tips]\n", + "Use `model.explain()` to inspect the model.\n", + "--\n", + "Use `h2o.display.toggle_user_tips()` to switch on/off this section." + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "aml_plugin_with_weight_and_group.train(y=target, training_frame=train)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -547,118 +4986,266 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "\n", + "
\n", "\n", - "\n", + "\n", "\n", "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", "\n", - "
model_id auc logloss aucpr mean_per_class_error rmse mse
model_id auc logloss aucpr mean_per_class_error rmse mse
GLM_1_AutoML_20200128_181828 0.808816 0.5237440.730139 0.2735450.4187590.175359
StackedEnsemble_BestOfFamily_AutoML_20200128_1818280.806858 0.5281420.715206 0.2468260.41849 0.175134
StackedEnsemble_AllModels_AutoML_20200128_181828 0.805563 0.5284720.709471 0.2500220.4183640.175028
XGBoost_3_AutoML_20200128_181828 0.801992 0.5343710.689133 0.2398580.4220960.178165
XGBoost_1_AutoML_20200128_181828 0.801877 0.5362840.673698 0.2245110.4228640.178814
XGBoost_2_AutoML_20200128_181828 0.794175 0.5444830.685078 0.2442630.4256220.181154
DRF_1_AutoML_20200128_181828 0.789165 0.5488110.686757 0.2866170.4267560.182121
GBM_2_AutoML_20200128_181828 0.787251 0.5529550.692255 0.2808590.4298910.184806
GBM_grid__1_AutoML_20200128_181828_model_1 0.785206 0.5529390.696239 0.2667930.4301760.185051
GBM_4_AutoML_20200128_181828 0.784602 0.5525720.684192 0.27248 0.4314630.18616
StackedEnsemble_BestOfFamily_1_AutoML_1_20240409_1113460.810745 0.5287040.707779 0.2358120.4179190.174657
GLM_1_AutoML_1_20240409_111346 0.808816 0.5237440.736683 0.2735450.4187590.175359
StackedEnsemble_AllModels_1_AutoML_1_20240409_111346 0.803115 0.5317480.712266 0.2445510.4196810.176132
XGBoost_2_AutoML_1_20240409_111346 0.796925 0.5461140.674287 0.2366620.4250420.180661
DRF_1_AutoML_1_20240409_111346 0.793139 0.5463910.694818 0.2707670.4257960.181302
XGBoost_1_AutoML_1_20240409_111346 0.790447 0.5461640.687742 0.2590480.4269980.182327
GBM_4_AutoML_1_20240409_111346 0.78751 0.5475970.683774 0.2586160.4302420.185108
GBM_3_AutoML_1_20240409_111346 0.786934 0.5469710.691114 0.2778790.4281290.183295
XGBoost_grid_1_AutoML_1_20240409_111346_model_3 0.786041 0.56919 0.698224 0.2640870.4355220.189679
GBM_1_AutoML_1_20240409_111346 0.784199 0.5549730.68929 0.2552760.43086 0.185641
XGBoost_3_AutoML_1_20240409_111346 0.78417 0.5691220.699191 0.2651520.4344230.188724
XRT_1_AutoML_1_20240409_111346 0.783651 0.5461690.706824 0.27844 0.4293790.184367
GBM_grid_1_AutoML_1_20240409_111346_model_2 0.783407 0.5486840.703781 0.2665770.4290010.184042
DeepLearning_grid_1_AutoML_1_20240409_111346_model_1 0.7825 0.6156130.68783 0.2739630.4404530.193999
GBM_2_AutoML_1_20240409_111346 0.781751 0.5566010.673937 0.2598970.4321340.18674
GBM_5_AutoML_1_20240409_111346 0.781319 0.5539670.700958 0.2955 0.4320960.186707
GBM_grid_1_AutoML_1_20240409_111346_model_1 0.780024 0.55801 0.697977 0.2796490.4351210.18933
XGBoost_grid_1_AutoML_1_20240409_111346_model_2 0.76508 0.6794810.700206 0.2999050.4650450.216267
DeepLearning_grid_2_AutoML_1_20240409_111346_model_1 0.762863 0.6272 0.682732 0.2990410.4496150.202154
XGBoost_grid_1_AutoML_1_20240409_111346_model_1 0.754168 0.7073560.693299 0.3172380.4654070.216604
DeepLearning_1_AutoML_1_20240409_111346 0.745904 0.60627 0.705214 0.2997470.4493660.20193
DeepLearning_grid_3_AutoML_1_20240409_111346_model_1 0.725519 0.6847580.649417 0.3367740.4691360.220089
" + "
[22 rows x 7 columns]
" + ], + "text/plain": [ + "model_id auc logloss aucpr mean_per_class_error rmse mse\n", + "------------------------------------------------------- -------- --------- -------- ---------------------- -------- --------\n", + "StackedEnsemble_BestOfFamily_1_AutoML_1_20240409_111346 0.810745 0.528704 0.707779 0.235812 0.417919 0.174657\n", + "GLM_1_AutoML_1_20240409_111346 0.808816 0.523744 0.736683 0.273545 0.418759 0.175359\n", + "StackedEnsemble_AllModels_1_AutoML_1_20240409_111346 0.803115 0.531748 0.712266 0.244551 0.419681 0.176132\n", + "XGBoost_2_AutoML_1_20240409_111346 0.796925 0.546114 0.674287 0.236662 0.425042 0.180661\n", + "DRF_1_AutoML_1_20240409_111346 0.793139 0.546391 0.694818 0.270767 0.425796 0.181302\n", + "XGBoost_1_AutoML_1_20240409_111346 0.790447 0.546164 0.687742 0.259048 0.426998 0.182327\n", + "GBM_4_AutoML_1_20240409_111346 0.78751 0.547597 0.683774 0.258616 0.430242 0.185108\n", + "GBM_3_AutoML_1_20240409_111346 0.786934 0.546971 0.691114 0.277879 0.428129 0.183295\n", + "XGBoost_grid_1_AutoML_1_20240409_111346_model_3 0.786041 0.56919 0.698224 0.264087 0.435522 0.189679\n", + "GBM_1_AutoML_1_20240409_111346 0.784199 0.554973 0.68929 0.255276 0.43086 0.185641\n", + "XGBoost_3_AutoML_1_20240409_111346 0.78417 0.569122 0.699191 0.265152 0.434423 0.188724\n", + "XRT_1_AutoML_1_20240409_111346 0.783651 0.546169 0.706824 0.27844 0.429379 0.184367\n", + "GBM_grid_1_AutoML_1_20240409_111346_model_2 0.783407 0.548684 0.703781 0.266577 0.429001 0.184042\n", + "DeepLearning_grid_1_AutoML_1_20240409_111346_model_1 0.7825 0.615613 0.68783 0.273963 0.440453 0.193999\n", + "GBM_2_AutoML_1_20240409_111346 0.781751 0.556601 0.673937 0.259897 0.432134 0.18674\n", + "GBM_5_AutoML_1_20240409_111346 0.781319 0.553967 0.700958 0.2955 0.432096 0.186707\n", + "GBM_grid_1_AutoML_1_20240409_111346_model_1 0.780024 0.55801 0.697977 0.279649 0.435121 0.18933\n", + "XGBoost_grid_1_AutoML_1_20240409_111346_model_2 0.76508 0.679481 0.700206 0.299905 0.465045 0.216267\n", + "DeepLearning_grid_2_AutoML_1_20240409_111346_model_1 0.762863 0.6272 0.682732 0.299041 0.449615 0.202154\n", + "XGBoost_grid_1_AutoML_1_20240409_111346_model_1 0.754168 0.707356 0.693299 0.317238 0.465407 0.216604\n", + "DeepLearning_1_AutoML_1_20240409_111346 0.745904 0.60627 0.705214 0.299747 0.449366 0.20193\n", + "DeepLearning_grid_3_AutoML_1_20240409_111346_model_1 0.725519 0.684758 0.649417 0.336774 0.469136 0.220089\n", + "[22 rows x 7 columns]\n" ] }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [] - }, - "execution_count": 13, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "aml.leaderboard" + "aml.leaderboard.head(30)" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "\n", + "
\n", "\n", - "\n", + "\n", "\n", "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", "\n", - "
model_id auc logloss aucpr mean_per_class_error rmse mse
model_id auc logloss aucpr mean_per_class_error rmse mse
GLM_1_AutoML_20200128_181849 0.808816 0.5237440.730139 0.2735450.4187590.175359
StackedEnsemble_BestOfFamily_AutoML_20200128_1818490.808183 0.5259230.714758 0.2522240.41704 0.173922
MyGLM_grid__AutoML_20200128_181849_model_1 0.806974 0.5262150.724679 0.2610350.4200710.17646
XGBoost_3_AutoML_20200128_181849 0.801992 0.5343710.689133 0.2398580.4220960.178165
StackedEnsemble_AllModels_AutoML_20200128_181849 0.801992 0.5376140.700143 0.2489560.4226970.178673
XGBoost_1_AutoML_20200128_181849 0.801877 0.5362840.673698 0.2245110.4228640.178814
XGBoost_grid__1_AutoML_20200128_181849_model_3 0.79871 0.5389290.68214 0.2298380.4231590.179064
XGBoost_grid__1_AutoML_20200128_181849_model_4 0.796637 0.5375560.703381 0.2632370.4240040.179779
XGBoost_2_AutoML_20200128_181849 0.794175 0.5444830.685078 0.2442630.4256220.181154
DRF_1_AutoML_20200128_181849 0.789165 0.5488110.686757 0.2866170.4267560.182121
GBM_2_AutoML_20200128_181849 0.787251 0.5529550.692255 0.2808590.4298910.184806
GBM_grid__1_AutoML_20200128_181849_model_1 0.785206 0.5529390.696239 0.2667930.4301760.185051
MyDRF_grid__AutoML_20200128_181849_model_2 0.784947 0.57803 0.66362 0.2505830.4417760.195166
GBM_4_AutoML_20200128_181849 0.784602 0.5525720.684192 0.27248 0.4314630.18616
XGBoost_grid__1_AutoML_20200128_181849_model_1 0.784343 0.5528530.695315 0.2810030.4305310.185357
MyDRF_grid__AutoML_20200128_181849_model_5 0.781607 0.5571990.650905 0.2505110.4335210.18794
MyDRF_grid__AutoML_20200128_181849_model_6 0.780484 0.5527540.690859 0.2743230.4307110.185512
GBM_5_AutoML_20200128_181849 0.780052 0.5558650.652507 0.2564140.4314540.186153
GBM_3_AutoML_20200128_181849 0.779563 0.5607740.696828 0.2807150.4344020.188705
MyDRF_grid__AutoML_20200128_181849_model_7 0.774452 0.5605550.670805 0.2797930.43473 0.18899
GBM_1_AutoML_20200128_181849 0.773718 0.5872870.676561 0.2960610.4420620.195419
MyDRF_grid__AutoML_20200128_181849_model_4 0.773286 0.5606430.692592 0.2652960.4334330.187864
XRT_1_AutoML_20200128_181849 0.770061 0.5624560.684192 0.2906630.4356480.189789
XGBoost_grid__1_AutoML_20200128_181849_model_2 0.766275 0.6123080.662335 0.2673550.4586950.210401
DeepLearning_1_AutoML_20200128_181849 0.757162 0.5951720.682784 0.3116960.4450110.198035
MyDRF_grid__AutoML_20200128_181849_model_1 0.754556 0.6051740.631186 0.2977020.4557870.207742
MyDRF_grid__AutoML_20200128_181849_model_3 0.726152 0.5930830.592718 0.2751 0.4494750.202028
StackedEnsemble_BestOfFamily_1_AutoML_2_20240409_1115540.816216 0.5177160.716022 0.23993 0.4137270.17117
StackedEnsemble_AllModels_1_AutoML_2_20240409_111554 0.815784 0.5164450.733739 0.2409230.4135950.17106
GLM_1_AutoML_2_20240409_111554 0.808816 0.5237440.736683 0.2735450.4187590.175359
DeepLearning_grid_1_AutoML_2_20240409_111554_model_1 0.805764 0.5762120.726422 0.2576950.43227 0.186857
GLM_grid_1_AutoML_2_20240409_111554_model_2 0.805448 0.5283660.72757 0.2712710.4210040.177245
GLM_grid_1_AutoML_2_20240409_111554_model_1 0.805188 0.5285850.72699 0.2643030.42119 0.177401
XGBoost_2_AutoML_2_20240409_111554 0.796925 0.5461140.674287 0.2366620.4250420.180661
DRF_1_AutoML_2_20240409_111554 0.793139 0.5463910.694818 0.2707670.4257960.181302
XGBoost_1_AutoML_2_20240409_111554 0.790447 0.5461640.687742 0.2590480.4269980.182327
GBM_4_AutoML_2_20240409_111554 0.78751 0.5475970.683774 0.2586160.4302420.185108
GBM_3_AutoML_2_20240409_111554 0.786934 0.5469710.691114 0.2778790.4281290.183295
DeepLearning_1_AutoML_2_20240409_111554 0.786876 0.5657790.72169 0.2598970.4318780.186518
DRF_grid_1_AutoML_2_20240409_111554_model_1 0.785768 0.5580290.666972 0.2633090.4345290.188815
GBM_1_AutoML_2_20240409_111554 0.784199 0.5549730.68929 0.2552760.43086 0.185641
XGBoost_3_AutoML_2_20240409_111554 0.78417 0.5691220.699191 0.2651520.4344230.188724
XRT_1_AutoML_2_20240409_111554 0.783651 0.5461690.706824 0.27844 0.4293790.184367
GBM_2_AutoML_2_20240409_111554 0.781751 0.5566010.673937 0.2598970.4321340.18674
GBM_5_AutoML_2_20240409_111554 0.781319 0.5539670.700958 0.2955 0.4320960.186707
GBM_grid_1_AutoML_2_20240409_111554_model_1 0.780024 0.55801 0.697977 0.2796490.4351210.18933
DRF_grid_1_AutoML_2_20240409_111554_model_2 0.767729 0.5771280.663684 0.3054480.4429330.196189
XGBoost_grid_1_AutoML_2_20240409_111554_model_2 0.76508 0.6794810.700206 0.2999050.4650450.216267
XGBoost_grid_1_AutoML_2_20240409_111554_model_1 0.754168 0.7073560.693299 0.3172380.4654070.216604
" + "
[22 rows x 7 columns]
" + ], + "text/plain": [ + "model_id auc logloss aucpr mean_per_class_error rmse mse\n", + "------------------------------------------------------- -------- --------- -------- ---------------------- -------- --------\n", + "StackedEnsemble_BestOfFamily_1_AutoML_2_20240409_111554 0.816216 0.517716 0.716022 0.23993 0.413727 0.17117\n", + "StackedEnsemble_AllModels_1_AutoML_2_20240409_111554 0.815784 0.516445 0.733739 0.240923 0.413595 0.17106\n", + "GLM_1_AutoML_2_20240409_111554 0.808816 0.523744 0.736683 0.273545 0.418759 0.175359\n", + "DeepLearning_grid_1_AutoML_2_20240409_111554_model_1 0.805764 0.576212 0.726422 0.257695 0.43227 0.186857\n", + "GLM_grid_1_AutoML_2_20240409_111554_model_2 0.805448 0.528366 0.72757 0.271271 0.421004 0.177245\n", + "GLM_grid_1_AutoML_2_20240409_111554_model_1 0.805188 0.528585 0.72699 0.264303 0.42119 0.177401\n", + "XGBoost_2_AutoML_2_20240409_111554 0.796925 0.546114 0.674287 0.236662 0.425042 0.180661\n", + "DRF_1_AutoML_2_20240409_111554 0.793139 0.546391 0.694818 0.270767 0.425796 0.181302\n", + "XGBoost_1_AutoML_2_20240409_111554 0.790447 0.546164 0.687742 0.259048 0.426998 0.182327\n", + "GBM_4_AutoML_2_20240409_111554 0.78751 0.547597 0.683774 0.258616 0.430242 0.185108\n", + "GBM_3_AutoML_2_20240409_111554 0.786934 0.546971 0.691114 0.277879 0.428129 0.183295\n", + "DeepLearning_1_AutoML_2_20240409_111554 0.786876 0.565779 0.72169 0.259897 0.431878 0.186518\n", + "DRF_grid_1_AutoML_2_20240409_111554_model_1 0.785768 0.558029 0.666972 0.263309 0.434529 0.188815\n", + "GBM_1_AutoML_2_20240409_111554 0.784199 0.554973 0.68929 0.255276 0.43086 0.185641\n", + "XGBoost_3_AutoML_2_20240409_111554 0.78417 0.569122 0.699191 0.265152 0.434423 0.188724\n", + "XRT_1_AutoML_2_20240409_111554 0.783651 0.546169 0.706824 0.27844 0.429379 0.184367\n", + "GBM_2_AutoML_2_20240409_111554 0.781751 0.556601 0.673937 0.259897 0.432134 0.18674\n", + "GBM_5_AutoML_2_20240409_111554 0.781319 0.553967 0.700958 0.2955 0.432096 0.186707\n", + "GBM_grid_1_AutoML_2_20240409_111554_model_1 0.780024 0.55801 0.697977 0.279649 0.435121 0.18933\n", + "DRF_grid_1_AutoML_2_20240409_111554_model_2 0.767729 0.577128 0.663684 0.305448 0.442933 0.196189\n", + "XGBoost_grid_1_AutoML_2_20240409_111554_model_2 0.76508 0.679481 0.700206 0.299905 0.465045 0.216267\n", + "XGBoost_grid_1_AutoML_2_20240409_111554_model_1 0.754168 0.707356 0.693299 0.317238 0.465407 0.216604\n", + "[22 rows x 7 columns]\n" ] }, + "execution_count": 16, "metadata": {}, - "output_type": "display_data" - }, + "output_type": "execute_result" + } + ], + "source": [ + "aml_plugin.leaderboard.head(30)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ { "data": { - "text/plain": [] + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
model_id auc logloss aucpr mean_per_class_error rmse mse
StackedEnsemble_BestOfFamily_5_AutoML_3_20240409_1116360.812934 0.5191220.723725 0.23872 0.4146390.171926
StackedEnsemble_BestOfFamily_2_AutoML_3_20240409_1116360.811062 0.5259530.710498 0.2400740.4163970.173386
GLM_1_AutoML_3_20240409_111636 0.808816 0.5237440.736683 0.2735450.4187590.175359
StackedEnsemble_AllModels_1_AutoML_3_20240409_111636 0.807665 0.5269370.715294 0.2346750.4173510.174181
StackedEnsemble_BestOfFamily_1_AutoML_3_20240409_1116360.807031 0.5286960.714458 0.23695 0.41878 0.175377
GLM_grid_1_AutoML_3_20240409_111636_model_2 0.805448 0.5283660.72757 0.2712710.4210040.177245
GLM_grid_1_AutoML_3_20240409_111636_model_1 0.805188 0.5285850.72699 0.2643030.42119 0.177401
StackedEnsemble_AllModels_5_AutoML_3_20240409_111636 0.804382 0.5292740.719191 0.2475310.4192180.175744
StackedEnsemble_AllModels_2_AutoML_3_20240409_111636 0.80349 0.53249 0.720249 0.2513750.4199720.176376
XRT_1_AutoML_3_20240409_111636 0.79717 0.5420080.700051 0.2635970.4260690.181535
XGBoost_2_AutoML_3_20240409_111636 0.795183 0.5365070.706128 0.2422760.4231950.179094
DRF_1_AutoML_3_20240409_111636 0.793139 0.5463910.694818 0.2707670.4257960.181302
XGBoost_1_AutoML_3_20240409_111636 0.792592 0.5502210.704123 0.2478910.4261990.181646
GBM_1_AutoML_3_20240409_111636 0.789957 0.5496220.695586 0.2663610.4289260.183978
GBM_2_AutoML_3_20240409_111636 0.787539 0.5499370.694678 0.2552040.4294920.184464
DRF_grid_1_AutoML_3_20240409_111636_model_1 0.785768 0.5580290.666972 0.2633090.4345290.188815
GBM_3_AutoML_3_20240409_111636 0.785437 0.5473450.686723 0.2732570.4300240.184921
StackedEnsemble_BestOfFamily_4_AutoML_3_20240409_1116360.78381 0.5869140.686619 0.2878980.44164 0.195046
DeepLearning_grid_1_AutoML_3_20240409_111636_model_1 0.783508 0.58695 0.685371 0.2749560.4358140.189934
GBM_4_AutoML_3_20240409_111636 0.783335 0.5544960.67887 0.2628780.43285 0.187359
GBM_5_AutoML_3_20240409_111636 0.781348 0.5552280.664622 0.2530740.43107 0.185821
GBM_grid_1_AutoML_3_20240409_111636_model_1 0.780024 0.55801 0.697977 0.2796490.4351210.18933
XGBoost_3_AutoML_3_20240409_111636 0.778037 0.5784640.70394 0.2807870.4375330.191435
DRF_grid_1_AutoML_3_20240409_111636_model_3 0.777576 0.5681750.681376 0.2331060.4368430.190832
DeepLearning_1_AutoML_3_20240409_111636 0.776482 0.5866750.717975 0.2796490.4367210.190725
DRF_grid_1_AutoML_3_20240409_111636_model_2 0.767729 0.5771280.663684 0.3054480.4429330.196189
DeepLearning_grid_2_AutoML_3_20240409_111636_model_1 0.757076 0.6397460.671857 0.2814920.4521570.204446
StackedEnsemble_AllModels_3_AutoML_3_20240409_111636 0.754312 0.78217 0.625279 0.2883160.4725710.223324
StackedEnsemble_BestOfFamily_3_AutoML_3_20240409_1116360.751749 0.7412710.622973 0.2738190.4626980.214089
StackedEnsemble_AllModels_4_AutoML_3_20240409_111636 0.742334 0.6561140.609994 0.2816360.4605480.212105
[30 rows x 7 columns]
" + ], + "text/plain": [ + "model_id auc logloss aucpr mean_per_class_error rmse mse\n", + "------------------------------------------------------- -------- --------- -------- ---------------------- -------- --------\n", + "StackedEnsemble_BestOfFamily_5_AutoML_3_20240409_111636 0.812934 0.519122 0.723725 0.23872 0.414639 0.171926\n", + "StackedEnsemble_BestOfFamily_2_AutoML_3_20240409_111636 0.811062 0.525953 0.710498 0.240074 0.416397 0.173386\n", + "GLM_1_AutoML_3_20240409_111636 0.808816 0.523744 0.736683 0.273545 0.418759 0.175359\n", + "StackedEnsemble_AllModels_1_AutoML_3_20240409_111636 0.807665 0.526937 0.715294 0.234675 0.417351 0.174181\n", + "StackedEnsemble_BestOfFamily_1_AutoML_3_20240409_111636 0.807031 0.528696 0.714458 0.23695 0.41878 0.175377\n", + "GLM_grid_1_AutoML_3_20240409_111636_model_2 0.805448 0.528366 0.72757 0.271271 0.421004 0.177245\n", + "GLM_grid_1_AutoML_3_20240409_111636_model_1 0.805188 0.528585 0.72699 0.264303 0.42119 0.177401\n", + "StackedEnsemble_AllModels_5_AutoML_3_20240409_111636 0.804382 0.529274 0.719191 0.247531 0.419218 0.175744\n", + "StackedEnsemble_AllModels_2_AutoML_3_20240409_111636 0.80349 0.53249 0.720249 0.251375 0.419972 0.176376\n", + "XRT_1_AutoML_3_20240409_111636 0.79717 0.542008 0.700051 0.263597 0.426069 0.181535\n", + "XGBoost_2_AutoML_3_20240409_111636 0.795183 0.536507 0.706128 0.242276 0.423195 0.179094\n", + "DRF_1_AutoML_3_20240409_111636 0.793139 0.546391 0.694818 0.270767 0.425796 0.181302\n", + "XGBoost_1_AutoML_3_20240409_111636 0.792592 0.550221 0.704123 0.247891 0.426199 0.181646\n", + "GBM_1_AutoML_3_20240409_111636 0.789957 0.549622 0.695586 0.266361 0.428926 0.183978\n", + "GBM_2_AutoML_3_20240409_111636 0.787539 0.549937 0.694678 0.255204 0.429492 0.184464\n", + "DRF_grid_1_AutoML_3_20240409_111636_model_1 0.785768 0.558029 0.666972 0.263309 0.434529 0.188815\n", + "GBM_3_AutoML_3_20240409_111636 0.785437 0.547345 0.686723 0.273257 0.430024 0.184921\n", + "StackedEnsemble_BestOfFamily_4_AutoML_3_20240409_111636 0.78381 0.586914 0.686619 0.287898 0.44164 0.195046\n", + "DeepLearning_grid_1_AutoML_3_20240409_111636_model_1 0.783508 0.58695 0.685371 0.274956 0.435814 0.189934\n", + "GBM_4_AutoML_3_20240409_111636 0.783335 0.554496 0.67887 0.262878 0.43285 0.187359\n", + "GBM_5_AutoML_3_20240409_111636 0.781348 0.555228 0.664622 0.253074 0.43107 0.185821\n", + "GBM_grid_1_AutoML_3_20240409_111636_model_1 0.780024 0.55801 0.697977 0.279649 0.435121 0.18933\n", + "XGBoost_3_AutoML_3_20240409_111636 0.778037 0.578464 0.70394 0.280787 0.437533 0.191435\n", + "DRF_grid_1_AutoML_3_20240409_111636_model_3 0.777576 0.568175 0.681376 0.233106 0.436843 0.190832\n", + "DeepLearning_1_AutoML_3_20240409_111636 0.776482 0.586675 0.717975 0.279649 0.436721 0.190725\n", + "DRF_grid_1_AutoML_3_20240409_111636_model_2 0.767729 0.577128 0.663684 0.305448 0.442933 0.196189\n", + "DeepLearning_grid_2_AutoML_3_20240409_111636_model_1 0.757076 0.639746 0.671857 0.281492 0.452157 0.204446\n", + "StackedEnsemble_AllModels_3_AutoML_3_20240409_111636 0.754312 0.78217 0.625279 0.288316 0.472571 0.223324\n", + "StackedEnsemble_BestOfFamily_3_AutoML_3_20240409_111636 0.751749 0.741271 0.622973 0.273819 0.462698 0.214089\n", + "StackedEnsemble_AllModels_4_AutoML_3_20240409_111636 0.742334 0.656114 0.609994 0.281636 0.460548 0.212105\n", + "[30 rows x 7 columns]\n" + ] }, - "execution_count": 14, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "aml_plugin.leaderboard.head(30)" + "aml_plugin_with_weight_and_group.leaderboard.head(30)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "H2O session _sid_88ad closed.\n" + ] + } + ], + "source": [ + "h2o.cluster().shutdown()" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -672,7 +5259,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.5" + "version": "3.10.0" }, "varInspector": { "cols": { diff --git a/tutorials/automl/java_plugin/src/my/automl/MyDRFStepsProvider.java b/tutorials/automl/java_plugin/src/my/automl/MyDRFStepsProvider.java index 6c76f79db..35805a31b 100644 --- a/tutorials/automl/java_plugin/src/my/automl/MyDRFStepsProvider.java +++ b/tutorials/automl/java_plugin/src/my/automl/MyDRFStepsProvider.java @@ -15,58 +15,61 @@ public class MyDRFStepsProvider implements ModelingStepsProvider { - public static class DRFSteps extends ModelingSteps { - - static abstract class DRFGridStep extends ModelingStep.GridStep { - - DRFGridStep(String id, int weight, AutoML autoML) { - super(Algo.DRF, id, weight, autoML); - } - - DRFParameters prepareModelParameters() { - DRFParameters drfParameters = new DRFParameters(); - drfParameters._sample_rate = 0.8; - drfParameters._col_sample_rate_per_tree = 0.8; - drfParameters._col_sample_rate_change_per_level = 0.9; - return drfParameters; - } - } - - private ModelingStep[] grids = new ModelingStep[] { - new DRFGridStep("grid_1", 10*DEFAULT_MODEL_TRAINING_WEIGHT, aml()) { - @Override - protected Job startJob() { - DRFParameters drfParameters = prepareModelParameters(); - - Map searchParams = new HashMap<>(); - searchParams.put("_ntrees", IntStream.rangeClosed(5, 1000).filter(i -> i % 50 == 0).boxed().toArray()); - searchParams.put("_nbins", IntStream.of(5, 10, 15, 20, 30).boxed().toArray()); - searchParams.put("_max_depth", IntStream.rangeClosed(3, 20).boxed().toArray()); - searchParams.put("_min_rows", IntStream.of(3, 5, 10, 20, 50, 80, 100).boxed().toArray()); - - return hyperparameterSearch(makeKey("MyDRF", false), drfParameters, searchParams); - } - }, - }; - - public DRFSteps(AutoML autoML) { - super(autoML); - } - - @Override - protected ModelingStep[] getGrids() { - return grids; - } + public static class DRFSteps extends ModelingSteps { + + static final String NAME = Algo.DRF.name(); + static abstract class DRFGridStep extends ModelingStep.GridStep { + + DRFGridStep(String id, AutoML autoML) { + super(NAME, Algo.DRF, id, autoML); + } + + public DRFParameters prepareModelParameters() { + DRFParameters drfParameters = new DRFParameters(); + drfParameters._sample_rate = 0.8; + drfParameters._col_sample_rate_per_tree = 0.8; + drfParameters._col_sample_rate_change_per_level = 0.9; + return drfParameters; + } + } + + private ModelingStep[] grids = new ModelingStep[]{ + new DRFGridStep("grid_1", aml()) { + @Override + public Map prepareSearchParameters() { + Map searchParams = new HashMap<>(); + searchParams.put("_ntrees", IntStream.rangeClosed(5, 1000).filter(i -> i % 50 == 0).boxed().toArray()); + searchParams.put("_nbins", IntStream.of(5, 10, 15, 20, 30).boxed().toArray()); + searchParams.put("_max_depth", IntStream.rangeClosed(3, 20).boxed().toArray()); + searchParams.put("_min_rows", IntStream.of(3, 5, 10, 20, 50, 80, 100).boxed().toArray()); + return searchParams; + } + }, + }; + + public DRFSteps(AutoML autoML) { + super(autoML); } @Override - public String getName() { - return "MyDRF"; + protected ModelingStep[] getGrids() { + return grids; } @Override - public DRFSteps newInstance(AutoML aml) { - return new DRFSteps(aml); + public String getProvider() { + return NAME; } + } + + @Override + public String getName() { + return "MyDRF"; + } + + @Override + public DRFSteps newInstance(AutoML aml) { + return new DRFSteps(aml); + } } diff --git a/tutorials/automl/java_plugin/src/my/automl/MyGLMStepsProvider.java b/tutorials/automl/java_plugin/src/my/automl/MyGLMStepsProvider.java index 01b974f2f..7dddc0e89 100644 --- a/tutorials/automl/java_plugin/src/my/automl/MyGLMStepsProvider.java +++ b/tutorials/automl/java_plugin/src/my/automl/MyGLMStepsProvider.java @@ -15,60 +15,73 @@ public class MyGLMStepsProvider implements ModelingStepsProvider { - public static class GLMSteps extends ModelingSteps { - - static abstract class GLMGridStep extends ModelingStep.GridStep { - - GLMGridStep(String id, int weight, AutoML autoML) { - super(Algo.GLM, id, weight, autoML); - } - - GLMParameters prepareModelParameters() { - GLMParameters glmParameters = new GLMParameters(); - glmParameters._lambda_search = true; - glmParameters._family = - aml().getResponseColumn().isBinary() && !(aml().getResponseColumn().isNumeric()) ? GLMParameters.Family.binomial - : aml().getResponseColumn().isCategorical() ? GLMParameters.Family.multinomial - : GLMParameters.Family.gaussian; // TODO: other continuous distributions! - return glmParameters; - } - } - - private ModelingStep[] grids = new ModelingStep[] { - new GLMGridStep("solvers", DEFAULT_MODEL_TRAINING_WEIGHT, aml()) { - @Override - protected Job startJob() { - GLMParameters glmParameters = prepareModelParameters(); - glmParameters._alpha = IntStream.rangeClosed(0, 10).asDoubleStream().map(i -> i / 10).toArray(); - glmParameters._missing_values_handling = GLMParameters.MissingValuesHandling.MeanImputation; - - Map searchParams = new HashMap<>(); - searchParams.put("_standardize", new Boolean[] { true, false }); - searchParams.put("_solver", GLMParameters.Solver.values()); - - return hyperparameterSearch(makeKey("MyGLM", false), glmParameters, searchParams); - } - }, - }; - - public GLMSteps(AutoML autoML) { - super(autoML); - } - - @Override - protected ModelingStep[] getGrids() { - return grids; - } + public static class GLMSteps extends ModelingSteps { + + static final String NAME = Algo.GLM.name(); + + static abstract class GLMGridStep extends ModelingStep.GridStep { + + GLMGridStep(String id,AutoML autoML) { + super(NAME, Algo.GLM, id, autoML); + } + + public GLMParameters prepareModelParameters() { + GLMParameters glmParameters = new GLMParameters(); + glmParameters._lambda_search = false; + glmParameters._family = + aml().getResponseColumn().isBinary() && !(aml().getResponseColumn().isNumeric()) ? GLMParameters.Family.binomial + : aml().getResponseColumn().isCategorical() ? GLMParameters.Family.multinomial + : GLMParameters.Family.gaussian; // TODO: other continuous distributions! + return glmParameters; + } + } + + private ModelingStep[] grids = new ModelingStep[]{ + new GLMGridStep("solvers", aml()) { + @Override + public Map prepareSearchParameters() { + GLMParameters glmParameters = prepareModelParameters(); + glmParameters._alpha = IntStream.rangeClosed(0, 10).asDoubleStream().map(i -> i / 10).toArray(); + glmParameters._missing_values_handling = GLMParameters.MissingValuesHandling.MeanImputation; + + Map searchParams = new HashMap<>(); + searchParams.put("_standardize", new Boolean[]{true, false}); + searchParams.put("_solver", new GLMParameters.Solver[]{ + GLMParameters.Solver.AUTO, + GLMParameters.Solver.IRLSM, + GLMParameters.Solver.L_BFGS, + GLMParameters.Solver.COORDINATE_DESCENT, + GLMParameters.Solver.COORDINATE_DESCENT_NAIVE + }); + return searchParams; + } + }, + }; + + public GLMSteps(AutoML autoML) { + super(autoML); } @Override - public String getName() { - return "MyGLM"; + protected ModelingStep[] getGrids() { + return grids; } @Override - public GLMSteps newInstance(AutoML aml) { - return new GLMSteps(aml); + public String getProvider() { + return NAME; } + + } + + @Override + public String getName() { + return "MyGLM"; + } + + @Override + public GLMSteps newInstance(AutoML aml) { + return new GLMSteps(aml); + } }