From d9e9e4179f53981638867e0b5511a8dc409b9e43 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 25 Jan 2020 16:13:26 +0200 Subject: [PATCH 001/103] CI Cleanup --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e7ad744..a4b874a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,8 +9,8 @@ jobs: strategy: max-parallel: 9 matrix: - python-version: [3.5, 3.6, 3.7] - os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [3.6, 3.7] + os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v1 From b3b4ef98633acc4325a793df7549a0382a9ea30a Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 25 Jan 2020 16:20:08 +0200 Subject: [PATCH 002/103] changed tensorflow version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4b874a9..37da0eb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: - name: Tests run: | export MPLBACKEND=agg - pip install tensorflow + pip install tensorflow==2.1-rc2 pip install coveralls coverage run --source=talos ./test_script.py - name: Coverage From 226eac203dc6a0b315c377e94e1b993745540a2a Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 25 Jan 2020 20:19:34 +0200 Subject: [PATCH 003/103] Update setup.py --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ce051120..32387d76 100755 --- a/setup.py +++ b/setup.py @@ -24,10 +24,12 @@ except ImportError: from distutils.core import setup -install_requires = ['wrangle', +install_requires = ['statsmodels>=0.11.0', + 'wrangle', 'numpy', 'pandas', - 'keras', + 'tensorflow<=1.5.0', + 'keras==2.3.0', 'astetik', 'sklearn', 'tqdm', From 79bd795ba1b682de0fba6d102224ba519e7986ad Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 25 Jan 2020 20:25:24 +0200 Subject: [PATCH 004/103] changed TensorFlow version to 1.15.0 for legacy --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37da0eb0..7c38bba6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: - name: Tests run: | export MPLBACKEND=agg - pip install tensorflow==2.1-rc2 + pip install tensorflow==1.15.0 pip install coveralls coverage run --source=talos ./test_script.py - name: Coverage From 38d4e4f5a0097b1b72c9bf4803940990e4079d50 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 25 Jan 2020 21:11:47 +0200 Subject: [PATCH 005/103] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 32387d76..351b71a0 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ 'wrangle', 'numpy', 'pandas', - 'tensorflow<=1.5.0', + 'tensorflow==1.4.0', 'keras==2.3.0', 'astetik', 'sklearn', From 2de8b68a4aab2e0b02f33f14527e295f8935deba Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 25 Jan 2020 21:12:30 +0200 Subject: [PATCH 006/103] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 351b71a0..649d122d 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ URL = 'http://autonom.io' LICENSE = 'MIT' DOWNLOAD_URL = 'https://github.com/autonomio/talos/' -VERSION = '0.6.4' +VERSION = '0.6.5' try: from setuptools import setup From 26f7465462be7da4fcc4386ceef5b5f144441c37 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 25 Jan 2020 21:25:35 +0200 Subject: [PATCH 007/103] Update setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 649d122d..db547a5f 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ URL = 'http://autonom.io' LICENSE = 'MIT' DOWNLOAD_URL = 'https://github.com/autonomio/talos/' -VERSION = '0.6.5' +VERSION = '0.6.6' try: from setuptools import setup @@ -28,7 +28,7 @@ 'wrangle', 'numpy', 'pandas', - 'tensorflow==1.4.0', + 'tensorflow==1.14.0', 'keras==2.3.0', 'astetik', 'sklearn', From 59d64ce6833bf44c1b8c3fd56a632c4338976de6 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 25 Jan 2020 21:53:35 +0200 Subject: [PATCH 008/103] locks wrangle version to >=0.6.7 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index db547a5f..762e949a 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ URL = 'http://autonom.io' LICENSE = 'MIT' DOWNLOAD_URL = 'https://github.com/autonomio/talos/' -VERSION = '0.6.6' +VERSION = '0.6.7' try: from setuptools import setup @@ -25,7 +25,7 @@ from distutils.core import setup install_requires = ['statsmodels>=0.11.0', - 'wrangle', + 'wrangle>=0.6.7', 'numpy', 'pandas', 'tensorflow==1.14.0', From 80f26fee528b14f9da2837d8872a518241377b03 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sun, 26 Jan 2020 10:31:56 +0200 Subject: [PATCH 009/103] fixed recover best model example notebook --- ...over Best Models from Experiment Log.ipynb | 288 ++---------------- 1 file changed, 17 insertions(+), 271 deletions(-) diff --git a/examples/Recover Best Models from Experiment Log.ipynb b/examples/Recover Best Models from Experiment Log.ipynb index 17bdcf7e..4bc05152 100644 --- a/examples/Recover Best Models from Experiment Log.ipynb +++ b/examples/Recover Best Models from Experiment Log.ipynb @@ -18,24 +18,10 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using TensorFlow backend.\n" - ] - } - ], + "outputs": [], "source": [ - "import sys\n", - "sys.path.insert(0, '/Users/mikko/Documents/GitHub/talos/')\n", - "\n", - "import sys\n", - "sys.path.insert(0, '/Users/mikko/Documents/GitHub/wrangle/')\n", - "\n", "import talos\n", "import wrangle\n", "from keras.models import Sequential\n", @@ -51,17 +37,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 10/10 [00:34<00:00, 3.28s/it]\n" - ] - } - ], + "outputs": [], "source": [ "# load the data\n", "x, y = talos.templates.datasets.iris()\n", @@ -103,7 +81,7 @@ " x_val=x_val,\n", " y_val=y_val,\n", " model=iris_model,\n", - " experiment_name='reactivate',\n", + " experiment_name='minimal_iris',\n", " params=p,\n", " round_limit=10)" ] @@ -117,50 +95,34 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "total 96\r\n", - "-rw-r--r-- 1 mikko staff 16K Sep 26 20:17 092619200831.csv\r\n", - "-rw-r--r-- 1 mikko staff 1.3K Sep 26 20:18 092619201824.csv\r\n", - "-rw-r--r-- 1 mikko staff 1.3K Sep 26 22:13 092619221236.csv\r\n", - "-rw-r--r-- 1 mikko staff 1.3K Sep 26 22:18 092619221803.csv\r\n", - "-rw-r--r-- 1 mikko staff 1.3K Sep 26 22:31 092619223042.csv\r\n", - "-rw-r--r-- 1 mikko staff 1.3K Sep 26 22:35 092619223459.csv\r\n", - "-rw-r--r-- 1 mikko staff 1.3K Sep 26 22:35 092619223524.csv\r\n", - "-rw-r--r-- 1 mikko staff 1.3K Sep 26 22:56 092619225556.csv\r\n", - "-rw-r--r-- 1 mikko staff 1.3K Sep 26 23:04 092619230425.csv\r\n" - ] - } - ], + "outputs": [], "source": [ "# get the name of the experiment log\n", - "!ls -lhtr reactivate" + "!ls -lhtr minimal_iris" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "In this case it will be the most recent one `092619223042.csv` so let's go ahead and recover the best models." + "What you want to do, is get the name of the `.csv` file you want to use, and use it as part of the input for `experiment_log` in the next step." ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from talos.utils.recover_best_model import recover_best_model\n", + "\n", "results, models = recover_best_model(x_train=x_train,\n", " y_train=y_train,\n", " x_val=x_val,\n", " y_val=y_val,\n", - " experiment_log='reactivate/092619221803.csv',\n", + " experiment_log='minimal_iris/012620102735.csv',\n", " input_model=iris_model,\n", " n_models=5,\n", " task='multi_label')" @@ -175,170 +137,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
round_epochsval_lossval_acclossaccactivationbatch_sizedropoutepochsfirst_neuronhidden_layerslossesoptimizershapescrossval_mean_f1score
0100.0287600.9555560.0314890.885714relu200.3101282logcoshAdambrick0.930236
9100.0289020.9333330.0294370.866667elu500.3101282logcoshAdambrick0.900427
2100.0232470.8888890.0201560.942857elu300.3101282logcoshAdambrick0.980606
4100.0447170.8666670.0487600.780952relu300.4101282logcoshAdambrick0.529231
8100.0477180.8666670.0528770.761905elu300.310641logcoshAdambrick0.860539
\n", - "
" - ], - "text/plain": [ - " round_epochs val_loss val_acc loss acc activation \\\n", - "0 10 0.028760 0.955556 0.031489 0.885714 relu \n", - "9 10 0.028902 0.933333 0.029437 0.866667 elu \n", - "2 10 0.023247 0.888889 0.020156 0.942857 elu \n", - "4 10 0.044717 0.866667 0.048760 0.780952 relu \n", - "8 10 0.047718 0.866667 0.052877 0.761905 elu \n", - "\n", - " batch_size dropout epochs first_neuron hidden_layers losses \\\n", - "0 20 0.3 10 128 2 logcosh \n", - "9 50 0.3 10 128 2 logcosh \n", - "2 30 0.3 10 128 2 logcosh \n", - "4 30 0.4 10 128 2 logcosh \n", - "8 30 0.3 10 64 1 logcosh \n", - "\n", - " optimizer shapes crossval_mean_f1score \n", - "0 Adam brick 0.930236 \n", - "9 Adam brick 0.900427 \n", - "2 Adam brick 0.980606 \n", - "4 Adam brick 0.529231 \n", - "8 Adam brick 0.860539 " - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "results" ] @@ -352,64 +153,9 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[4.0995691e-02, 7.6433158e-01, 1.9467270e-01],\n", - " [5.5384491e-02, 8.0346000e-01, 1.4115542e-01],\n", - " [1.4972138e-01, 7.5501865e-01, 9.5259979e-02],\n", - " [1.1326348e-01, 7.1604651e-01, 1.7069000e-01],\n", - " [3.7939314e-02, 6.4578909e-01, 3.1627160e-01],\n", - " [7.1953669e-02, 7.3772454e-01, 1.9032180e-01],\n", - " [1.9050457e-02, 5.2020442e-01, 4.6074522e-01],\n", - " [3.7893724e-02, 6.0179770e-01, 3.6030859e-01],\n", - " [9.7751850e-03, 3.3405378e-01, 6.5617102e-01],\n", - " [1.1282027e-02, 3.9736888e-01, 5.9134912e-01],\n", - " [3.4884610e-03, 2.8260693e-01, 7.1390456e-01],\n", - " [1.4066804e-02, 4.4939899e-01, 5.3653419e-01],\n", - " [9.8388308e-01, 1.5294192e-02, 8.2262064e-04],\n", - " [9.4405776e-03, 2.6316279e-01, 7.2739667e-01],\n", - " [5.3356937e-03, 2.5790238e-01, 7.3676193e-01],\n", - " [6.6629532e-03, 2.9248431e-01, 7.0085275e-01],\n", - " [1.0408297e-02, 2.8865287e-01, 7.0093888e-01],\n", - " [9.2165405e-03, 4.4728053e-01, 5.4350299e-01],\n", - " [6.1816044e-02, 7.1130496e-01, 2.2687899e-01],\n", - " [5.8197163e-02, 7.0744330e-01, 2.3435953e-01],\n", - " [9.7906607e-01, 1.9747239e-02, 1.1866244e-03],\n", - " [4.9119215e-02, 6.6068804e-01, 2.9019269e-01],\n", - " [9.7612959e-01, 2.2444952e-02, 1.4254292e-03],\n", - " [4.5029860e-02, 6.6044801e-01, 2.9452211e-01],\n", - " [9.9142039e-01, 8.2095861e-03, 3.7001175e-04],\n", - " [9.7515500e-01, 2.3289582e-02, 1.5554430e-03],\n", - " [5.3465478e-02, 6.7216325e-01, 2.7437130e-01],\n", - " [3.2891510e-03, 1.4500402e-01, 8.5170686e-01],\n", - " [4.0943369e-02, 7.4083865e-01, 2.1821795e-01],\n", - " [9.7216946e-01, 2.5830602e-02, 1.9999193e-03],\n", - " [5.3125862e-02, 7.0550537e-01, 2.4136871e-01],\n", - " [1.1228154e-01, 7.2705197e-01, 1.6066651e-01],\n", - " [2.1662652e-02, 5.3180271e-01, 4.4653463e-01],\n", - " [4.6057135e-02, 7.8316230e-01, 1.7078057e-01],\n", - " [3.8668580e-02, 5.8559459e-01, 3.7573683e-01],\n", - " [2.0084916e-02, 4.8307988e-01, 4.9683511e-01],\n", - " [2.2133207e-02, 5.7616937e-01, 4.0169743e-01],\n", - " [9.6933258e-01, 2.8047977e-02, 2.6194439e-03],\n", - " [6.2396564e-02, 6.9619435e-01, 2.4140903e-01],\n", - " [9.8845774e-01, 1.0924198e-02, 6.1809190e-04],\n", - " [1.1867868e-02, 2.8613701e-01, 7.0199513e-01],\n", - " [1.3917084e-02, 4.5086253e-01, 5.3522038e-01],\n", - " [3.7958041e-02, 7.3275220e-01, 2.2928977e-01],\n", - " [1.5825152e-02, 5.4144788e-01, 4.4272691e-01],\n", - " [9.7893941e-01, 1.9864958e-02, 1.1956602e-03]], dtype=float32)" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "models[0].predict(x_val)" ] @@ -431,7 +177,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.8" + "version": "3.6.9" } }, "nbformat": 4, From c16d2a32c783fcffa22c76330bdc70502063d19d Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sun, 26 Jan 2020 10:34:11 +0200 Subject: [PATCH 010/103] removed travis.yml --- .travis.yml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2a718cd6..00000000 --- a/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -dist: trusty -sudo: required -before_install: -language: python -notifications: - email: - recipients: - - mailme@mikkokotila.com - on_success: never - on_failure: always -python: - - '3.6' - -script: - - echo - -deploy: - provider: pypi - user: "mikkokotila" - password: - secure: "riqFFcVCp3d2U5o/ly66JoM6NvyplyFKfhl8EB4bpyZRv13u1pk59b62435aq49rYVOlGy7i8V4Z+y7GPx+gxTrNfz8SKKfQSRmgEhQNmGmV7lY0KKDh5e7ud9YvH3gpZnh9j3Ke9NLQtIUUfRcGe3w2kmHsyuLQ5W7AbppWPXvaYtyyVvcOJnSZmR6B+MsZF9wRBWJltQExRxSQWsh6Lh3j9Uy9gL3XkFq1jRAnHy1Phmvlm/0amZI+vjmtKICmbPVac5OdGRLDj4Ed7oLz1o2fxmLKJoMbIr4yG46UBt3VcarOUk/Vm1i02KKdFhpRLK8XyfXtnaZ61Tfez6oCd1LrDCgzwlZo7U+o+8IgzPTjMSNvA1XyhlSfMRwhgpIAvwojke2JAvBpUeud0Z8u7+HyqMKlojPvKNs+6JQYBV8nOUiPW3+QAyd/jjPb+mciNdB/fgwGNcVR+Z/OR9ZziR1XDlQ+nuDCGecIlf7tEOCLhBzJwl/HLkVAUiDwIT4Qj8UkFuGbCjvKH8mu2Tk2FZLhAFbN6jpVQLsBGwXwJoEou5yNY0toPQ0kARLuxAMyC2okGHPsJbmVk9cNKVZA6tIz37eK4zP+c+sjH0U1SYIzw+NXTcNUqW+vI9LVjgVkm0nJxIeixhI1i99u4C4cN78weyqTTXJ2VL5zJXCqePw=" - on: - branch: production From d7fa4cfcc1d22bf43096b35d4a43f73c0ee9cd7e Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Thu, 20 Feb 2020 17:54:32 +0200 Subject: [PATCH 011/103] Fixes the issue with wrong version marking --- talos/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/talos/__init__.py b/talos/__init__.py index fcb93e2d..c0c12582 100755 --- a/talos/__init__.py +++ b/talos/__init__.py @@ -34,4 +34,4 @@ del commands, scan, model, metrics, key del sub, keep_from_templates, template_sub, warnings -__version__ = "0.6.4" +__version__ = "0.6.7" From 7d5374c7e0bea5ce0893d497935401a628d96509 Mon Sep 17 00:00:00 2001 From: Abel Serrano Juste Date: Tue, 5 May 2020 12:09:27 +0200 Subject: [PATCH 012/103] Delete unexpected random character MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delete å character in the wild --- docs/Restore.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Restore.md b/docs/Restore.md index f991d72d..54a6d311 100644 --- a/docs/Restore.md +++ b/docs/Restore.md @@ -1,7 +1,7 @@ # Restore() The `Deploy()` .zip package can be read back into a copy of the original experiment assets with `Restore()`. -å + ```python from talos import Restore From 2eb8fc48604fb88e21152b655ac73f9701172413 Mon Sep 17 00:00:00 2001 From: Abel Serrano Juste Date: Wed, 6 May 2020 08:04:09 +0200 Subject: [PATCH 013/103] Added docs for task argument in evaluate() --- docs/Evaluate.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Evaluate.md b/docs/Evaluate.md index 70f69bd4..81ea851c 100644 --- a/docs/Evaluate.md +++ b/docs/Evaluate.md @@ -24,6 +24,7 @@ Parameter | Default | Description --------- | ------- | ----------- `x` | NA | the predictor data x `y` | NA | the prediction data y (truth) +`task`| NA | One of the following strings: 'binary', 'multi_class', 'multi_label', or 'continuous'. `model_id` | None | the model_id to be used `folds` | None | number of folds to be used for cross-validation `shuffle` | None | if data is shuffled before splitting From 898dc76270090dc6ace3b08df8f01adf437dbf05 Mon Sep 17 00:00:00 2001 From: Arturo Moncada-Torres <37126116+arturomoncadatorres@users.noreply.github.com> Date: Tue, 20 Oct 2020 14:35:49 +0200 Subject: [PATCH 014/103] Updated README.md Fixed link for GitHub issue tracker --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cd5add98..d383e78e 100755 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ If you use Talos for published work, please cite: [MIT License](https://github.com/autonomio/talos/blob/master/LICENSE) -[github issue tracker]: https://github.com/automio/talos/issues +[github issue tracker]: https://github.com/autonomio/talos/issues [docs]: https://autonomio.github.io/talos/ [wiki]: https://github.com/autonomio/talos/wiki [stack overflow]: https://stackoverflow.com/questions/tagged/talos From c56a9ea783314ac9de725d7b23171400654ac87e Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 9 Nov 2020 15:39:25 +0000 Subject: [PATCH 015/103] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6815b9c..66f35b1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: - name: Tests run: | export MPLBACKEND=agg - pip install tensorflow==1.15.0 + pip install tensorflow>=2.0 pip install coveralls coverage run --source=talos ./test-ci.py - name: Coverage From d4e724599a678de7026beca10f72bc8f7d4234a3 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 9 Nov 2020 15:49:07 +0000 Subject: [PATCH 016/103] Fix TF input format --- tests/commands/test_latest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/commands/test_latest.py b/tests/commands/test_latest.py index 7a68e2b8..792aae85 100644 --- a/tests/commands/test_latest.py +++ b/tests/commands/test_latest.py @@ -36,7 +36,7 @@ def iris_model(x_train, y_train, x_val, y_val, params): callbacks=[talos.utils.ExperimentLogCallback('test_latest', params)], batch_size=params['batch_size'], epochs=params['epochs'], - validation_data=[x_val, y_val], + validation_data=(x_val, y_val), verbose=0) return out, model From f80c56787660a532995cf6cbb31916a6da983c25 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 9 Nov 2020 15:58:30 +0000 Subject: [PATCH 017/103] fix TF inputs --- tests/commands/test_scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/commands/test_scan.py b/tests/commands/test_scan.py index f859601d..8ae91b02 100644 --- a/tests/commands/test_scan.py +++ b/tests/commands/test_scan.py @@ -42,7 +42,7 @@ def iris_model(x_train, y_train, x_val, y_val, params): out = model.fit(x_train, y_train, batch_size=25, epochs=params['epochs'], - validation_data=[x_val, y_val], + validation_data=(x_val, y_val), verbose=0) return out, model From 9003d53f7c4a4e160a5cf6fbb0d3e7efd3a6a2cd Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 9 Nov 2020 15:59:08 +0000 Subject: [PATCH 018/103] Update recover_best_model.py --- tests/commands/recover_best_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/commands/recover_best_model.py b/tests/commands/recover_best_model.py index 3bcbc198..0d9c99e0 100644 --- a/tests/commands/recover_best_model.py +++ b/tests/commands/recover_best_model.py @@ -31,7 +31,7 @@ def iris_model(x_train, y_train, x_val, y_val, params): out = model.fit(x_train, y_train, batch_size=params['batch_size'], epochs=params['epochs'], - validation_data=[x_val, y_val], + validation_data=(x_val, y_val), verbose=0) return out, model From b7077b6a998af2c788c4ec5147310ba6c2d7771d Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 9 Nov 2020 16:05:05 +0000 Subject: [PATCH 019/103] fix TF inputs --- talos/templates/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/talos/templates/models.py b/talos/templates/models.py index bd9c1c96..586ea93d 100644 --- a/talos/templates/models.py +++ b/talos/templates/models.py @@ -31,7 +31,7 @@ def breast_cancer(x_train, y_train, x_val, y_val, params): batch_size=params['batch_size'], epochs=params['epochs'], verbose=0, - validation_data=[x_val, y_val], + validation_data=(x_val, y_val), callbacks=[early_stopper(params['epochs'], mode='moderate', monitor='val_f1score')]) @@ -72,7 +72,7 @@ def cervical_cancer(x_train, y_train, x_val, y_val, params): batch_size=params['batch_size'], epochs=params['epochs'], verbose=0, - validation_data=[x_val, y_val], + validation_data=(x_val, y_val), callbacks=[early_stopper(params['epochs'], mode='moderate', monitor='val_f1score')]) @@ -107,7 +107,7 @@ def titanic(x_train, y_train, x_val, y_val, params): batch_size=params['batch_size'], epochs=2, verbose=0, - validation_data=[x_val, y_val]) + validation_data=(x_val, y_val)) return out, model @@ -146,7 +146,7 @@ def iris(x_train, y_train, x_val, y_val, params): batch_size=params['batch_size'], epochs=params['epochs'], verbose=0, - validation_data=[x_val, y_val], + validation_data=(x_val, y_val), callbacks=[early_stopper(params['epochs'], mode=[1, 1])]) return out, model From d2ab963e00a9a37e3b55c1cbf174cacb4522a815 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 9 Nov 2020 16:12:14 +0000 Subject: [PATCH 020/103] fix TF inputs --- talos/autom8/automodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/talos/autom8/automodel.py b/talos/autom8/automodel.py index 3371fc23..2419acff 100644 --- a/talos/autom8/automodel.py +++ b/talos/autom8/automodel.py @@ -120,7 +120,7 @@ def _create_input_model(self, x_train, y_train, x_val, y_val, params): epochs=params['epochs'], verbose=0, callbacks=[self.callback(self.experiment_name, params)], - validation_data=[x_val, y_val]) + validation_data=(x_val, y_val)) # pass the output to Talos return out, model From c64c9d351906002c29e97b052861b5f914bec3db Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 9 Nov 2020 16:21:11 +0000 Subject: [PATCH 021/103] disable analyze tests temp --- test-ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-ci.py b/test-ci.py index 0a49d7b1..b668586d 100755 --- a/test-ci.py +++ b/test-ci.py @@ -14,7 +14,7 @@ test_autom8() test_templates() - test_analyze(scan_object) + # test_analyze(scan_object) test_lr_normalizer() test_predict() From 56a100310e9d40097b18720f1046819b538a874b Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 9 Nov 2020 16:31:35 +0000 Subject: [PATCH 022/103] Updated to v1.0 --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d383e78e..9a61e314 100755 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@
-

Hyperparameter Optimization for Keras

+

Hyperparameter Optimization for TensorFlow (tf.keras) and PyTorch

@@ -31,7 +31,7 @@

-Talos radically changes the ordinary Keras workflow by fully automating hyperparameter tuning and model evaluation. Talos exposes Keras functionality entirely and there is no new syntax or templates to learn. +Talos radically changes the ordinary TensorFlow (tf.keras) and PyTorch workflow by fully automating hyperparameter tuning and model evaluation. Talos exposes TensorFlow (tf.keras) and PyTorch functionality entirely and there is no new syntax or templates to learn.

@@ -41,14 +41,14 @@ Talos radically changes the ordinary Keras workflow by fully automating TL;DR -Talos radically transforms ordinary Keras workflows without taking away any of Keras. +Talos radically transforms ordinary TensorFlow (tf.keras) and PyTorch workflows without taking away. - - works with ANY Keras model + - works with ANY TensorFlow (tf.keras) or PyTorch model - takes minutes to implement - no new syntax to learn - adds zero new overhead to your workflow -Talos is made for data scientists and data engineers that want to remain in **complete control of their Keras models**, but are tired of mindless parameter hopping and confusing optimization solutions that add complexity instead of reducing it. Within minutes, without learning any new syntax, Talos allows you to configure, perform, and evaluate hyperparameter optimization experiments that yield state-of-the-art results across a wide range of prediction tasks. Talos provides the **simplest and yet most powerful** available method for hyperparameter optimization with Keras. +Talos is made for data scientists and data engineers that want to remain in **complete control of their TensorFlow (tf.keras) and PyTorch models**, but are tired of mindless parameter hopping and confusing optimization solutions that add complexity instead of reducing it. Within minutes, without learning any new syntax, Talos allows you to configure, perform, and evaluate hyperparameter optimization experiments that yield state-of-the-art results across a wide range of prediction tasks. Talos provides the **simplest and yet most powerful** available method for hyperparameter optimization with TensorFlow (tf.keras) and PyTorch.

@@ -90,7 +90,7 @@ The *Simple* example below is more than enough for starting to use Talos with an [Field Report](https://towardsdatascience.com/hyperparameter-optimization-with-keras-b82e6364ca53) [~15 mins] -For more information on how Talos can help with your Keras workflow, visit the [User Manual](https://autonomio.github.io/talos/). +For more information on how Talos can help with your TensorFlow (tf.keras) and PyTorch workflow, visit the [User Manual](https://autonomio.github.io/talos/). You may also want to check out a visualization of the [Talos Hyperparameter Tuning workflow](https://github.com/autonomio/talos/wiki/Workflow). From 650a5b4b33301d8312078c0a2d2288124f959e5c Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 21 Nov 2020 12:07:24 +0000 Subject: [PATCH 023/103] Update bug-report.md --- .github/ISSUE_TEMPLATE/bug-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 528a4f1d..d1b65ef3 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -31,7 +31,7 @@ Thank you very much for reporting a bug on Talos. Before you do, please go throu - [ ] My bug report includes a `Scan()` command - [ ] My bug report question includes a link to a sample of the data -NOTE: If the data is sensitive and can't be shared, [create dummy data](https://scikit-learn.org/stable/modules/classes.html#samples-generator) that mimics it. +NOTE: If the data is sensitive and can't be shared, [create dummy data](https://scikit-learn.org/stable/modules/classes.html#samples-generator) that mimics it or provide a command for generating it. **A self-contained Jupyter Notebook, Google Colab, or similar is highly preferred and will speed up helping you with your issue.** From 8527b97cad3c7490d76a0f4f88869bbea3b13ba5 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 21 Nov 2020 12:10:47 +0000 Subject: [PATCH 024/103] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d383e78e..e0cf88e5 100755 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@

Talos • Key Features • - Examples • + Examples • Install • Support • Docs • @@ -74,7 +74,7 @@ Talos works on **Linux, Mac OSX**, and **Windows** systems and can be operated c

-### 📈 Examples +### :arrow_forward: Examples Get the below code [here](https://gist.github.com/mikkokotila/4c0d6298ff0a22dc561fb387a1b4b0bb). More examples further below. From 21452f07b281017c7035ac4a84a011b1b82b170e Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 21 Nov 2020 12:15:30 +0000 Subject: [PATCH 025/103] Update README.md --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index e0cf88e5..709016c8 100755 --- a/README.md +++ b/README.md @@ -19,14 +19,14 @@

- Talos • - Key Features • - Examples • - Install • - Support • + Talos • + Key Features • + Examples • + Install • + Support • Docs • Issues • - License • + License • Download

@@ -96,7 +96,7 @@ You may also want to check out a visualization of the [Talos Hyperparameter Tuni
-### 💾 Install +### :floppy_disk: Install Stable version: @@ -108,7 +108,7 @@ Daily development version:
-### 💬 How to get Support +### :speech_balloon: How to get Support | I want to... | Go to... | | -------------------------------- | ---------------------------------------------------------- | @@ -120,7 +120,7 @@ Daily development version:
-### 📢 Citations +### :loudspeaker: Citations If you use Talos for published work, please cite: @@ -128,7 +128,7 @@ If you use Talos for published work, please cite:
-### 📃 License +### :page_with_curl: License [MIT License](https://github.com/autonomio/talos/blob/master/LICENSE) From 7724bb50e2222c592cf35fe46556844d73c10da2 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 21 Nov 2020 16:34:56 +0000 Subject: [PATCH 026/103] Update README.md --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 9a61e314..7a6d721c 100755 --- a/README.md +++ b/README.md @@ -116,7 +116,6 @@ Daily development version: | **...report a bug** | [GitHub Issue Tracker] | | **...suggest a new feature** | [GitHub Issue Tracker] | | **...get support** | [Stack Overflow] · [Spectrum Chat] | -| **...have a discussion** | [Spectrum Chat] |
@@ -124,7 +123,7 @@ Daily development version: If you use Talos for published work, please cite: -`Autonomio Talos [Computer software]. (2019). Retrieved from http://github.com/autonomio/talos.` +`Autonomio Talos [Computer software]. (2020). Retrieved from http://github.com/autonomio/talos.`
@@ -136,4 +135,3 @@ If you use Talos for published work, please cite: [docs]: https://autonomio.github.io/talos/ [wiki]: https://github.com/autonomio/talos/wiki [stack overflow]: https://stackoverflow.com/questions/tagged/talos -[spectrum chat]: https://spectrum.chat/talos From bdd4fb327d20007f818e8bac8a475568e6a3ccb6 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 21 Nov 2020 16:35:26 +0000 Subject: [PATCH 027/103] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a6d721c..40a5751d 100755 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Daily development version: | **...troubleshoot** | [Docs] · [Wiki] · [GitHub Issue Tracker] | | **...report a bug** | [GitHub Issue Tracker] | | **...suggest a new feature** | [GitHub Issue Tracker] | -| **...get support** | [Stack Overflow] · [Spectrum Chat] | +| **...get support** | [Stack Overflow] |
From 3e1d400705ef042f314a88ed6c4ad7115bd3c93c Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 21 Nov 2020 16:37:12 +0000 Subject: [PATCH 028/103] Update README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 40a5751d..de9f91a7 100755 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@
-

Hyperparameter Optimization for TensorFlow (tf.keras) and PyTorch

+

Hyperparameter Optimization for Keras, TensorFlow (tf.keras) and PyTorch

@@ -31,7 +31,7 @@

-Talos radically changes the ordinary TensorFlow (tf.keras) and PyTorch workflow by fully automating hyperparameter tuning and model evaluation. Talos exposes TensorFlow (tf.keras) and PyTorch functionality entirely and there is no new syntax or templates to learn. +Talos radically changes the ordinary Keras, TensorFlow (tf.keras), and PyTorch workflow by fully automating hyperparameter tuning and model evaluation. Talos exposes Keras and TensorFlow (tf.keras) and PyTorch functionality entirely and there is no new syntax or templates to learn.

@@ -41,9 +41,9 @@ Talos radically changes the ordinary TensorFlow (tf.keras) and PyTorch workflow TL;DR -Talos radically transforms ordinary TensorFlow (tf.keras) and PyTorch workflows without taking away. +Talos radically transforms ordinary Keras, TensorFlow (tf.keras), and PyTorch workflows without taking away. - - works with ANY TensorFlow (tf.keras) or PyTorch model + - works with ANY Keras, TensorFlow (tf.keras) or PyTorch model - takes minutes to implement - no new syntax to learn - adds zero new overhead to your workflow @@ -90,7 +90,7 @@ The *Simple* example below is more than enough for starting to use Talos with an [Field Report](https://towardsdatascience.com/hyperparameter-optimization-with-keras-b82e6364ca53) [~15 mins] -For more information on how Talos can help with your TensorFlow (tf.keras) and PyTorch workflow, visit the [User Manual](https://autonomio.github.io/talos/). +For more information on how Talos can help with your Keras, TensorFlow (tf.keras) and PyTorch workflow, visit the [User Manual](https://autonomio.github.io/talos/). You may also want to check out a visualization of the [Talos Hyperparameter Tuning workflow](https://github.com/autonomio/talos/wiki/Workflow). From faee68a9c7b2acf8ca8b83975ff6dc9d01d100ad Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 13 Mar 2021 16:51:31 +0000 Subject: [PATCH 029/103] fixes recovery issue mentioned in #534 --- setup.py | 2 +- talos/__init__.py | 2 +- talos/utils/recover_best_model.py | 12 +++++++----- tests/commands/recover_best_model.py | 1 + 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index e2097eef..b692961e 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ URL = 'http://autonom.io' LICENSE = 'MIT' DOWNLOAD_URL = 'https://github.com/autonomio/talos/' -VERSION = '1.0' +VERSION = '1.0.1' try: diff --git a/talos/__init__.py b/talos/__init__.py index 7df1a719..4ceec546 100755 --- a/talos/__init__.py +++ b/talos/__init__.py @@ -34,4 +34,4 @@ del commands, scan, model, metrics, key del sub, keep_from_templates, template_sub, warnings -__version__ = "1.0.0" +__version__ = "1.0.1" diff --git a/talos/utils/recover_best_model.py b/talos/utils/recover_best_model.py index 151dd56a..bb6408c8 100644 --- a/talos/utils/recover_best_model.py +++ b/talos/utils/recover_best_model.py @@ -4,6 +4,7 @@ def recover_best_model(x_train, y_val, experiment_log, input_model, + metric, x_cross=None, y_cross=None, n_models=5, @@ -15,10 +16,11 @@ def recover_best_model(x_train, y_train | array | same as was used in the experiment x_val | array | same as was used in the experiment y_val | array | same as was used in the experiment - x_cross | array | data for the cross-validation or None for use x_val - y_cross | array | data for the cross-validation or None for use y_val experiment_log | str | path to the Talos experiment log input_model | function | model used in the experiment + metric | str | use this metric to pick evaluation candidates + x_cross | array | data for the cross-validation or None for use x_val + y_cross | array | data for the cross-validation or None for use y_val n_models | int | number of models to cross-validate task | str | binary, multi_class, multi_label or continuous @@ -48,8 +50,8 @@ def recover_best_model(x_train, for i in range(n_models): # get the params for the model and train it - params = df.sort_values('val_acc', ascending=False).drop('val_acc', 1).iloc[i].to_dict() - history, model = input_model(x_train, y_train, x_val, y_val, params) + params = df.sort_values(metric, ascending=False).drop(metric, 1).iloc[i].to_dict() + _history, model = input_model(x_train, y_train, x_val, y_val, params) # start kfold cross-validation out = [] @@ -83,7 +85,7 @@ def recover_best_model(x_train, results.append(np.mean(out)) models.append(model) - out = df.sort_values('val_acc', ascending=False).head(n_models) + out = df.sort_values(metric, ascending=False).head(n_models) out['crossval_mean_f1score'] = results return out, models diff --git a/tests/commands/recover_best_model.py b/tests/commands/recover_best_model.py index 0d9c99e0..fc4ffaf6 100644 --- a/tests/commands/recover_best_model.py +++ b/tests/commands/recover_best_model.py @@ -42,6 +42,7 @@ def iris_model(x_train, y_train, x_val, y_val, params): y_val=y, experiment_log=experiment_log, input_model=iris_model, + metric='acc', x_cross=x, y_cross=y, n_models=5, From ade653f87a3e1d5e8f8346cb7e3f81112891016f Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 25 May 2021 19:03:16 +0300 Subject: [PATCH 030/103] set heatmap mask to false --- talos/commands/analyze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/talos/commands/analyze.py b/talos/commands/analyze.py index e5ae1575..d6498ad4 100644 --- a/talos/commands/analyze.py +++ b/talos/commands/analyze.py @@ -108,7 +108,7 @@ def plot_corr(self, metric, exclude, color_grades=5): try: import astetik as ast cols = self._cols(metric, exclude) - return ast.corr(self.data[cols], color_grades=color_grades) + return ast.corr(self.data[cols], color_grades=color_grades, mask=False) except RuntimeError: print('Matplotlib Runtime Error. Plots will not work.') From 0230b86775390d803e0beae3e7c038277710f2af Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 25 May 2021 19:07:48 +0300 Subject: [PATCH 031/103] Update setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e2097eef..36cad3c1 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #! /usr/bin/env python # -# Copyright (C) 2020 Mikko Kotila +# Copyright (C) 2021 Mikko Kotila DESCRIPTION = "Talos Hyperparameter Tuning for Keras" LONG_DESCRIPTION = """\ @@ -18,7 +18,7 @@ URL = 'http://autonom.io' LICENSE = 'MIT' DOWNLOAD_URL = 'https://github.com/autonomio/talos/' -VERSION = '1.0' +VERSION = '1.0.1' try: From 3d0c4b8e4015fc36d6d24526cb87963ac3f16c6c Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 25 May 2021 19:08:14 +0300 Subject: [PATCH 032/103] change version --- talos/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/talos/__init__.py b/talos/__init__.py index 7df1a719..4ceec546 100755 --- a/talos/__init__.py +++ b/talos/__init__.py @@ -34,4 +34,4 @@ del commands, scan, model, metrics, key del sub, keep_from_templates, template_sub, warnings -__version__ = "1.0.0" +__version__ = "1.0.1" From 240ad054a0c855b45f74a281282c6cfb4e36646b Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 25 May 2021 19:22:29 +0300 Subject: [PATCH 033/103] fix keras import to tf.keras --- talos/templates/datasets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/talos/templates/datasets.py b/talos/templates/datasets.py index 4839f5ec..33a270cb 100755 --- a/talos/templates/datasets.py +++ b/talos/templates/datasets.py @@ -139,11 +139,11 @@ def mnist(): x_train, y_train, x_val, y_val''' - import keras + import tensorflow as tf import numpy as np # the data, split between train and test sets - (x_train, y_train), (x_val, y_val) = keras.datasets.mnist.load_data() + (x_train, y_train), (x_val, y_val) = tf.keras.datasets.mnist.load_data() # input image dimensions img_rows, img_cols = 28, 28 From 98cdf3edb619c95aa4ef832a2970240102428f9a Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 25 May 2021 19:28:40 +0300 Subject: [PATCH 034/103] fix import --- talos/templates/datasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/talos/templates/datasets.py b/talos/templates/datasets.py index 33a270cb..c8dd7a0a 100755 --- a/talos/templates/datasets.py +++ b/talos/templates/datasets.py @@ -148,7 +148,7 @@ def mnist(): # input image dimensions img_rows, img_cols = 28, 28 - if keras.backend.image_data_format() == 'channels_first': + if tf.keras.backend.image_data_format() == 'channels_first': x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) x_val = x_val.reshape(x_val.shape[0], 1, img_rows, img_cols) From d959a5918c4e64af61716afdd84eef0e385ded8a Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 25 May 2021 20:38:41 +0300 Subject: [PATCH 035/103] fix imports --- talos/templates/datasets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/talos/templates/datasets.py b/talos/templates/datasets.py index c8dd7a0a..2460eb86 100755 --- a/talos/templates/datasets.py +++ b/talos/templates/datasets.py @@ -167,8 +167,8 @@ def mnist(): classes = len(np.unique(y_train)) # convert class vectors to binary class matrices - y_train = keras.utils.to_categorical(y_train, classes) - y_val = keras.utils.to_categorical(y_val, classes) + y_train = tf.keras.utils.to_categorical(y_train, classes) + y_val = tf.keras.utils.to_categorical(y_val, classes) print("Use input_shape %s" % str(input_shape)) From 48168f0147369e5c895c8f695d655cebef4b2f7b Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 25 May 2021 21:04:07 +0300 Subject: [PATCH 036/103] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 36cad3c1..12f9d848 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ URL = 'http://autonom.io' LICENSE = 'MIT' DOWNLOAD_URL = 'https://github.com/autonomio/talos/' -VERSION = '1.0.1' +VERSION = '1.0.2' try: From ac1b372cfeaf617bc84bb637d78ca838f053ec37 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 25 May 2021 21:04:58 +0300 Subject: [PATCH 037/103] change version --- talos/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/talos/__init__.py b/talos/__init__.py index 4ceec546..bfa68bd9 100755 --- a/talos/__init__.py +++ b/talos/__init__.py @@ -34,4 +34,4 @@ del commands, scan, model, metrics, key del sub, keep_from_templates, template_sub, warnings -__version__ = "1.0.1" +__version__ = "1.0.2" From e91d1e66dd6bff57aea35f87c72b9895846d58b0 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 25 May 2021 21:05:45 +0300 Subject: [PATCH 038/103] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 12f9d848..36cad3c1 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ URL = 'http://autonom.io' LICENSE = 'MIT' DOWNLOAD_URL = 'https://github.com/autonomio/talos/' -VERSION = '1.0.2' +VERSION = '1.0.1' try: From 3e8bd31c0407d87273b07c60b237f1aa95bfeb50 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 25 May 2021 21:06:10 +0300 Subject: [PATCH 039/103] Update __init__.py --- talos/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/talos/__init__.py b/talos/__init__.py index bfa68bd9..4ceec546 100755 --- a/talos/__init__.py +++ b/talos/__init__.py @@ -34,4 +34,4 @@ del commands, scan, model, metrics, key del sub, keep_from_templates, template_sub, warnings -__version__ = "1.0.2" +__version__ = "1.0.1" From 9a42e99ec7b1a1d64b57c53770566c29200a441e Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 25 May 2021 21:07:01 +0300 Subject: [PATCH 040/103] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b692961e..f0726df6 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ URL = 'http://autonom.io' LICENSE = 'MIT' DOWNLOAD_URL = 'https://github.com/autonomio/talos/' -VERSION = '1.0.1' +VERSION = '1.0.2' try: From 8e61c15088317fbde658bf9222e273f25ab94fe1 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 25 May 2021 21:07:30 +0300 Subject: [PATCH 041/103] Update __init__.py --- talos/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/talos/__init__.py b/talos/__init__.py index 4ceec546..bfa68bd9 100755 --- a/talos/__init__.py +++ b/talos/__init__.py @@ -34,4 +34,4 @@ del commands, scan, model, metrics, key del sub, keep_from_templates, template_sub, warnings -__version__ = "1.0.1" +__version__ = "1.0.2" From 80ea4274554dc7fc0e7402018dd46971b23be59b Mon Sep 17 00:00:00 2001 From: Abhijith Neil Abraham Date: Fri, 28 Jan 2022 14:15:57 +0530 Subject: [PATCH 042/103] Update keras_metrics.py --- talos/metrics/keras_metrics.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/talos/metrics/keras_metrics.py b/talos/metrics/keras_metrics.py index d81f6ddf..4872af7d 100644 --- a/talos/metrics/keras_metrics.py +++ b/talos/metrics/keras_metrics.py @@ -1,25 +1,25 @@ def mae(y_true, y_pred): - from keras import backend as K + from tensorflow.keras import backend as K return K.mean(K.abs(y_pred - y_true), axis=-1) def mse(y_true, y_pred): - from keras import backend as K + from tensorflow.keras import backend as K return K.mean(K.square(y_pred - y_true), axis=-1) def rmae(y_true, y_pred): - from keras import backend as K + from tensorflow.keras import backend as K return K.sqrt(K.mean(K.abs(y_pred - y_true), axis=-1)) def rmse(y_true, y_pred): - from keras import backend as K + from tensorflow.keras import backend as K return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) def mape(y_true, y_pred): - from keras import backend as K + from tensorflow.keras import backend as K diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true), K.epsilon(), None)) @@ -27,14 +27,14 @@ def mape(y_true, y_pred): def msle(y_true, y_pred): - from keras import backend as K + from tensorflow.keras import backend as K first_log = K.log(K.clip(y_pred, K.epsilon(), None) + 1.) second_log = K.log(K.clip(y_true, K.epsilon(), None) + 1.) return K.mean(K.square(first_log - second_log), axis=-1) def rmsle(y_true, y_pred): - from keras import backend as K + from tensorflow.keras import backend as K first_log = K.log(K.clip(y_pred, K.epsilon(), None) + 1.) second_log = K.log(K.clip(y_true, K.epsilon(), None) + 1.) return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1)) @@ -42,7 +42,7 @@ def rmsle(y_true, y_pred): def matthews(y_true, y_pred): - from keras import backend as K + from tensorflow.keras import backend as K y_pred_pos = K.round(K.clip(y_pred, 0, 1)) y_pred_neg = 1 - y_pred_pos @@ -63,7 +63,7 @@ def matthews(y_true, y_pred): def precision(y_true, y_pred): - from keras import backend as K + from tensorflow.keras import backend as K true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1))) precision = true_positives / (predicted_positives + K.epsilon()) @@ -72,7 +72,7 @@ def precision(y_true, y_pred): def recall(y_true, y_pred): - from keras import backend as K + from tensorflow.keras import backend as K true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) recall = true_positives / (possible_positives + K.epsilon()) @@ -81,7 +81,7 @@ def recall(y_true, y_pred): def fbeta(y_true, y_pred, beta=1): - from keras import backend as K + from tensorflow.keras import backend as K if beta < 0: raise ValueError('The lowest choosable beta is zero (only precision).') From 6b8ac90dee94f3da2ca404aa4b1b9dec4b316f52 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Fri, 28 Jan 2022 12:32:11 +0200 Subject: [PATCH 043/103] updated CI with new python versions --- .github/workflows/ci.yml | 2 +- .github/workflows/greetings.yml | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) delete mode 100644 .github/workflows/greetings.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66f35b1f..88d97d38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: strategy: max-parallel: 9 matrix: - python-version: [3.6, 3.7] + python-version: [3.7, 3.8] os: [ubuntu-latest, macos-latest] steps: diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml deleted file mode 100644 index cb1f5edb..00000000 --- a/.github/workflows/greetings.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Greetings - -on: [pull_request] - -jobs: - greeting: - runs-on: ubuntu-latest - steps: - - uses: actions/first-interaction@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - issue-message: 'Welcome to Talos community! Thanks so much for creating your first issue :)' - pr-message: 'Thanks so much for creating your first PR, the Talos community thanks you :)' From a8fc649cc771490b9d5293efb8b3a7384e250c38 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Fri, 28 Jan 2022 14:53:12 +0200 Subject: [PATCH 044/103] fixed predict_classes to predict --- talos/autom8/autopredict.py | 5 ++++- test-ci.py | 20 ++++++++++---------- test-local.sh | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/talos/autom8/autopredict.py b/talos/autom8/autopredict.py index 1591e613..c20dfbef 100644 --- a/talos/autom8/autopredict.py +++ b/talos/autom8/autopredict.py @@ -77,6 +77,8 @@ def AutoPredict(scan_object, ''' + import numpy as np + # evaluate and add the evaluation scores scan_object.evaluate_models(x_val, y_val, @@ -94,7 +96,8 @@ def AutoPredict(scan_object, scan_object.preds_probabilities = scan_object.preds_model.predict(x_pred) # make (class) predictiosn with the model - scan_object.preds_classes = scan_object.preds_model.predict_classes(x_pred) + preds = scan_object.preds_model.predict(x_pred) + scan_object.preds_classes = np.argmax(preds, axis=1) # get the hyperparameter for the model scan_object.preds_parameters = scan_object.data.sort_values('eval_f1score_mean', diff --git a/test-ci.py b/test-ci.py index b668586d..64585d87 100755 --- a/test-ci.py +++ b/test-ci.py @@ -4,21 +4,21 @@ from tests.commands import * - test_latest() + #test_latest() - scan_object = test_scan() - recover_best_model() + #scan_object = test_scan() + #recover_best_model() - test_random_methods() + #test_random_methods() - test_autom8() - test_templates() + #test_autom8() + #test_templates() # test_analyze(scan_object) - test_lr_normalizer() - test_predict() - test_reducers() - test_rest(scan_object) + #test_lr_normalizer() + #test_predict() + #test_reducers() + #test_rest(scan_object) print("\n All tests successfully completed :) Good work. \n ") diff --git a/test-local.sh b/test-local.sh index 6567e70a..ae1f083f 100755 --- a/test-local.sh +++ b/test-local.sh @@ -7,7 +7,7 @@ mprof run ./tests/performance/memory_pressure.py python3 ./tests/performance/memory_pressure_check.py # run CI tests -python3 test-ci.py +python test-ci.py # cleanup rm mprofile_*.dat From 6afc1cc28db2eaa3dd2f3f39e34e0be1f28a3c9d Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Fri, 28 Jan 2022 14:53:53 +0200 Subject: [PATCH 045/103] removed debug code --- test-ci.py | 20 ++++++++++---------- test-local.sh | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test-ci.py b/test-ci.py index 64585d87..b668586d 100755 --- a/test-ci.py +++ b/test-ci.py @@ -4,21 +4,21 @@ from tests.commands import * - #test_latest() + test_latest() - #scan_object = test_scan() - #recover_best_model() + scan_object = test_scan() + recover_best_model() - #test_random_methods() + test_random_methods() - #test_autom8() - #test_templates() + test_autom8() + test_templates() # test_analyze(scan_object) - #test_lr_normalizer() - #test_predict() - #test_reducers() - #test_rest(scan_object) + test_lr_normalizer() + test_predict() + test_reducers() + test_rest(scan_object) print("\n All tests successfully completed :) Good work. \n ") diff --git a/test-local.sh b/test-local.sh index ae1f083f..6567e70a 100755 --- a/test-local.sh +++ b/test-local.sh @@ -7,7 +7,7 @@ mprof run ./tests/performance/memory_pressure.py python3 ./tests/performance/memory_pressure_check.py # run CI tests -python test-ci.py +python3 test-ci.py # cleanup rm mprofile_*.dat From 24601006c2aaac3de2e2df57a65460c208779bdf Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Fri, 28 Jan 2022 21:59:38 +0200 Subject: [PATCH 046/103] fixed predict_classes in predict --- talos/commands/predict.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/talos/commands/predict.py b/talos/commands/predict.py index f8291c0d..2b89d2fc 100644 --- a/talos/commands/predict.py +++ b/talos/commands/predict.py @@ -44,6 +44,8 @@ def predict_classes(self, x, metric, asc, model_id=None): ''' + import numpy as np + if model_id is None: from ..utils.best_model import best_model model_id = best_model(self.scan_object, metric, asc) @@ -51,4 +53,8 @@ def predict_classes(self, x, metric, asc, model_id=None): from ..utils.best_model import activate_model model = activate_model(self.scan_object, model_id) - return model.predict_classes(x) + # make (class) predictiosn with the model + preds = model.predict(x) + preds_classes = np.argmax(preds, axis=1) + + return preds_classes From dace33af88ff2bed993ca3a16e9ccb54fd0a2f0f Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Fri, 28 Jan 2022 22:06:04 +0200 Subject: [PATCH 047/103] cleanup --- tests/commands/test_latest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/commands/test_latest.py b/tests/commands/test_latest.py index 792aae85..9e358f30 100644 --- a/tests/commands/test_latest.py +++ b/tests/commands/test_latest.py @@ -1,5 +1,9 @@ def test_latest(): + import warnings + + warnings.simplefilter('ignore') + print('\n >>> start Latest Features... \n') import talos From 595cec91cd763dc15b0fb9126d703c5064bfeca3 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Fri, 28 Jan 2022 23:36:44 +0200 Subject: [PATCH 048/103] updated docs version --- docs/_coverpage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_coverpage.md b/docs/_coverpage.md index b68c0bd2..690519cb 100644 --- a/docs/_coverpage.md +++ b/docs/_coverpage.md @@ -1,6 +1,6 @@ ![logo](_media/talos_logo_bg.png) -## v1.0 +## v1.0.2 > Hyperparameter Experiments with Tensorflow, PyTorch and Keras From e40384702fadc246dbef32e779d7fd9cc5fce9c6 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 14 Mar 2022 16:34:27 +0200 Subject: [PATCH 049/103] fixed tuple input reference from 'step' to 'n' --- docs/Examples_Typical.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Examples_Typical.md b/docs/Examples_Typical.md index bf829b27..8cbe9e94 100644 --- a/docs/Examples_Typical.md +++ b/docs/Examples_Typical.md @@ -62,7 +62,7 @@ p = {'activation':['relu', 'elu'], 'epochs': [10, 20]} ``` -Note that the parameter dictionary allows either list of values, or tuples with range in the form `(min, max, step)` +Note that the parameter dictionary allows either list of values, or tuples with range in the form `(min, max, number_of_values)` ### Scan() From 8f0ad82592ea40ee0813e293062015ed0d073146 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 14 Mar 2022 17:45:15 +0200 Subject: [PATCH 050/103] clean up _param_space_creation --- talos/parameters/ParamSpace.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/talos/parameters/ParamSpace.py b/talos/parameters/ParamSpace.py index 03ec9b9a..41b79a2c 100644 --- a/talos/parameters/ParamSpace.py +++ b/talos/parameters/ParamSpace.py @@ -130,10 +130,10 @@ def _param_space_creation(self): ''' # handle the cases where parameter space is still large - if len(self.param_index) > 100000: + if len(self.param_index) < 100000: final_grid = list(it.product(*self._params_temp)) - out = np.array(final_grid, dtype='object') + return np.array(final_grid, dtype='object') # handle the cases where parameter space is already smaller else: @@ -145,9 +145,7 @@ def _param_space_creation(self): p.insert(0, l[s]) final_grid.append(tuple(p)) - out = np.array(final_grid, dtype='object') - - return out + return np.array(final_grid, dtype='object') def _check_time_limit(self): From 552283934bd303399931c345e08a1b8efe8bdb4d Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 14 Mar 2022 17:45:58 +0200 Subject: [PATCH 051/103] remove redudant comments --- talos/parameters/ParamSpace.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/talos/parameters/ParamSpace.py b/talos/parameters/ParamSpace.py index 41b79a2c..cf9255d6 100644 --- a/talos/parameters/ParamSpace.py +++ b/talos/parameters/ParamSpace.py @@ -129,13 +129,11 @@ def _param_space_creation(self): actual parameter permutations for the experiment. ''' - # handle the cases where parameter space is still large if len(self.param_index) < 100000: final_grid = list(it.product(*self._params_temp)) return np.array(final_grid, dtype='object') - # handle the cases where parameter space is already smaller else: final_grid = [] for i in self.param_index: From d5f719acb20fb3970d3537d2a8e88bb59cea244c Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 14 Mar 2022 19:10:22 +0200 Subject: [PATCH 052/103] added DistributeParamSpace --- talos/parameters/DistributeParamSpace.py | 65 ++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 talos/parameters/DistributeParamSpace.py diff --git a/talos/parameters/DistributeParamSpace.py b/talos/parameters/DistributeParamSpace.py new file mode 100644 index 00000000..24b1d64a --- /dev/null +++ b/talos/parameters/DistributeParamSpace.py @@ -0,0 +1,65 @@ +class DistributeParamSpace: + + def __init__(self, + params, + param_keys, + random_method='uniform_mersenne', + fraction_limit=None, + round_limit=None, + time_limit=None, + boolean_limit=None, + machines=2): + + '''Splits ParamSpace object based on number + of machines. + + params | object | ParamSpace class object + machines | int | number of machines to split for + + ''' + + import numpy as np + import copy + + from talos.parameters.ParamSpace import ParamSpace + + self._params = ParamSpace(params=params, + param_keys=param_keys, + random_method='uniform_mersenne', + fraction_limit=None, + round_limit=None, + time_limit=None, + boolean_limit=None) + + self.machines = machines + + self.param_spaces = self._split_param_space() + + def _split_param_space(self): + + '''Takes in a ParamSpace object and splits it so that + it can be used in DistributeScan experiments.''' + + out = {} + + # randomly shuffle the param_space + rand = np.random.default_rng() + rand.shuffle(self._params.param_space, axis=0) + + # split into n arras + param_spaces = np.array_split(self._params.param_space, self.machines) + + # remove keys to allow copy + param_keys = self._params.param_keys + self._params.param_keys = [] + + # create the individual ParamSpace objects + for i in range(self.machines): + + out[i] = copy.deepcopy(self._params) + out[i].param_space = param_spaces[i] + out[i].dimensions = len(out[i].param_space) + out[i].param_index = list(range(out[i].dimensions)) + out[i].param_keys = param_keys + + return out From 134ea1ab6070fc9e32d88d95d2de4203ecff3e3a Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 14 Mar 2022 19:13:02 +0200 Subject: [PATCH 053/103] added simple tests --- tests/commands/test_latest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/commands/test_latest.py b/tests/commands/test_latest.py index 9e358f30..ee9e696a 100644 --- a/tests/commands/test_latest.py +++ b/tests/commands/test_latest.py @@ -22,6 +22,10 @@ def test_latest(): 'batch_size': [20, 30, 40, 50], 'epochs': [10]} + from talos.parameters.DistributeParamSpace import DistributeParamSpace + + _ = DistributeParamSpace(params=p, param_keys=p.keys(), machines=5) + def iris_model(x_train, y_train, x_val, y_val, params): model = Sequential() From 0cdcda29f26856aab485a37456b81f5b0195b88f Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 14 Mar 2022 19:20:04 +0200 Subject: [PATCH 054/103] fixed missing imports --- talos/parameters/DistributeParamSpace.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/talos/parameters/DistributeParamSpace.py b/talos/parameters/DistributeParamSpace.py index 24b1d64a..a9a51584 100644 --- a/talos/parameters/DistributeParamSpace.py +++ b/talos/parameters/DistributeParamSpace.py @@ -18,9 +18,6 @@ def __init__(self, ''' - import numpy as np - import copy - from talos.parameters.ParamSpace import ParamSpace self._params = ParamSpace(params=params, @@ -40,6 +37,9 @@ def _split_param_space(self): '''Takes in a ParamSpace object and splits it so that it can be used in DistributeScan experiments.''' + import numpy as np + import copy + out = {} # randomly shuffle the param_space From 6e40074983f397e3195509396a2711a27d4c9d59 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Mon, 14 Mar 2022 19:20:28 +0200 Subject: [PATCH 055/103] changed version to 1.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 12f9d848..d42f3b6d 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ URL = 'http://autonom.io' LICENSE = 'MIT' DOWNLOAD_URL = 'https://github.com/autonomio/talos/' -VERSION = '1.0.2' +VERSION = '1.1.0' try: From b852316e8bf136e69fcd203c24870266ca4ed409 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 15 Mar 2022 10:04:57 +0200 Subject: [PATCH 056/103] clean for pep8 --- talos/parameters/DistributeParamSpace.py | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/talos/parameters/DistributeParamSpace.py b/talos/parameters/DistributeParamSpace.py index a9a51584..894be1e3 100644 --- a/talos/parameters/DistributeParamSpace.py +++ b/talos/parameters/DistributeParamSpace.py @@ -1,5 +1,5 @@ class DistributeParamSpace: - + def __init__(self, params, param_keys, @@ -9,7 +9,7 @@ def __init__(self, time_limit=None, boolean_limit=None, machines=2): - + '''Splits ParamSpace object based on number of machines. @@ -19,21 +19,21 @@ def __init__(self, ''' from talos.parameters.ParamSpace import ParamSpace - + self._params = ParamSpace(params=params, - param_keys=param_keys, - random_method='uniform_mersenne', - fraction_limit=None, - round_limit=None, - time_limit=None, - boolean_limit=None) - + param_keys=param_keys, + random_method='uniform_mersenne', + fraction_limit=None, + round_limit=None, + time_limit=None, + boolean_limit=None) + self.machines = machines - + self.param_spaces = self._split_param_space() - + def _split_param_space(self): - + '''Takes in a ParamSpace object and splits it so that it can be used in DistributeScan experiments.''' From 55b4233a8bce47af208783c9bbf385c1e2c5b198 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 15 Mar 2022 10:15:50 +0200 Subject: [PATCH 057/103] improved PR template --- .github/PULL_REQUEST_TEMPLATE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6fc0af79..d2e59b2d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -8,11 +8,12 @@ the below items: - [ ] I'm aware of the implications of the proposed changes - [ ] Code is [PEP8](https://www.python.org/dev/peps/pep-0008/) - [ ] I'm making the PR to `master` +- [ ] I've changed the version in `setup.py` based on [Semantic Versioning](https://semver.org/) #### Docs - [ ] [Docs](https://autonomio.github.io/talos) are updated -- [ ] [Docs](https://autonomio.github.io/talos) version is correct (index.html and \_coverpage.md) +- [ ] [Docs](https://autonomio.github.io/talos) version is correct on (index.html and _coverpage.md) #### Tests From 254e2a56bb287d926f54949876191fd0673e89e7 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 15 Mar 2022 10:53:05 +0200 Subject: [PATCH 058/103] Scan allows ParamSpace as input --- talos/scan/scan_prepare.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/talos/scan/scan_prepare.py b/talos/scan/scan_prepare.py index 0ec17f89..cc5d4556 100644 --- a/talos/scan/scan_prepare.py +++ b/talos/scan/scan_prepare.py @@ -16,19 +16,30 @@ def scan_prepare(self): elif self.x_val is not None and self.y_val is not None: self.custom_val_split = True - # create reference for parameter keys - self._param_dict_keys = sorted(list(self.params.keys())) - - # create the parameter object and move to self - from ..parameters.ParamSpace import ParamSpace - self.param_object = ParamSpace(params=self.params, - param_keys=self._param_dict_keys, - random_method=self.random_method, - fraction_limit=self.fraction_limit, - round_limit=self.round_limit, - time_limit=self.time_limit, - boolean_limit=self.boolean_limit - ) + # handle the case where self.params is dictionary + if isinstance(self.params, dict): + + # create reference for parameter keys + self._param_dict_keys = sorted(list(self.params.keys())) + + # create the parameter object and move to self + from ..parameters.ParamSpace import ParamSpace + self.param_object = ParamSpace(params=self.params, + param_keys=self._param_dict_keys, + random_method=self.random_method, + fraction_limit=self.fraction_limit, + round_limit=self.round_limit, + time_limit=self.time_limit, + boolean_limit=self.boolean_limit) + + # handle the case when self.params already is ParamSpace object + elif type(self.params) == 'talos.parameters.ParamSpace.ParamSpace': + + self._param_dict_keys = sorted(list(self.params.param_keys)) + self.param_object = self.params + + else: + raise TypeError('params has to be either dictionary or ParamSpace object.') # mark that it's a first round self.first_round = True From cf2f23661bb6586ab07183a5fee1563e1fa63530 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 15 Mar 2022 10:54:05 +0200 Subject: [PATCH 059/103] changed version to 1.1.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d42f3b6d..e0efb4f3 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ URL = 'http://autonom.io' LICENSE = 'MIT' DOWNLOAD_URL = 'https://github.com/autonomio/talos/' -VERSION = '1.1.0' +VERSION = '1.1.1' try: From 76f65f50fc0c8f43729e77518b2d6b3d6b165ac4 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Tue, 15 Mar 2022 11:13:48 +0200 Subject: [PATCH 060/103] fixed failing type check --- talos/parameters/DistributeParamSpace.py | 4 ++++ talos/scan/scan_prepare.py | 2 +- tests/commands/test_latest.py | 12 +++++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/talos/parameters/DistributeParamSpace.py b/talos/parameters/DistributeParamSpace.py index 894be1e3..e27b8950 100644 --- a/talos/parameters/DistributeParamSpace.py +++ b/talos/parameters/DistributeParamSpace.py @@ -16,6 +16,10 @@ def __init__(self, params | object | ParamSpace class object machines | int | number of machines to split for + NOTE: `Scan()` limits will not be applied if ParamSpace object + is passed directly into `Scan()` as `params` argument so they + should be passed directly into `DistributeParamSpace` instead. + ''' from talos.parameters.ParamSpace import ParamSpace diff --git a/talos/scan/scan_prepare.py b/talos/scan/scan_prepare.py index cc5d4556..2ced18dc 100644 --- a/talos/scan/scan_prepare.py +++ b/talos/scan/scan_prepare.py @@ -33,7 +33,7 @@ def scan_prepare(self): boolean_limit=self.boolean_limit) # handle the case when self.params already is ParamSpace object - elif type(self.params) == 'talos.parameters.ParamSpace.ParamSpace': + elif 'talos.parameters.ParamSpace.ParamSpace' in str(type(self.params)): self._param_dict_keys = sorted(list(self.params.param_keys)) self.param_object = self.params diff --git a/tests/commands/test_latest.py b/tests/commands/test_latest.py index ee9e696a..ca48c80e 100644 --- a/tests/commands/test_latest.py +++ b/tests/commands/test_latest.py @@ -24,7 +24,7 @@ def test_latest(): from talos.parameters.DistributeParamSpace import DistributeParamSpace - _ = DistributeParamSpace(params=p, param_keys=p.keys(), machines=5) + param_spaces = DistributeParamSpace(params=p, param_keys=p.keys(), machines=100) def iris_model(x_train, y_train, x_val, y_val, params): @@ -49,6 +49,16 @@ def iris_model(x_train, y_train, x_val, y_val, params): return out, model + print(type(param_spaces.param_spaces[0])) + + scan_object = talos.Scan(x, y, + model=iris_model, + params=param_spaces.param_spaces[0], + experiment_name='test_latest', + round_limit=5, + reduction_method='gamify', + save_weights=False) + scan_object = talos.Scan(x, y, model=iris_model, params=p, From 0448b87c7f079a8fddd4f867334990f0eaabd9af Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 2 Apr 2022 10:43:26 +0300 Subject: [PATCH 061/103] cleaned up pep8 --- talos/scan/scan_prepare.py | 16 ++++++++-------- tests/commands/test_latest.py | 19 +++++++++++-------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/talos/scan/scan_prepare.py b/talos/scan/scan_prepare.py index 2ced18dc..5c6baddf 100644 --- a/talos/scan/scan_prepare.py +++ b/talos/scan/scan_prepare.py @@ -25,21 +25,21 @@ def scan_prepare(self): # create the parameter object and move to self from ..parameters.ParamSpace import ParamSpace self.param_object = ParamSpace(params=self.params, - param_keys=self._param_dict_keys, - random_method=self.random_method, - fraction_limit=self.fraction_limit, - round_limit=self.round_limit, - time_limit=self.time_limit, - boolean_limit=self.boolean_limit) + param_keys=self._param_dict_keys, + random_method=self.random_method, + fraction_limit=self.fraction_limit, + round_limit=self.round_limit, + time_limit=self.time_limit, + boolean_limit=self.boolean_limit) # handle the case when self.params already is ParamSpace object elif 'talos.parameters.ParamSpace.ParamSpace' in str(type(self.params)): - + self._param_dict_keys = sorted(list(self.params.param_keys)) self.param_object = self.params else: - raise TypeError('params has to be either dictionary or ParamSpace object.') + raise TypeError('params has to be either dict or ParamSpace object.') # mark that it's a first round self.first_round = True diff --git a/tests/commands/test_latest.py b/tests/commands/test_latest.py index ca48c80e..10459a77 100644 --- a/tests/commands/test_latest.py +++ b/tests/commands/test_latest.py @@ -24,7 +24,9 @@ def test_latest(): from talos.parameters.DistributeParamSpace import DistributeParamSpace - param_spaces = DistributeParamSpace(params=p, param_keys=p.keys(), machines=100) + param_spaces = DistributeParamSpace(params=p, + param_keys=p.keys(), + machines=100) def iris_model(x_train, y_train, x_val, y_val, params): @@ -51,13 +53,14 @@ def iris_model(x_train, y_train, x_val, y_val, params): print(type(param_spaces.param_spaces[0])) - scan_object = talos.Scan(x, y, - model=iris_model, - params=param_spaces.param_spaces[0], - experiment_name='test_latest', - round_limit=5, - reduction_method='gamify', - save_weights=False) + scan_object = talos.Scan(x, + y, + model=iris_model, + params=param_spaces.param_spaces[0], + experiment_name='test_latest', + round_limit=5, + reduction_method='gamify', + save_weights=False) scan_object = talos.Scan(x, y, model=iris_model, From dfeb587d6275109083b58566f7ad9baed3259f38 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sat, 2 Apr 2022 10:44:23 +0300 Subject: [PATCH 062/103] cleaned up pep8 --- tests/commands/test_latest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/commands/test_latest.py b/tests/commands/test_latest.py index 10459a77..6ad5cb10 100644 --- a/tests/commands/test_latest.py +++ b/tests/commands/test_latest.py @@ -53,7 +53,7 @@ def iris_model(x_train, y_train, x_val, y_val, params): print(type(param_spaces.param_spaces[0])) - scan_object = talos.Scan(x, + scan_object = talos.Scan(x, y, model=iris_model, params=param_spaces.param_spaces[0], From c4251c15f7d1c19d0bea9813a4111ff908890889 Mon Sep 17 00:00:00 2001 From: Abhijith Neil Abraham Date: Sun, 3 Apr 2022 12:42:42 +0530 Subject: [PATCH 063/103] Update Scan.md --- docs/Scan.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Scan.md b/docs/Scan.md index 1f4b7497..08ef83f4 100644 --- a/docs/Scan.md +++ b/docs/Scan.md @@ -17,7 +17,7 @@ Argument | Input | Description --------- | ------- | ----------- `x` | array or list of arrays | prediction features `y` | array or list of arrays | prediction outcome variable -`params` | dict | the parameter dictionary +`params` | dict or ParamSpace object | the parameter dictionary or the ParamSpace object after splitting `model` | function | the Keras model as a function `experiment_name` | str | Used for creating the experiment logging folder `x_val` | array or list of arrays | validation data for x From 1757b6b8f4ad4f6d3c9c77ec73e80aa9fa877ab8 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Wed, 13 Apr 2022 18:35:17 +0300 Subject: [PATCH 064/103] added save_models option --- talos/scan/Scan.py | 8 +++++++- talos/scan/scan_round.py | 33 ++++++++++++++++++++------------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/talos/scan/Scan.py b/talos/scan/Scan.py index 3a84ab4f..05a15e34 100755 --- a/talos/scan/Scan.py +++ b/talos/scan/Scan.py @@ -125,6 +125,10 @@ def model(): If set to False, then model weights will not be saved and best_model and some other features will not work. Will reduce memory pressure on very large models and high number of rounds/permutations. + save_models : bool + If set to True, models will be saved on the local disk in the experiment + folder. When `save_models` is set to True, you should consider setting + `save_weights` to False. """ def __init__(self, @@ -152,7 +156,8 @@ def __init__(self, disable_progress_bar=False, print_params=False, clear_session=True, - save_weights=True): + save_weights=True, + save_models=False): self.x = x self.y = y @@ -189,6 +194,7 @@ def __init__(self, # performance self.clear_session = clear_session self.save_weights = save_weights + self.save_models = save_models # input parameters section ends # start runtime diff --git a/talos/scan/scan_round.py b/talos/scan/scan_round.py index d333117f..cba92033 100644 --- a/talos/scan/scan_round.py +++ b/talos/scan/scan_round.py @@ -27,23 +27,30 @@ def scan_round(self): from ..reducers.reduce_run import reduce_run self = reduce_run(self) - try: - # save model and weights - self.saved_models.append(self.round_model.to_json()) - - if self.save_weights: - self.saved_weights.append(self.round_model.get_weights()) - else: - self.saved_weights.append(None) - - except AttributeError as e: - # make sure that the error message is from torch - if str(e) == "'Model' object has no attribute 'to_json'": + # handle the case where the actual model is to be saved + if self.save_models: + file_path = self.experiment_name + '/' + str(len(self.round_history) - 1) + self.round_model.save(file_path) + + # handle other cases + else: + try: + # save model and weights + self.saved_models.append(self.round_model.to_json()) + if self.save_weights: - self.saved_models.append(self.round_model.state_dict()) + self.saved_weights.append(self.round_model.get_weights()) else: self.saved_weights.append(None) + except AttributeError as e: + # make sure that the error message is from torch + if str(e) == "'Model' object has no attribute 'to_json'": + if self.save_weights: + self.saved_models.append(self.round_model.state_dict()) + else: + self.saved_weights.append(None) + # clear tensorflow sessions if self.clear_session is True: From 481e2d0805967e0893f34e7956aef14cd6c9f1cf Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Wed, 13 Apr 2022 18:39:22 +0300 Subject: [PATCH 065/103] added tests --- tests/commands/test_latest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/commands/test_latest.py b/tests/commands/test_latest.py index 6ad5cb10..9028545f 100644 --- a/tests/commands/test_latest.py +++ b/tests/commands/test_latest.py @@ -60,7 +60,7 @@ def iris_model(x_train, y_train, x_val, y_val, params): experiment_name='test_latest', round_limit=5, reduction_method='gamify', - save_weights=False) + save_models=True) scan_object = talos.Scan(x, y, model=iris_model, From f28c1103d8f2c2cb214da1050e1eab352a7e1928 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Wed, 13 Apr 2022 18:40:33 +0300 Subject: [PATCH 066/103] appended docs --- docs/Scan.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/Scan.md b/docs/Scan.md index 08ef83f4..a6265c9b 100644 --- a/docs/Scan.md +++ b/docs/Scan.md @@ -39,7 +39,8 @@ Argument | Input | Description `disable_progress_bar` | bool | Disable live updating progress bar `print_params` | bool | Print each permutation hyperparameters `clear_session` | bool | Clear backend session between permutations -`save_weights` | bool | Save model weights (increases memory pressure for large models) +`save_weights` | bool | Keep model weights (increases memory pressure for large models) +`save_models` | bool | Save models in the experiment folder in local machine NOTE: `boolean_limit` will only work if its the last argument in `Scan()` and the following bracket is on a newline: From 1fdbe2dfd02b70eaafbe859d32e50c7950b6a442 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Wed, 13 Apr 2022 18:41:38 +0300 Subject: [PATCH 067/103] bumbed version to 1.2 --- setup.py | 4 ++-- talos/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e0efb4f3..01b4c0e2 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #! /usr/bin/env python # -# Copyright (C) 2021 Mikko Kotila +# Copyright (C) 2022 Mikko Kotila DESCRIPTION = "Talos Hyperparameter Tuning for Keras" LONG_DESCRIPTION = """\ @@ -18,7 +18,7 @@ URL = 'http://autonom.io' LICENSE = 'MIT' DOWNLOAD_URL = 'https://github.com/autonomio/talos/' -VERSION = '1.1.1' +VERSION = '1.2' try: diff --git a/talos/__init__.py b/talos/__init__.py index bfa68bd9..e9eb1669 100755 --- a/talos/__init__.py +++ b/talos/__init__.py @@ -34,4 +34,4 @@ del commands, scan, model, metrics, key del sub, keep_from_templates, template_sub, warnings -__version__ = "1.0.2" +__version__ = "1.2" From 6396aa305387b3c26409f3169dbdc5eda37f0f9d Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Wed, 13 Apr 2022 20:33:07 +0300 Subject: [PATCH 068/103] updated functions to support custom objects --- talos/commands/deploy.py | 26 +++++++------ talos/commands/evaluate.py | 43 ++++++++++----------- talos/commands/predict.py | 15 +++++--- talos/scan/scan_addon.py | 67 ++++++++++++++++++++------------- talos/utils/best_model.py | 26 +++++++++---- talos/utils/validation_split.py | 38 ++++++++++++++++--- 6 files changed, 135 insertions(+), 80 deletions(-) diff --git a/talos/commands/deploy.py b/talos/commands/deploy.py index 73c71216..adcc64aa 100644 --- a/talos/commands/deploy.py +++ b/talos/commands/deploy.py @@ -2,7 +2,13 @@ class Deploy: '''Functionality for deploying a model to a filename''' - def __init__(self, scan_object, model_name, metric, asc=False): + def __init__(self, + scan_object, + model_name, + metric, + asc=False, + saved=False, + custom_objects=None): '''Deploy a model to be used later or in a different system. @@ -12,15 +18,13 @@ def __init__(self, scan_object, model_name, metric, asc=False): Deploy() takes in the object from Scan() and creates a package locally that can be later activated with Restore(). - scan_object : object - The object that is returned from Scan() upon completion. - model_name : str - Name for the .zip file to be created. - metric : str - The metric to be used for picking the best model. - asc: bool - Make this True for metrics that are to be minimized (e.g. loss) , - and False when the metric is to be maximized (e.g. acc) + scan_object | object | The object that is returned from Scan() upon completion. + model_name | str | Name for the .zip file to be created. + metric | str | The metric to be used for picking the best model. + asc | bool | Make this True for metrics that are to be minimized (e.g. loss) , + and False when the metric is to be maximized (e.g. acc) + saved | bool | if a model saved on local machine should be used + custom_objects | dict | if the model has a custom object, pass it here ''' @@ -36,7 +40,7 @@ def __init__(self, scan_object, model_name, metric, asc=False): from ..utils.best_model import best_model, activate_model self.best_model = best_model(scan_object, metric, asc) - self.model = activate_model(scan_object, self.best_model) + self.model = activate_model(scan_object, self.best_model, saved, custom_objects) # runtime self.save_model_as() diff --git a/talos/commands/evaluate.py b/talos/commands/evaluate.py index 8877ab0b..ab867d9f 100644 --- a/talos/commands/evaluate.py +++ b/talos/commands/evaluate.py @@ -20,34 +20,28 @@ def evaluate(self, folds=5, shuffle=True, asc=False, + saved=False, + custom_objects=None, print_out=False): '''Evaluate a model based on f1_score (all except regression) or mae (for regression). Supports 'binary', 'multi_class', 'multi_label', and 'regression' evaluation. - x : array - The input data for making predictions - y : array - The ground truth for x - model_id : int - It's possible to evaluate a specific model based on ID. - Can be None. - folds : int - Number of folds to use for cross-validation - sort_metric : string - A column name referring to the metric that was used in the - scan_object as a performance metric. This is used for sorting - the results to pick for evaluation. - shuffle : bool - Data is shuffled before evaluation. - task : string - 'binary', 'multi_class', 'multi_label', or 'continuous'. - asc : bool - False if the metric is to be optimized upwards - (e.g. accuracy or f1_score) - print_out : bool - Print out the results. + x | array | The input data for making predictions + y | array | The ground truth for x + model_id | int | It's possible to evaluate a specific model based on ID. + folds | int | Number of folds to use for cross-validation + sort_metric | string | A column name referring to the metric that was used in the + scan_object as a performance metric. This is used for sorting + the results to pick for evaluation. + shuffle | bool | Data is shuffled before evaluation. + task | string | 'binary', 'multi_class', 'multi_label', or 'continuous'. + asc | bool | False if the metric is to be optimized upwards + (e.g. accuracy or f1_score) + saved | bool | if a model saved on local machine should be used + custom_objects | dict | if the model has a custom object, pass it here + print_out | bool | Print out the results. TODO: add possibility to input custom metrics. @@ -62,7 +56,10 @@ def evaluate(self, model_id = best_model(self.scan_object, metric, asc) from ..utils.best_model import activate_model - model = activate_model(self.scan_object, model_id) + model = activate_model(self.scan_object, + model_id, + saved=saved, + custom_objects=custom_objects) from ..utils.validation_split import kfold kx, ky = kfold(x, y, folds, shuffle) diff --git a/talos/commands/predict.py b/talos/commands/predict.py index 2b89d2fc..797c53ef 100644 --- a/talos/commands/predict.py +++ b/talos/commands/predict.py @@ -11,7 +11,7 @@ def __init__(self, scan_object): self.scan_object = scan_object self.data = scan_object.data - def predict(self, x, metric, asc, model_id=None): + def predict(self, x, metric, asc, model_id=None, saved=False, custom_objects=None): '''Makes a probability prediction from input x. If model_id is not given, then best_model will be used. @@ -20,6 +20,8 @@ def predict(self, x, metric, asc, model_id=None): model_id | int | the id of the model from the Scan() object metric | str | the metric to be used for picking best model asc | bool | True if `metric` is something to be minimized + saved | bool | if a model saved on local machine should be used + custom_objects | dict | if the model has a custom object, pass it here ''' @@ -28,11 +30,11 @@ def predict(self, x, metric, asc, model_id=None): model_id = best_model(self.scan_object, metric, asc) from ..utils.best_model import activate_model - model = activate_model(self.scan_object, model_id) + model = activate_model(self.scan_object, model_id, saved, custom_objects) return model.predict(x) - def predict_classes(self, x, metric, asc, model_id=None): + def predict_classes(self, x, metric, asc, model_id=None, saved=False, custom_objects=None): '''Makes a class prediction from input x. If model_id is not given, then best_model will be used. @@ -41,7 +43,8 @@ def predict_classes(self, x, metric, asc, model_id=None): model_id | int | the id of the model from the Scan() object metric | str | the metric to be used for picking best model asc | bool | True if `metric` is something to be minimized - + saved | bool | if a model saved on local machine should be used + custom_objects | dict | if the model has a custom object, pass it here ''' import numpy as np @@ -51,9 +54,9 @@ def predict_classes(self, x, metric, asc, model_id=None): model_id = best_model(self.scan_object, metric, asc) from ..utils.best_model import activate_model - model = activate_model(self.scan_object, model_id) + model = activate_model(self.scan_object, model_id, saved, custom_objects) - # make (class) predictiosn with the model + # make (class) predictions with the model preds = model.predict(x) preds_classes = np.argmax(preds, axis=1) diff --git a/talos/scan/scan_addon.py b/talos/scan/scan_addon.py index 485f69a1..cd52e7a1 100644 --- a/talos/scan/scan_addon.py +++ b/talos/scan/scan_addon.py @@ -1,8 +1,15 @@ -def func_best_model(scan_object, metric='val_acc', asc=False): +def func_best_model(scan_object, metric='val_acc', asc=False, saved=False, custom_objects=None): '''Picks the best model based on a given metric and returns the index number for the model. + scan_object | object | The object that is returned from Scan() upon completion. + metric | str | The metric to be used for picking the best model. + asc | bool | Make this True for metrics that are to be minimized (e.g. loss) , + and False when the metric is to be maximized (e.g. acc) + saved | bool | if a model saved on local machine should be used + custom_objects | dict | if the model has a custom object, pass it here + NOTE: for loss 'asc' should be True''' import warnings as warnings @@ -11,7 +18,7 @@ def func_best_model(scan_object, metric='val_acc', asc=False): from ..utils.best_model import best_model, activate_model model_no = best_model(scan_object, metric, asc) - out = activate_model(scan_object, model_no) + out = activate_model(scan_object, model_no, saved, custom_objects) return out @@ -24,35 +31,34 @@ def func_evaluate(scan_object, metric='val_acc', folds=5, shuffle=True, - asc=False): + asc=False, + saved=False, + custom_objects=None): '''K-fold Cross Evaluator For creating scores from kfold cross-evaluation and adding them to the data frame. - scan_object : python class - The class object returned by Scan() upon completion of the experiment. - x_val : array or list of arrays - Input data (features) in the same format as used in Scan(), but should - not be the same data (or it will not be much of validation). - y_val : array or list of arrays - Input data (labels) in the same format as used in Scan(), but should - not be the same data (or it will not be much of validation). - task : string - 'binary', 'multi_class', 'multi_label', or 'continuous'. - n_models : int - The number of models to be evaluated. If set to 10, then 10 models - with the highest metric value are evaluated. See below. - metric : str - The metric to be used for picking the models to be evaluated. - folds : int - The number of folds to be used in the evaluation. - shuffle : bool - If the data is to be shuffled or not. Set always to False for - timeseries but keep in mind that you might get periodical/seasonal bias. - asc : bool - Set to True if the metric is to be minimized. + scan_object | python class | The class object returned by Scan() upon + completion of the experiment. + x_val | array or list of arrays | Input data (features) in the same format + as used in Scan(), but should not be the + same data (or it will not be much of validation). + y_val | array or list of arrays | Input data (labels) in the same format as used in + Scan(), but should not be the same data (or it will + not be much of validation). + task | string | 'binary', 'multi_class', 'multi_label', or 'continuous'. + n_models | int | The number of models to be evaluated. If set to 10, then 10 models + with the highest metric value are evaluated. See below. + metric | str | The metric to be used for picking the models to be evaluated. + folds | int | The number of folds to be used in the evaluation. + shuffle | bool | If the data is to be shuffled or not. Set always to False for + timeseries but keep in mind that you might get periodical/seasonal + bias. + asc | bool | Set to True if the metric is to be minimized. + saved | bool | if a model saved on local machine should be used + custom_objects | dict | if the model has a custom object, pass it here ''' import warnings as warnings @@ -74,18 +80,25 @@ def func_evaluate(scan_object, pbar = tqdm(total=data_len) from ..commands.evaluate import Evaluate + for i in range(len(scan_object.data)): + if i in list(picks): + evaluate_object = Evaluate(scan_object) - temp = evaluate_object.evaluate(x_val, y_val, + temp = evaluate_object.evaluate(x_val,y_val, task=task, model_id=i, metric=metric, folds=folds, shuffle=shuffle, - asc=asc) + asc=asc, + saved=saved, + custom_objects=custom_objects) + out.append([np.mean(temp), np.std(temp)]) pbar.update(1) + else: out.append([np.nan, np.nan]) diff --git a/talos/utils/best_model.py b/talos/utils/best_model.py index d1bb93e4..a318d881 100644 --- a/talos/utils/best_model.py +++ b/talos/utils/best_model.py @@ -1,6 +1,3 @@ -from tensorflow.keras.models import model_from_json - - def best_model(self, metric, asc): '''Picks the best model based on a given metric and @@ -13,11 +10,26 @@ def best_model(self, metric, asc): return best -def activate_model(self, model_id): +def activate_model(self, model_id, saved=False, custom_objects=None): + + '''Loads the model from the json that is stored in the Scan object + or from local + + model_id | int | the sequential id of the model + saved | bool | if a model saved on local machine should be used + custom_object | dict | if the model has a custom object, pass it here + + ''' + + import tensorflow as tf + from tensorflow.keras.models import model_from_json - '''Loads the model from the json that is stored in the Scan object''' + if saved: + file_path = self.details['experiment_name'] + '/' + str(model_id) + model = tf.keras.models.load_model(file_path, custom_objects=custom_objects) - model = model_from_json(self.saved_models[model_id]) - model.set_weights(self.saved_weights[model_id]) + else: + model = model_from_json(self.saved_models[model_id]) + model.set_weights(self.saved_weights[model_id]) return model diff --git a/talos/utils/validation_split.py b/talos/utils/validation_split.py index f58fd703..54752838 100644 --- a/talos/utils/validation_split.py +++ b/talos/utils/validation_split.py @@ -1,16 +1,19 @@ def validation_split(self): - """Defines the attributes `x_train`, `y_train`, `x_val` and `y_val`. - The validation (cross-validation, aka development) sets are determined - by the attribute val_split, which is a number in (0, 1) which determines - the proportion of the input data to be allocated for cross-validation.""" + + '''Defines the attributes `x_train`, `y_train`, `x_val` and `y_val`. + The validation sets are determined by the attribute val_split, + which is a number in (0, 1) which determines the proportion of + the input data to be allocated for cross-validation.''' + # If split is done in `Scan()` do nothing if self.custom_val_split: + self.x_train = self.x self.y_train = self.y - # self.x/y_val are already set + # Otherwise handle splitting else: - # shuffle the data before splitting + random_shuffle(self) # deduce the midway point for input data @@ -88,3 +91,26 @@ def kfold(x, y, folds=10, shuffled=True): hi += step return out_x, out_y + +def shuffle(data): + + import numpy as np + + if isinstance(data, list) is False: + data = [data] + is_array = True + + rng = np.random.default_rng() + state = rng.bit_generator.state + + out = [] + + for ar in data: + rng.bit_generator.state = state + rng.shuffle(ar) + + if is_array: + data = data[0] + + return data + \ No newline at end of file From ce94039a9e8263145edf8a467e1844ca87164da8 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Wed, 13 Apr 2022 20:39:17 +0300 Subject: [PATCH 069/103] updated docs --- docs/Deploy.md | 2 ++ docs/Evaluate.md | 2 ++ docs/Predict.md | 13 ++++++++++--- docs/Scan.md | 2 ++ docs/_coverpage.md | 2 +- docs/index.html | 2 +- 6 files changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/Deploy.md b/docs/Deploy.md index 515b95ce..897c54c8 100644 --- a/docs/Deploy.md +++ b/docs/Deploy.md @@ -21,6 +21,8 @@ Parameter | type | Description `model_name` | str | Name for the .zip file to be created. `metric` | str | The metric to be used for picking the best model. `asc` | bool | Make this True for metrics that are to be minimized (e.g. loss) +`saved` | bool | if a model saved on local machine should be used +`custom_objects` | dict | if the model has a custom object, pass it here ## Deploy Package Contents diff --git a/docs/Evaluate.md b/docs/Evaluate.md index 81ea851c..3e91ae97 100644 --- a/docs/Evaluate.md +++ b/docs/Evaluate.md @@ -31,5 +31,7 @@ Parameter | Default | Description `average` | 'binary' | 'binary', 'micro', 'macro', 'samples', or 'weighted' `metric` | None | the metric against which the validation is performed `asc` | None | should be True if metric is a loss +`saved` | bool | if a model saved on local machine should be used +`custom_objects` | dict | if the model has a custom object, pass it here The above arguments are for the evaluate attribute of the Evaluate object. diff --git a/docs/Predict.md b/docs/Predict.md index afb7ec7e..10f58448 100644 --- a/docs/Predict.md +++ b/docs/Predict.md @@ -26,6 +26,13 @@ scan_object.data ### Predict Arguments -Parameter | Description ---------- | ----------- -`scan_object` | The resulting class object from `Scan()` +### Predict.predict Arguments + +Parameter | Default | Description +--------- | ------- | ----------- +`x` | NA | the predictor data x +`model_id` | None | the model_id to be used +`metric` | None | the metric against which the validation is performed +`asc` | None | should be True if metric is a loss +`saved` | bool | if a model saved on local machine should be used +`custom_objects` | dict | if the model has a custom object, pass it here diff --git a/docs/Scan.md b/docs/Scan.md index a6265c9b..60790c51 100644 --- a/docs/Scan.md +++ b/docs/Scan.md @@ -113,6 +113,8 @@ Argument | Description `shuffle` | If the data is to be shuffled or not. Set always to False for timeseries but keep in mind that you might get periodical/seasonal bias. `average` |One of the supported averaging methods: 'binary', 'micro', or 'macro' `asc` |Set to True if the metric is to be minimized. +`saved` | bool | if a model saved on local machine should be used +`custom_objects` | dict | if the model has a custom object, pass it here

diff --git a/docs/_coverpage.md b/docs/_coverpage.md index 690519cb..89961a9e 100644 --- a/docs/_coverpage.md +++ b/docs/_coverpage.md @@ -1,6 +1,6 @@ ![logo](_media/talos_logo_bg.png) -## v1.0.2 +## v1.2 > Hyperparameter Experiments with Tensorflow, PyTorch and Keras diff --git a/docs/index.html b/docs/index.html index 71c3a67d..c9e846d1 100644 --- a/docs/index.html +++ b/docs/index.html @@ -16,7 +16,7 @@