From 39554bc9fccc298dd91f680936598ae1fe1ce6d9 Mon Sep 17 00:00:00 2001 From: Giridhar Ganapavarapu Date: Thu, 27 Apr 2023 08:47:59 -0700 Subject: [PATCH 1/5] first commit to resolve tf issue Signed-off-by: Giridhar Ganapavarapu --- setup.py | 158 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 115 insertions(+), 43 deletions(-) diff --git a/setup.py b/setup.py index 45b11da..5176b31 100644 --- a/setup.py +++ b/setup.py @@ -1,53 +1,125 @@ import setuptools -version = '0.2.1' +version = "0.2.1" -with open("aix360/version.py", 'w') as f: +with open("aix360/version.py", "w") as f: f.write('# generated by setup.py\nversion = "{}"\n'.format(version)) +extra_requires = { + "default": [ + "numpy", + "pandas", + "scikit-learn", + "matplotlib", + ], + "rbm": [ + "matplotlib", + "pandas<=1.4.3", + "scipy>=0.17", + "scikit-learn<1.2.0", + "cvxpy>=1.1", + ], + "profwt": [ + "keras==2.3.1", + "scipy>=0.17", + "tensorflow==1.14", + ], + "cofrnet": [ + "pandas<=1.4.3", + "torch", + "tqdm", + ], + "ted": [ + "pandas", + "scikit-learn", + ], + "dipvae": [ + "matplotlib", + "torch", + "torchvision", + ], + "rule_induction": [ + "matplotlib", + "numba", + "pandas<=1.4.3", + "scikit-learn", + "nyoka", + "cvxpy", + "xmltodict==0.12.0", + ], + "lime": [ + "lime", + "tqdm", + "pandas", + ], + "matching": ["otoc @ git+https://github.com/IBM/otoc@main#egg=otoc"], + "protodash": [ + "scikit-learn", + "xport", + "cvxpy", + "requests", + ], + "contrastive": [ + "keras==2.3.1", + "tensorflow==1.14", + "requests", + "scipy>=0.17", + ], + "shap": [ + "keras==2.3.1", + "tensorflow==1.14", + "matplotlib", + "numba", + "pandas<=1.4.3", + "shap", + "tqdm", + ], +} + +# group algorithms those depend on tensorflow 1.14/x +extra_requires["tf_1x"] = extra_requires["profwt"] + extra_requires["shap"] + +# group algorithms those depend on tensorflow 2.x. +extra_requires[ + "tf_2x" +] = [] # This can be updated as algorithms upgrade to use tensorflow 2.x. + +# group algorithms those depend on pytorch. +extra_requires["torch"] = extra_requires["cofrnet"] + extra_requires["dipvae"] + +# group algorithms those depend on cvxpy. +extra_requires["cvxpy"] = ( + extra_requires["rbm"] + + extra_requires["rule_induction"] + + extra_requires["protodash"] +) + +# minimal dependencies in install_requires +install_requires = extra_requires["default"] # ted is supported by default. + setuptools.setup( - name='aix360', + name="aix360", version=version, - description='IBM AI Explainability 360', - authos='aix360 developers', - url='https://github.com/IBM/AIX360', - author_email='aix360@us.ibm.com', + description="IBM AI Explainability 360", + authos="aix360 developers", + url="https://github.com/IBM/AIX360", + author_email="aix360@us.ibm.com", packages=setuptools.find_packages(), - license='Apache License 2.0', - long_description=open('README.md', 'r', encoding='utf-8').read(), - long_description_content_type='text/markdown', - install_requires=[ - 'joblib>=0.11', - 'scikit-learn>=0.21.2', - 'torch', - 'torchvision', - 'cvxpy>==1.1', - 'Image', - 'tensorflow==1.14', - 'h5py<3.0.0', - 'keras==2.3.1', - 'matplotlib', - 'numpy', - 'pandas', - 'scipy>=0.17', - 'xport', - 'scikit-image', - 'requests', - 'xgboost==1.1.0', - 'bleach>=2.1.0', - 'docutils>=0.13.1', - 'Pygments', - 'osqp', - 'lime==0.1.1.37', - 'shap==0.34.0', - 'nyoka==5.2.0', - 'xmltodict==0.12.0', - 'numba', - 'tqdm', - 'kaggle', - 'otoc @ git+https://github.com/IBM/otoc@main#egg=otoc' - ], - package_data={'aix360': ['data/*', 'data/*/*', 'data/*/*/*', 'models/*', 'models/*/*', 'models/*/*/*']}, + license="Apache License 2.0", + long_description=open("README.md", "r", encoding="utf-8").read(), + long_description_content_type="text/markdown", + install_requires=install_requires, + extras_require=extra_requires, + package_data={ + "aix360": [ + "data/*", + "data/*/*", + "data/*/*/*", + "models/*", + "models/*/*", + "models/*/*/*", + ] + }, include_package_data=True, - zip_safe=False + zip_safe=False, ) From 1439993bac24197541d1d211e6b43f0b9d01f179 Mon Sep 17 00:00:00 2001 From: Giridhar Ganapavarapu Date: Mon, 1 May 2023 12:29:54 -0400 Subject: [PATCH 2/5] update build with parallel jobs for py 3.6 and 3.10 Signed-off-by: Giridhar Ganapavarapu --- .github/workflows/Build.yml | 162 ++++++++++++++++++++++++++++-------- setup.py | 6 +- 2 files changed, 130 insertions(+), 38 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 6f893f7..d7cbdd8 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -6,70 +6,160 @@ name: Build on: # Triggers the workflow on push or pull request events but only for the "master" branch push: - branches: [ "master" ] + branches: ["master"] pull_request: - branches: [ "master" ] + branches: ["master"] # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - - build-on-all: + # job to build algorithms that work on python 3.6 and tensorflow 1.14 + build-on-py36: # The type of runner that the job will run on - runs-on: '${{ matrix.os }}' + runs-on: "${{ matrix.os }}" strategy: matrix: #os: [ubuntu-18.04, ubuntu-latest, macos-latest, windows-latest] - os: [ubuntu-20.04, macos-latest, windows-latest] - python-version: ['3.6'] + os: [ubuntu-20.04, macos-latest, windows-latest] + python-version: ["3.6"] # Steps represent a sequence of tasks that will be executed as part of the job steps: - name: Step 1 - checkout aix360 repository uses: actions/checkout@v3 - + - name: Step 2 - set up python version uses: actions/setup-python@v4 with: - python-version: '${{ matrix.python-version }}' + python-version: "${{ matrix.python-version }}" - name: Step 3 - upgrade setuptools run: pip3 install pytest nbmake wheel --upgrade setuptools - - - name: Step 4 - Install aix360 + + - name: Step 4 - Install aix360 run: pip3 install . - - - name: Step 5 - Test CEM MAF + + - name: Step 5 - Test TED + run: python ./tests/ted/test_TED_Cartesian.py + + - name: Step 6 - Install aix360 with tensorflow 1.14 related dependencies + run: pip3 install .[tf_1x] + + - name: Step 7 - Test CEM MAF run: python ./tests/contrastive/test_CEM_MAF.py - - - name: Step 6 - Test CEM + + - name: Step 8 - Test CEM run: python ./tests/contrastive/test_CEM.py - - - name: Step 7 - Test Protodash - run: python ./tests/protodash/test_PDASH.py - - - name: Step 8 - Test TED + + # job to build algorithms that work on python 3.10 with default set of dependencies. + build-on-py310: + # The type of runner that the job will run on + runs-on: "${{ matrix.os }}" + strategy: + matrix: + #os: [ubuntu-18.04, ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-20.04, macos-latest, windows-latest] + python-version: ["3.10"] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: Step 1 - checkout aix360 repository + uses: actions/checkout@v3 + + - name: Step 2 - set up python version + uses: actions/setup-python@v4 + with: + python-version: "${{ matrix.python-version }}" + + - name: Step 3 - upgrade setuptools + run: pip3 install pytest nbmake wheel --upgrade setuptools + + - name: Step 4 - Install aix360 + run: pip3 install . + + - name: Step 5 - Test TED run: python ./tests/ted/test_TED_Cartesian.py - - - name: Step 9 - Test DIPVAE - run: python ./tests/dipvae/test_DIPVAE.py - - name: Step 10 - Test BRCG - run: python ./tests/rbm/test_Boolean_Rule_CG.py - - - name: Step 11 - Test Linear Rule Regression - run: python ./tests/rbm/test_Linear_Rule_Regression.py - - - name: Step 12 - Test Logistic Rule Regression - run: python ./tests/rbm/test_Logistic_Rule_Regression.py + - name: Step 6 - Install matching algorithm related dependencies + run: pip3 install .[matching] - - name: Step 13 - Test Matching Explainer + - name: Step 7 - Test Matching Explainer run: python ./tests/matching/test_order_constraints.py -# - name: Step 14 - Test Rule Induction -# run: bash ./tests/rule_induction/test_rule_induction.sh + # job to build algorithms that work on python 3.10 with tensorflow 2.x. + # build-tf-deps-on-py310: # this can be enabled when algorithms support tensorflow 2.x + + # job to build algorithms that work on python 3.10 with pytorch. + build-torch-deps-on-py310: + # The type of runner that the job will run on + runs-on: "${{ matrix.os }}" + strategy: + matrix: + #os: [ubuntu-18.04, ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-20.04, macos-latest, windows-latest] + python-version: ["3.10"] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: Step 1 - checkout aix360 repository + uses: actions/checkout@v3 + + - name: Step 2 - set up python version + uses: actions/setup-python@v4 + with: + python-version: "${{ matrix.python-version }}" + + - name: Step 3 - upgrade setuptools + run: pip3 install pytest nbmake wheel --upgrade setuptools + + - name: Step 4 - Install aix360 with pytorch related dependencies + run: pip3 install .[torch] + + - name: Step 5 - Test DIPVAE + run: python ./tests/dipvae/test_DIPVAE.py - - name: Step 15 - Test CoFrNet + - name: Step 6 - Test CoFrNet run: python ./tests/cofrnet/cofrnet_test.py - - name: Step 15b - Test CoFrNet Notebook + - name: Step 7 - Test CoFrNet Notebook run: pytest --nbmake ./examples/cofrnet/cofrnet_example.ipynb + + # job to build algorithms that work on python 3.10 with cvxpy. + build-cvxpy-deps-on-py310: + # The type of runner that the job will run on + runs-on: "${{ matrix.os }}" + strategy: + matrix: + #os: [ubuntu-18.04, ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-20.04, macos-latest, windows-latest] + python-version: ["3.10"] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: Step 1 - checkout aix360 repository + uses: actions/checkout@v3 + + - name: Step 2 - set up python version + uses: actions/setup-python@v4 + with: + python-version: "${{ matrix.python-version }}" + + - name: Step 3 - upgrade setuptools + run: pip3 install pytest nbmake wheel --upgrade setuptools + + - name: Step 4 - Install aix360 with cvxpy related dependencies + run: pip3 install .[cvxpy] + + - name: Step 5 - Test Protodash + run: python ./tests/protodash/test_PDASH.py + + - name: Step 6 - Test BRCG + run: python ./tests/rbm/test_Boolean_Rule_CG.py + + - name: Step 7 - Test Linear Rule Regression + run: python ./tests/rbm/test_Linear_Rule_Regression.py + + - name: Step 8 - Test Logistic Rule Regression + run: python ./tests/rbm/test_Logistic_Rule_Regression.py + + # - name: Step 14 - Test Rule Induction + # run: bash ./tests/rule_induction/test_rule_induction.sh diff --git a/setup.py b/setup.py index 5176b31..eee77a0 100644 --- a/setup.py +++ b/setup.py @@ -76,8 +76,10 @@ ], } -# group algorithms those depend on tensorflow 1.14/x -extra_requires["tf_1x"] = extra_requires["profwt"] + extra_requires["shap"] +# group algorithms those depend on tensorflow 1.14/x. This set works only on Python 3.6. +extra_requires["tf_1x"] = ( + extra_requires["profwt"] + extra_requires["shap"] + extra_requires["contrastive"] +) # group algorithms those depend on tensorflow 2.x. extra_requires[ From b75c02c7531a7626e64f0ba4a1008769743d9eec Mon Sep 17 00:00:00 2001 From: Giridhar Ganapavarapu Date: Mon, 1 May 2023 12:47:38 -0400 Subject: [PATCH 3/5] conditional imports in datasets init file. Signed-off-by: Giridhar Ganapavarapu --- aix360/datasets/__init__.py | 53 ++++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/aix360/datasets/__init__.py b/aix360/datasets/__init__.py index 7006033..acd8a4c 100644 --- a/aix360/datasets/__init__.py +++ b/aix360/datasets/__init__.py @@ -1,9 +1,44 @@ -from .mnist_dataset import MNISTDataset -from .meps_dataset import MEPSDataset -from .heloc_dataset import HELOCDataset -from .celeba_dataset import CelebADataset -from .cdc_dataset import CDCDataset -from .cifar_dataset import CIFARDataset -from .ted_dataset import TEDDataset -from .fashion_mnist_dataset import FMnistDataset -from .esnli_dataset import eSNLIDataset +try: + from .mnist_dataset import MNISTDataset +except: + pass + +try: + from .meps_dataset import MEPSDataset +except: + pass + +try: + from .heloc_dataset import HELOCDataset +except: + pass + +try: + from .celeba_dataset import CelebADataset +except: + pass + +try: + from .cdc_dataset import CDCDataset +except: + pass + +try: + from .cifar_dataset import CIFARDataset +except: + pass + +try: + from .ted_dataset import TEDDataset +except: + pass + +try: + from .fashion_mnist_dataset import FMnistDataset +except: + pass + +try: + from .esnli_dataset import eSNLIDataset +except: + pass From cdd1e620a7fb12f78805d5d55a4952129cbdc887 Mon Sep 17 00:00:00 2001 From: Giridhar Ganapavarapu Date: Mon, 1 May 2023 13:57:18 -0400 Subject: [PATCH 4/5] update contrastive deps to fix units Signed-off-by: Giridhar Ganapavarapu --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index eee77a0..41005ce 100644 --- a/setup.py +++ b/setup.py @@ -64,6 +64,9 @@ "tensorflow==1.14", "requests", "scipy>=0.17", + "scikit-image", + "torch", + "h5py<3.0.0", # to resolve keras error: 'str' object has no attribute 'decode' ], "shap": [ "keras==2.3.1", From 6d77ea1908e7329c8282ea04ab183642dbd82843 Mon Sep 17 00:00:00 2001 From: Giridhar Ganapavarapu Date: Wed, 3 May 2023 10:56:00 -0400 Subject: [PATCH 5/5] algorithm specific builds, removed grouped deps and builds Signed-off-by: Giridhar Ganapavarapu --- .github/workflows/Build.yml | 135 ++++++++++++++++++++++++++---------- setup.py | 20 ------ 2 files changed, 99 insertions(+), 56 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index d7cbdd8..52b5b43 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -12,8 +12,8 @@ on: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # job to build algorithms that work on python 3.6 and tensorflow 1.14 - build-on-py36: + # job to build CEM on python 3.6. + build-cem-on-py36: # The type of runner that the job will run on runs-on: "${{ matrix.os }}" strategy: @@ -35,23 +35,17 @@ jobs: - name: Step 3 - upgrade setuptools run: pip3 install pytest nbmake wheel --upgrade setuptools - - name: Step 4 - Install aix360 - run: pip3 install . - - - name: Step 5 - Test TED - run: python ./tests/ted/test_TED_Cartesian.py - - - name: Step 6 - Install aix360 with tensorflow 1.14 related dependencies - run: pip3 install .[tf_1x] + - name: Step 4 - Install aix360 with contrastive (CEM) + run: pip3 install .[contrastive] - - name: Step 7 - Test CEM MAF + - name: Step 5 - Test CEM MAF run: python ./tests/contrastive/test_CEM_MAF.py - - name: Step 8 - Test CEM + - name: Step 6 - Test CEM run: python ./tests/contrastive/test_CEM.py - # job to build algorithms that work on python 3.10 with default set of dependencies. - build-on-py310: + # job to build default: ted algorithm on python 3.10. + build-ted-on-py310: # The type of runner that the job will run on runs-on: "${{ matrix.os }}" strategy: @@ -73,23 +67,43 @@ jobs: - name: Step 3 - upgrade setuptools run: pip3 install pytest nbmake wheel --upgrade setuptools - - name: Step 4 - Install aix360 + - name: Step 4 - Install aix360. (Default installs ted algorithm related dependencies) run: pip3 install . - name: Step 5 - Test TED run: python ./tests/ted/test_TED_Cartesian.py - - name: Step 6 - Install matching algorithm related dependencies + # job to build matching algorithm on python 3.10. + build-matching-on-py310: + # The type of runner that the job will run on + runs-on: "${{ matrix.os }}" + strategy: + matrix: + #os: [ubuntu-18.04, ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-20.04, macos-latest, windows-latest] + python-version: ["3.10"] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: Step 1 - checkout aix360 repository + uses: actions/checkout@v3 + + - name: Step 2 - set up python version + uses: actions/setup-python@v4 + with: + python-version: "${{ matrix.python-version }}" + + - name: Step 3 - upgrade setuptools + run: pip3 install pytest nbmake wheel --upgrade setuptools + + - name: Step 4 - Install matching algorithm related dependencies run: pip3 install .[matching] - - name: Step 7 - Test Matching Explainer + - name: Step 5 - Test Matching Explainer run: python ./tests/matching/test_order_constraints.py - # job to build algorithms that work on python 3.10 with tensorflow 2.x. - # build-tf-deps-on-py310: # this can be enabled when algorithms support tensorflow 2.x - - # job to build algorithms that work on python 3.10 with pytorch. - build-torch-deps-on-py310: + # job to build dipvae algorithm on python 3.10. + build-dipvae-on-py310: # The type of runner that the job will run on runs-on: "${{ matrix.os }}" strategy: @@ -111,20 +125,46 @@ jobs: - name: Step 3 - upgrade setuptools run: pip3 install pytest nbmake wheel --upgrade setuptools - - name: Step 4 - Install aix360 with pytorch related dependencies - run: pip3 install .[torch] + - name: Step 4 - Install aix360 with dipvae algorithm related dependencies + run: pip3 install .[dipvae] - name: Step 5 - Test DIPVAE run: python ./tests/dipvae/test_DIPVAE.py - - name: Step 6 - Test CoFrNet + # job to build cofrnet algorithm on python 3.10. + build-cofrnet-on-py310: + # The type of runner that the job will run on + runs-on: "${{ matrix.os }}" + strategy: + matrix: + #os: [ubuntu-18.04, ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-20.04, macos-latest, windows-latest] + python-version: ["3.10"] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: Step 1 - checkout aix360 repository + uses: actions/checkout@v3 + + - name: Step 2 - set up python version + uses: actions/setup-python@v4 + with: + python-version: "${{ matrix.python-version }}" + + - name: Step 3 - upgrade setuptools + run: pip3 install pytest nbmake wheel --upgrade setuptools + + - name: Step 4 - Install aix360 with cofrnet algorithm related dependencies + run: pip3 install .[cofrnet] + + - name: Step 5 - Test CoFrNet run: python ./tests/cofrnet/cofrnet_test.py - - name: Step 7 - Test CoFrNet Notebook + - name: Step 6 - Test CoFrNet Notebook run: pytest --nbmake ./examples/cofrnet/cofrnet_example.ipynb - # job to build algorithms that work on python 3.10 with cvxpy. - build-cvxpy-deps-on-py310: + # job to build protodash algorithm on python 3.10. + build-protodash-on-py310: # The type of runner that the job will run on runs-on: "${{ matrix.os }}" strategy: @@ -146,20 +186,43 @@ jobs: - name: Step 3 - upgrade setuptools run: pip3 install pytest nbmake wheel --upgrade setuptools - - name: Step 4 - Install aix360 with cvxpy related dependencies - run: pip3 install .[cvxpy] + - name: Step 4 - Install aix360 with protodash algorithm related dependencies + run: pip3 install .[protodash] - name: Step 5 - Test Protodash run: python ./tests/protodash/test_PDASH.py - - name: Step 6 - Test BRCG + # job to build rbm algorithm on python 3.10. + build-rbm-on-py310: + # The type of runner that the job will run on + runs-on: "${{ matrix.os }}" + strategy: + matrix: + #os: [ubuntu-18.04, ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-20.04, macos-latest, windows-latest] + python-version: ["3.10"] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: Step 1 - checkout aix360 repository + uses: actions/checkout@v3 + + - name: Step 2 - set up python version + uses: actions/setup-python@v4 + with: + python-version: "${{ matrix.python-version }}" + + - name: Step 3 - upgrade setuptools + run: pip3 install pytest nbmake wheel --upgrade setuptools + + - name: Step 4 - Install aix360 with rbm algorithm related dependencies + run: pip3 install .[rbm] + + - name: Step 5 - Test BRCG run: python ./tests/rbm/test_Boolean_Rule_CG.py - - name: Step 7 - Test Linear Rule Regression + - name: Step 6 - Test Linear Rule Regression run: python ./tests/rbm/test_Linear_Rule_Regression.py - - name: Step 8 - Test Logistic Rule Regression + - name: Step 7 - Test Logistic Rule Regression run: python ./tests/rbm/test_Logistic_Rule_Regression.py - - # - name: Step 14 - Test Rule Induction - # run: bash ./tests/rule_induction/test_rule_induction.sh diff --git a/setup.py b/setup.py index 41005ce..bc69405 100644 --- a/setup.py +++ b/setup.py @@ -79,26 +79,6 @@ ], } -# group algorithms those depend on tensorflow 1.14/x. This set works only on Python 3.6. -extra_requires["tf_1x"] = ( - extra_requires["profwt"] + extra_requires["shap"] + extra_requires["contrastive"] -) - -# group algorithms those depend on tensorflow 2.x. -extra_requires[ - "tf_2x" -] = [] # This can be updated as algorithms upgrade to use tensorflow 2.x. - -# group algorithms those depend on pytorch. -extra_requires["torch"] = extra_requires["cofrnet"] + extra_requires["dipvae"] - -# group algorithms those depend on cvxpy. -extra_requires["cvxpy"] = ( - extra_requires["rbm"] - + extra_requires["rule_induction"] - + extra_requires["protodash"] -) - # minimal dependencies in install_requires install_requires = extra_requires["default"] # ted is supported by default.