Merge pull request #775 from Trusted-AI/dev_1.5.0

Update to ART 1.5.0

.github/dependabot.yml

@@ -11,3 +11,9 @@ updates:
       interval: "daily"
     assignees:
       - "beat-buesser"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    assignees:
+      - "beat-buesser"

.github/workflows/ci.yml

@@ -25,32 +25,108 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - python: 3.6
-            tensorflow: 1.15.3
+          - name: TensorFlow 1.15.4 (Keras 2.2.5 Python 3.7)
+            framework: tensorflow
+            python: 3.7
+            tensorflow: 1.15.4
+            tf_version: v1
             keras: 2.2.5
-          - python: 3.7
+          - name: TensorFlow 2.2.0 (Keras 2.3.1 Python 3.7)
+            framework: tensorflow
+            python: 3.7
             tensorflow: 2.2.0
+            tf_version: v2
             keras: 2.3.1
-    name: Run Tests (Python ${{ matrix.python }})
+          - name: TensorFlow 2.2.0v1 (Keras 2.3.1 Python 3.7)
+            framework: tensorflow2v1
+            python: 3.7
+            tensorflow: 2.2.0
+            tf_version: v2
+            keras: 2.3.1
+          - name: TensorFlow 2.3.1 (Keras 2.4.3 Python 3.7)
+            framework: tensorflow
+            python: 3.7
+            tensorflow: 2.3.1
+            tf_version: v2
+            keras: 2.4.3
+          - name: TensorFlow 2.4.0rc3 (Keras 2.4.3 Python 3.8)
+            framework: tensorflow
+            python: 3.8
+            tensorflow: 2.4.0rc3
+            tf_version: v2
+            keras: 2.4.3
+          - name: Keras 2.3.1 (TensorFlow 2.2.1 Python 3.7)
+            framework: keras
+            python: 3.7
+            tensorflow: 2.2.1
+            keras: 2.3.1
+          - name: TensorFlow-Keras 2.3.1 (Keras 2.4.3 Python 3.7)
+            framework: kerastf
+            python: 3.7
+            tensorflow: 2.3.1
+            keras: 2.4.3
+          - name: PyTorch (Python 3.7)
+            framework: pytorch
+            python: 3.7
+          - name: scikit-learn 0.22.2 (Python 3.7)
+            framework: scikitlearn
+            scikit-learn: 0.22.2
+            python: 3.7
+          - name: scikit-learn 0.23.2 (Python 3.8)
+            framework: scikitlearn
+            scikit-learn: 0.23.2
+            python: 3.8
+          - name: mxnet (Python 3.7)
+            framework: mxnet
+            python: 3.7
+          - name: legacy (TensorFlow 2.2.1 Keras 2.3.1 scikit-learn 0.22.2 Python 3.7)
+            framework: legacy
+            python: 3.7
+            tensorflow: 2.2.1
+            keras: 2.3.1
+            scikit-learn: 0.22.2
+          - name: legacy (TensorFlow 2.3.1 Keras 2.4.3 scikit-learn 0.23.2 Python 3.8)
+            framework: legacy
+            python: 3.8
+            tensorflow: 2.3.1
+            keras: 2.4.3
+            scikit-learn: 0.23.2
+
+    name: Run ${{ matrix.name }} Tests
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v2
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}
-      - name: Pre-install
-        run: |
-          sudo apt-get -y -q install ffmpeg libavcodec-extra
-          sed -i "s/^tensorflow$/tensorflow==${{ matrix.tensorflow }}/" requirements.txt
-          sed -i "s/^keras$/keras==${{ matrix.keras }}/" requirements.txt
       - name: Install Dependencies
         run: |
+          sudo apt-get update
+          sudo apt-get -y -q install ffmpeg libavcodec-extra
           python -m pip install --upgrade pip setuptools wheel
           pip3 install -q -r requirements.txt
           pip list
-      - name: Run Tests
-        run: ./run_tests.sh
+      - name: Pre-install legacy
+        if: ${{ matrix.framework == 'legacy' }}
+        run: |
+          pip install tensorflow==${{ matrix.tensorflow }}
+          pip install keras==${{ matrix.keras }}
+          pip install scikit-learn==${{ matrix.scikit-learn }}
+          pip list
+      - name: Pre-install tensorflow
+        if: ${{ matrix.framework == 'tensorflow' || matrix.framework == 'keras' || matrix.framework == 'kerastf' }}
+        run: |
+          pip install tensorflow==${{ matrix.tensorflow }}
+          pip install keras==${{ matrix.keras }}
+          pip list
+      - name: Pre-install scikit-learn
+        if: ${{ matrix.framework == 'scikitlearn' }}
+        run: |
+          pip install scikit-learn==${{ matrix.scikit-learn }}
+          pip list
+      - name: Run ${{ matrix.name }} Tests
+        run: ./run_tests.sh ${{ matrix.framework }}
   style:
     name: Style Check
     runs-on: ubuntu-latest
@@ -63,9 +139,10 @@ jobs:
           python-version: 3.7
       - name: Pre-install
         run: |
+          sudo apt-get update
           sudo apt-get -y -q install ffmpeg libavcodec-extra
-          sed -i "s/^tensorflow$/tensorflow==2.2.0/" requirements.txt
-          sed -i "s/^keras$/keras==2.3.1/" requirements.txt
+          pip install tensorflow==2.2.0
+          pip install keras==2.3.1
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel

.gitignore

@@ -113,3 +113,4 @@ demo/pics/*
 !notebooks/*.ipynb
 !notebooks/adaptive_defence_evaluations/*.ipynb
 !notebooks/adversarial_patch/*.ipynb
+!notebooks/art_evaluations/*.ipynb

CODE_OF_CONDUCT.md

@@ -1,5 +1,5 @@
-The Adversarial Robustness Toolbox is dedicated to providing a harassment-free experience for everyone, regardless of gender, gender identity and expression, sexual orientation, disability, physical appearance, body size, age, race, or religion. We do not tolerate harassment of participants in any form.
+The Adversarial Robustness Toolbox (ART) is dedicated to providing a harassment-free experience for everyone, regardless of gender, gender identity and expression, sexual orientation, disability, physical appearance, body size, age, race, or religion. We do not tolerate harassment of participants in any form.
 
-This code of conduct applies to all Adversarial Robustness Toolbox spaces, both online and off. Anyone who violates this code of conduct may be sanctioned or expelled from these spaces at the discretion of the IBM Research AI team.
+This code of conduct applies to all Adversarial Robustness Toolbox spaces, both online and off. Anyone who violates this code of conduct may be sanctioned or expelled from these spaces at the discretion of the Trusted-AI team.
 
 We may add additional rules over time, which will be made clearly available to participants. Participants are responsible for knowing and abiding by these rules.

Dockerfile

@@ -4,14 +4,23 @@ RUN pip3 install keras==2.3.1
 #FROM tensorflow/tensorflow:1.15.2
 #RUN pip3 install keras==2.2.5
 
-RUN pip3 install numpy==1.19.1 scipy==1.4.1 matplotlib==3.3.1 scikit-learn==0.22.2 six==1.15.0 Pillow==7.2.0
+RUN pip3 install numpy==1.19.1 scipy==1.4.1 matplotlib==3.3.1 scikit-learn==0.22.2 six==1.15.0 Pillow==7.2.0 pytest-cov==2.10.1
 RUN pip3 install tqdm==4.48.2 statsmodels==0.11.1 pydub==0.24.1 resampy==0.2.2 ffmpeg-python==0.2.0 cma==3.0.3 mypy==0.770
+RUN pip3 install ffmpeg-python==0.2.0
 RUN pip3 install pandas==1.1.1
 
 #TODO check if jupyter notebook works
 RUN pip3 install jupyter==1.0.0 && pip3 install jupyterlab==2.1.0
 # https://stackoverflow.com/questions/49024624/how-to-dockerize-jupyter-lab
 
+# Lingvo ASR dependencies
+# supported versions: (lingvo==0.6.4 with tensorflow-gpu==2.1.0)
+# note: due to conflicts with other TF1/2 version supported by ART, the dependencies are not installed by default:
+# Replace line 1 with: FROM tensorflow/tensorflow:2.1.0
+# Comment other TF related lines and uncomment:
+# RUN pip3 install tensorflow-gpu==2.1.0
+# RUN pip3 install lingvo==0.6.4
+
 RUN pip3 install h5py==2.10.0
 RUN pip3 install tensorflow-addons==0.11.1
 RUN pip3 install mxnet==1.6.0
@@ -20,6 +29,7 @@ RUN pip3 install catboost==0.24
 RUN pip3 install GPy==1.9.9
 RUN pip3 install lightgbm==2.3.1
 RUN pip3 install xgboost==1.1.1
+RUN pip3 install kornia==0.3.1
 
 RUN pip3 install pytest==5.4.1 pytest-pep8==1.0.6 pytest-mock==3.2.0 codecov==2.1.8 requests==2.24.0
 
@@ -31,6 +41,9 @@ WORKDIR /project
 ADD . /project/
 RUN pip3 install --upgrade -r /project/requirements.txt
 
+RUN apt-get update
+RUN apt-get -y -q install ffmpeg libavcodec-extra
+
 RUN echo "You should think about possibly upgrading these outdated packages"
 RUN pip3 list --outdated
 

Makefile

@@ -1,13 +1,13 @@
 PROJECT_HOME_DIR := ${CURDIR}
 
 build:
-    # Builds a Tensorflow 2 ART docker container
+    # Builds a TensorFlow 2 ART docker container
     # IMPORTANT ! If you have an existing python env folder make sure to first add it to the `.dockerIgnore` \
     to reduce the size of your the art docker image
 	docker build -t project-art-tf2 .
 
 build1:
-	# Builds a Tensorflow 1 ART docker container
+	# Builds a TensorFlow 1 ART docker container
     # IMPORTANT ! If you have an existing python env folder make sure to first add it to the `.dockerIgnore` \
     to reduce the size of your the art docker image
 	docker build -t project-art-tf1 .

README-cn.md

@@ -4,7 +4,8 @@
 </p>
 <br />
 
-[![Build Status](https://travis-ci.com/Trusted-AI/adversarial-robustness-toolbox.svg?branch=main)](https://travis-ci.com/Trusted-AI/adversarial-robustness-toolbox)
+![Continuous Integration](https://github.com/Trusted-AI/adversarial-robustness-toolbox/workflows/Continuous%20Integration/badge.svg)
+![CodeQL](https://github.com/Trusted-AI/adversarial-robustness-toolbox/workflows/CodeQL/badge.svg)
 [![Documentation Status](https://readthedocs.org/projects/adversarial-robustness-toolbox/badge/?version=latest)](http://adversarial-robustness-toolbox.readthedocs.io/en/latest/?badge=latest)
 [![GitHub version](https://badge.fury.io/gh/Trusted-AI%2Fadversarial-robustness-toolbox.svg)](https://badge.fury.io/gh/Trusted-AI%2Fadversarial-robustness-toolbox)
 [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/Trusted-AI/adversarial-robustness-toolbox.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/Trusted-AI/adversarial-robustness-toolbox/context:python)

README.md

@@ -4,7 +4,8 @@
 </p>
 <br />
 
-[![Build Status](https://travis-ci.com/Trusted-AI/adversarial-robustness-toolbox.svg?branch=main)](https://travis-ci.com/Trusted-AI/adversarial-robustness-toolbox)
+![Continuous Integration](https://github.com/Trusted-AI/adversarial-robustness-toolbox/workflows/Continuous%20Integration/badge.svg)
+![CodeQL](https://github.com/Trusted-AI/adversarial-robustness-toolbox/workflows/CodeQL/badge.svg)
 [![Documentation Status](https://readthedocs.org/projects/adversarial-robustness-toolbox/badge/?version=latest)](http://adversarial-robustness-toolbox.readthedocs.io/en/latest/?badge=latest)
 [![GitHub version](https://badge.fury.io/gh/Trusted-AI%2Fadversarial-robustness-toolbox.svg)](https://badge.fury.io/gh/Trusted-AI%2Fadversarial-robustness-toolbox)
 [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/Trusted-AI/adversarial-robustness-toolbox.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/Trusted-AI/adversarial-robustness-toolbox/context:python)

art/attacks/__init__.py

@@ -3,6 +3,7 @@
 """
 from art.attacks.attack import Attack, EvasionAttack, PoisoningAttack, PoisoningAttackBlackBox, PoisoningAttackWhiteBox
 from art.attacks.attack import PoisoningAttackTransformer, ExtractionAttack, InferenceAttack, AttributeInferenceAttack
+from art.attacks.attack import ReconstructionAttack
 
 from art.attacks import evasion
 from art.attacks import extraction

art/attacks/attack.py

@@ -317,7 +317,7 @@ class AttributeInferenceAttack(InferenceAttack):
 
     attack_params = InferenceAttack.attack_params + ["attack_feature"]
 
-    def __init__(self, estimator, attack_feature: int = 0):
+    def __init__(self, estimator, attack_feature: Union[int, slice] = 0):
         """
         :param estimator: A trained estimator targeted for inference attack.
         :type estimator: :class:`.art.estimators.estimator.BaseEstimator`
@@ -346,6 +346,36 @@ def set_params(self, **kwargs) -> None:
         super().set_params(**kwargs)
         self._check_params()
 
-    def _check_params(self) -> None:
-        if self.attack_feature < 0:
-            raise ValueError("Attack feature must be positive.")
+
+class ReconstructionAttack(Attack):
+    """
+    Abstract base class for reconstruction attack classes.
+    """
+
+    attack_params = InferenceAttack.attack_params
+
+    def __init__(self, estimator):
+        """
+        :param estimator: A trained estimator targeted for reconstruction attack.
+        """
+        super().__init__(estimator)
+
+    @abc.abstractmethod
+    def reconstruct(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> Tuple[np.ndarray, np.ndarray]:
+        """
+        Reconstruct the training dataset of and from the targeted estimator. This method
+        should be overridden by all concrete inference attack implementations.
+
+        :param x: An array with known records of the training set of `estimator`.
+        :param y: An array with known labels of the training set of `estimator`, if None predicted labels will be used.
+        :return: A tuple of two arrays for the reconstructed training input and labels.
+        """
+        raise NotImplementedError
+
+    def set_params(self, **kwargs) -> None:
+        """
+        Take in a dictionary of parameters and applies attack-specific checks before saving them as attributes.
+        """
+        # Save attack-specific parameters
+        super().set_params(**kwargs)
+        self._check_params()

art/attacks/evasion/__init__.py

@@ -4,16 +4,26 @@
 from art.attacks.evasion.adversarial_patch.adversarial_patch import AdversarialPatch
 from art.attacks.evasion.adversarial_patch.adversarial_patch_numpy import AdversarialPatchNumpy
 from art.attacks.evasion.adversarial_patch.adversarial_patch_tensorflow import AdversarialPatchTensorFlowV2
+from art.attacks.evasion.auto_attack import AutoAttack
+from art.attacks.evasion.auto_projected_gradient_descent import AutoProjectedGradientDescent
+from art.attacks.evasion.brendel_bethge import BrendelBethgeAttack
 from art.attacks.evasion.boundary import BoundaryAttack
 from art.attacks.evasion.carlini import CarliniL2Method, CarliniLInfMethod
 from art.attacks.evasion.decision_tree_attack import DecisionTreeAttack
 from art.attacks.evasion.deepfool import DeepFool
+from art.attacks.evasion.dpatch import DPatch
+from art.attacks.evasion.dpatch_robust import RobustDPatch
 from art.attacks.evasion.elastic_net import ElasticNet
 from art.attacks.evasion.fast_gradient import FastGradientMethod
+from art.attacks.evasion.frame_saliency import FrameSaliencyAttack
+from art.attacks.evasion.feature_adversaries import FeatureAdversaries
 from art.attacks.evasion.hclu import HighConfidenceLowUncertainty
 from art.attacks.evasion.hop_skip_jump import HopSkipJump
+from art.attacks.evasion.imperceptible_asr.imperceptible_asr import ImperceptibleASR
+from art.attacks.evasion.imperceptible_asr.imperceptible_asr_pytorch import ImperceptibleASRPyTorch
 from art.attacks.evasion.iterative_method import BasicIterativeMethod
 from art.attacks.evasion.newtonfool import NewtonFool
+from art.attacks.evasion.pixel_threshold import PixelAttack
 from art.attacks.evasion.projected_gradient_descent.projected_gradient_descent import ProjectedGradientDescent
 from art.attacks.evasion.projected_gradient_descent.projected_gradient_descent_numpy import (
     ProjectedGradientDescentNumpy,
@@ -25,21 +35,14 @@
     ProjectedGradientDescentTensorFlowV2,
 )
 from art.attacks.evasion.saliency_map import SaliencyMapMethod
+from art.attacks.evasion.shadow_attack import ShadowAttack
+from art.attacks.evasion.shapeshifter import ShapeShifter
+from art.attacks.evasion.simba import SimBA
 from art.attacks.evasion.spatial_transformation import SpatialTransformation
+from art.attacks.evasion.square_attack import SquareAttack
+from art.attacks.evasion.pixel_threshold import ThresholdAttack
 from art.attacks.evasion.universal_perturbation import UniversalPerturbation
 from art.attacks.evasion.targeted_universal_perturbation import TargetedUniversalPerturbation
 from art.attacks.evasion.virtual_adversarial import VirtualAdversarialMethod
 from art.attacks.evasion.wasserstein import Wasserstein
 from art.attacks.evasion.zoo import ZooAttack
-from art.attacks.evasion.pixel_threshold import PixelAttack
-from art.attacks.evasion.pixel_threshold import ThresholdAttack
-from art.attacks.evasion.frame_saliency import FrameSaliencyAttack
-from art.attacks.evasion.feature_adversaries import FeatureAdversaries
-from art.attacks.evasion.dpatch import DPatch
-from art.attacks.evasion.shadow_attack import ShadowAttack
-from art.attacks.evasion.auto_attack import AutoAttack
-from art.attacks.evasion.auto_projected_gradient_descent import AutoProjectedGradientDescent
-from art.attacks.evasion.square_attack import SquareAttack
-from art.attacks.evasion.simba import SimBA
-from art.attacks.evasion.shapeshifter import ShapeShifter
-from art.attacks.evasion.imperceptible_asr.imperceptible_asr_pytorch import ImperceptibleASRPytorch

art/attacks/evasion/adversarial_patch/adversarial_patch.py

@@ -127,6 +127,10 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> T
 
         :param x: An array with the original input images of shape NHWC or NCHW or input videos of shape NFHWC or NFCHW.
         :param y: An array with the original true labels.
+        :param mask: An boolean array of shape equal to the shape of a single samples (1, H, W) or the shape of `x`
+                     (N, H, W) without their channel dimensions. Any features for which the mask is True can be the
+                     center location of the patch during sampling.
+        :type mask: `np.ndarray`
         :return: An array with adversarial patch and an array of the patch mask.
         """
         logger.info("Creating adversarial patch.")