Bump types-python-dateutil from 2.8.19.7 to 2.8.19.10

.circleci/config.yml

@@ -14,41 +14,6 @@ commands:
           command: |
             echo 'export INSTANCE_ROLE="<< parameters.instance_role >>"' >> $BASH_ENV
             echo 'export INSTANCE_WAS_CREATED="<< parameters.instance_created >>"' >> $BASH_ENV
-  start-tunnel:
-    description: "Opens an ssh tunnel to the demisto servers and wait until the tunnels are established"
-    parameters:
-      timeout:
-        type: integer
-        default: 10
-    steps:
-      - run:
-          name: add ssh configurations
-          shell: /bin/bash
-          command: |
-            if [ -z $INSTANCE_WAS_CREATED ];
-              then
-                echo "Skipping - instance was not created"
-                exit 0
-            fi
-            # Modifying ssh config file
-            echo "Host 10.0.*
-              StrictHostKeyChecking no
-              LogLevel ERROR
-              ProxyJump [email protected]  # disable-secrets-detection
-             Host content-build-lb.demisto.works
-              Port 43567
-              UserKnownHostsFile /dev/null
-              StrictHostKeyChecking no
-              LogLevel ERROR" >> ~/.ssh/config
-      - run:
-          name: Open SSH Tunnel
-          command: |
-            if [ -z $INSTANCE_WAS_CREATED ];
-              then
-                echo "Skipping - instance was not created"
-                exit 0
-            fi
-            ./Tests/scripts/open_ssh_tunnel.sh
 
 parameters:
   artifact_build_num:
@@ -91,6 +56,31 @@ references:
       NIGHTLY_PARAMETER: << pipeline.parameters.nightly >>
       GCS_MARKET_BUCKET: << pipeline.parameters.gcs_market_bucket >>
 
+  install_neo4j: &install_neo4j
+    run:
+      name: Install Neo4j
+      command: |
+        # workaround for 402 Payment required issue: https://github.com/actions/runner-images/issues/1983
+        sudo rm -fv /etc/apt/sources.list.d/github_git-lfs.list
+        sudo rm -fv /etc/apt/sources.list.d/github_git-lfs.list.save
+        wget -O - https://debian.neo4j.com/neotechnology.gpg.key | sudo apt-key add -
+        echo 'deb https://debian.neo4j.com stable 4.4' | sudo tee /etc/apt/sources.list.d/neo4j.list
+        sudo apt-get update
+        apt list -a neo4j
+        sudo apt-get install neo4j
+        sudo chown -R circleci /var/log/neo4j
+        sudo chown -R circleci /var/lib/neo4j
+        sudo chown -R circleci /etc/neo4j
+        mkdir -p /var/lib/neo4j/plugins
+        wget -O /var/lib/neo4j/plugins/apoc-4.4.0.8-all.jar https://github.com/neo4j-contrib/neo4j-apoc-procedures/releases/download/4.4.0.8/apoc-4.4.0.8-all.jar
+        neo4j_conf_file="/etc/neo4j/neo4j.conf"
+        sudo echo "dbms.security.procedures.unrestricted=apoc.*" >> $neo4j_conf_file
+        sudo echo "dbms.security.procedures.allowlist=apoc.*" >> $neo4j_conf_file
+        apoc_conf_file="/etc/neo4j/apoc.conf"
+        sudo echo "apoc.export.file.enabled=true" > $apoc_conf_file
+        sudo echo "apoc.import.file.enabled=true" >> $apoc_conf_file
+        sudo echo "apoc.import.file.use_neo4j_config=true" >> $apoc_conf_file
+        neo4j-admin set-initial-password test
 
   container_config: &container_config
     docker:
@@ -112,19 +102,12 @@ references:
     attach_workspace:
       at: *workspace_root
 
-  add_ssh_keys: &add_ssh_keys
-    add_ssh_keys:
-      fingerprints:
-        - "02:df:a5:6a:53:9a:f5:5d:bd:a6:fc:b2:db:9b:c9:47" # disable-secrets-detection
-        - "f5:25:6a:e5:ac:4b:84:fb:60:54:14:82:f1:e9:6c:f9" # disable-secrets-detection
-
   prepare_environment: &prepare_environment
     run:
       name: Prepare Environment
       when: always
       command: |
         poetry --version
-
         # Check if CircleCI's config file and poetry files files are up to date
         # if poetry isn't up-to-date, checkout from origin/master.
         ./Tests/scripts/is_file_up_to_date.sh .circleci/config.yml $CIRCLE_BRANCH
@@ -210,12 +193,19 @@ references:
           exit 0
         fi
 
+        neo4j start
+        # poll for neo4j status until available
+        while ! neo4j status; do sleep 1; done
+
         ./Tests/scripts/linters_runner.sh
         ./Tests/scripts/validate.sh
 
   run_unit_testing_and_lint: &run_unit_testing_and_lint
     run:
-      name: Run Unit Testing and Lint
+      parameters:
+        dockerimageflag:
+          type: string
+      name: Run Unit Testing And Lint - Docker Image:<< parameters.dockerimageflag >>
       when: always
       no_output_timeout: 5h
       command: |
@@ -233,7 +223,7 @@ references:
         echo "vulture py2 version: $(python2 -m vulture --version 2>&1)"
         echo "vulture py3 version: $(python3 -m vulture --version 2>&1)"
         mkdir ./unit-tests
-        demisto-sdk lint -p 8 -g -vvv --test-xml ./unit-tests --log-path ./artifacts --failure-report ./artifacts --coverage-report $ARTIFACTS_FOLDER/coverage_report
+        demisto-sdk lint -p 8 -g -vvv --test-xml ./unit-tests --log-path ./artifacts --failure-report ./artifacts --coverage-report $ARTIFACTS_FOLDER/coverage_report --docker-image << parameters.dockerimageflag >>
 
   generate_coverage_reports: &generate_coverage_reports
     run:
@@ -267,32 +257,6 @@ references:
           ./Tests/scripts/sdk_pylint_check.sh
         fi
 
-  create_id_set: &create_id_set
-    run:
-      name: Create ID Set
-      when: always
-      command: |
-        demisto-sdk create-id-set -o ./Tests/id_set.json --fail-duplicates
-        cp ./Tests/id_set.json $CIRCLE_ARTIFACTS
-
-  merge_public_and_private_id_sets: &merge_public_and_private_id_sets
-    run:
-      name: Merge public and private ID sets
-      when: always
-      command: |
-        if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then
-            echo "Skipping, Should not run on contributor's branch."
-            exit 0
-        fi
-
-        # Download private ID set
-        gsutil cp gs://marketplace-dist/content/private_id_set.json $CIRCLE_ARTIFACTS/unified_id_set.json
-        echo "successfully downloaded private ID set"
-
-        # Merge public and private ID sets
-        demisto-sdk merge-id-sets -i1 ./Tests/id_set.json -i2 $CIRCLE_ARTIFACTS/unified_id_set.json -o $CIRCLE_ARTIFACTS/unified_id_set.json
-        echo "successfully merged public and private ID sets"
-
   get_contribution_pack: &get_contribution_pack
     when:
       condition: << pipeline.parameters.contrib_branch >>
@@ -323,6 +287,10 @@ references:
         context: nightly_env
         requires:
           - Setup Environment
+        matrix:
+          parameters:
+            dockerimageflag: [ "native:ga", "native:maintenance", "native:dev", "from-yml" ]
+        name: Run Unit Testing And Lint - Docker Image:<< matrix.dockerimageflag >>
     - Run Validations:
         requires:
           - Setup Environment
@@ -343,19 +311,21 @@ jobs:
             - node_modules
           key: virtualenv-venv-{{ checksum "pyproject.toml" }}-{{ checksum "poetry.lock" }}-{{ checksum "package-lock.json" }}
       - *get_contribution_pack
-      - *add_ssh_keys
       - *persist_to_workspace
 
   Run Unit Testing And Lint:
     <<: *container_config
     resource_class: medium
     <<: *environment
+    parameters:
+      dockerimageflag:
+        type: string
     steps:
       - *attach_workspace
       - *remote_docker
       - *restore_cache
-      - *add_ssh_keys
       - *prepare_environment
+      - *install_neo4j
       - *infrastructure_testing
       - *run_unit_testing_and_lint
       - *generate_coverage_reports
@@ -365,16 +335,14 @@ jobs:
 
   Run Validations:
     <<: *container_config
-    resource_class: medium
+    resource_class: large
     <<: *environment
     steps:
       - *attach_workspace
       - *restore_cache
-      - *add_ssh_keys
       - *prepare_environment
+      - *install_neo4j
       - *secrets
-      - *create_id_set
-      - *merge_public_and_private_id_sets
       - *validate_files_and_yaml
       - run:
           name: Spell Checks
@@ -417,6 +385,10 @@ workflows:
       - Run Unit Testing And Lint:
           requires:
             - Setup Environment
+          matrix:
+            parameters:
+              dockerimageflag: [ "native:ga", "native:maintenance", "native:dev", "from-yml" ]
+          name: Run Unit Testing And Lint - Docker Image:<< matrix.dockerimageflag >>
       - Run Validations:
           requires:
             - Setup Environment
@@ -437,5 +409,4 @@ workflows:
     # will initiate when using the trigger script.
     when: << pipeline.parameters.nightly >>
     jobs:
-      *nightly_jobs
-
+      *nightly_jobs

.circleci/gitlab-ci-env-variables.sh

@@ -2,6 +2,7 @@ echo 'export CI_BUILD_ID="$CIRCLE_BUILD_NUM"' >> $BASH_ENV
 echo 'export CI_PIPELINE_ID="$CIRCLE_WORKFLOW_ID"' >> $BASH_ENV
 echo 'export CI_COMMIT_BRANCH="$CIRCLE_BRANCH"' >> $BASH_ENV
 echo 'export ARTIFACTS_FOLDER=/home/circleci/project/artifacts' >> $BASH_ENV
+echo 'export PIPELINE_JOBS_FOLDER=/home/circleci/project/pipeline_jobs_folder' >> $BASH_ENV
 echo 'export CI_COMMIT_SHA="$CIRCLE_SHA1"' >> $BASH_ENV
 echo 'export CI_JOB_URL="$CIRCLE_BUILD_URL"' >> $BASH_ENV
 echo 'export CI_JOB_NAME="$CIRCLE_JOB"' >> $BASH_ENV

.devcontainer/Dockerfile

@@ -1,34 +1,35 @@
 FROM python:3.10-slim-bullseye
 
-ADD createCerts.sh .
-RUN apt-get update && apt-get install dos2unix -y \
-    && dos2unix /createCerts.sh \
-    && chmod +x /createCerts.sh \ 
-    && /createCerts.sh /usr/local/share/ca-certificates/certs.crt \
-    && update-ca-certificates \
-    && apt-get install python2 -y
-
 ENV USERNAME demisto
 ENV HOME /home/$USERNAME
 ENV NODE_EXTRA_CA_CERTS /usr/local/share/ca-certificates/certs.crt
 ENV PATH $PATH:$HOME/.local/bin:/root/.local/bin:/usr/local/share/nvm/current/bin
 
-# This is a workaround for VSCode devcontainer features in self signed certificate
-
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/common-utils/install.sh /install-common.sh
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/git/install.sh /install-git.sh
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/docker-in-docker/install.sh /install-dind.sh
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/node/install.sh /install-node.sh
+ADD createCerts.sh .
+RUN apt-get update && apt-get install dos2unix git python2 curl -y \
+    && dos2unix /createCerts.sh \
+    && chmod +x /createCerts.sh \ 
+    && /createCerts.sh $NODE_EXTRA_CA_CERTS \
+    && update-ca-certificates \
+    && rm -rf /features \
+    && git clone https://github.com/devcontainers/features.git /features \
+    && cd /features \
+    # locking to the latest master commit in this repo to prevent breaking changes
+    # We should update this commit hash from time to time to
+    && git checkout fbdc4556d519512736a8e2abfb3e03fcb2c9e0c7
 
-RUN UID="1000" GID="1000" bash install-common.sh 
-RUN VERSION="os-provided" bash install-git.sh
-RUN VERSION="latest" bash install-dind.sh
-RUN VERSION="lts" bash install-node.sh
+# This is a workaround for VSCode devcontainer features in self signed certificate
+RUN UID="1000" GID="1000" bash /features/src/common-utils/install.sh
+RUN VERSION="os-provided" bash /features/src/git/install.sh
+RUN VERSION="latest" bash /features/src/docker-in-docker/install.sh
+RUN VERSION="lts" bash /features/src/node/install.sh
+RUN bash /features/src/sshd/install.sh
+RUN bash /features/src/github-cli/install.sh
 
 # install poetry, configure certificate for git and tools for oh my zsh
-RUN curl -sSL https://install.python-poetry.org | python3 - \
-    && python -m pip install --user pipx \
+RUN python -m pip install --user pipx \
     && python -m pipx ensurepath \
+    && pipx install poetry \
     && git clone https://github.com/zsh-users/zsh-syntax-highlighting.git $HOME/.zsh/zsh-syntax-highlighting \
     && echo "source ~/.zsh/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh" >> $HOME/.zshrc \
     && git clone https://github.com/zsh-users/zsh-autosuggestions $HOME/.zsh/zsh-autosuggestions \

.devcontainer/createCerts.sh

@@ -5,6 +5,13 @@
 # It tries to connect to a server (for instance, github), show the local certificates and save it to a file.
 
 # We connect to a random server and not paloaltonetworks.com to get external certificates. 
+# if command fails 
+
+if curl -I https://github.com > /dev/null; then
+    echo "No need to update certificate"
+    exit
+fi
+
 CONNECT_SERVER="github.com:443"
 
 FILE=$1
@@ -21,3 +28,6 @@ if [ ! -f "$FILE" ]; then
     exit
 fi
 
+git config --system http.sslCAInfo $FILE
+
+

.devcontainer/createCommand.sh

@@ -6,22 +6,19 @@ echo "Fixing permissions"
 
 sudo chown demisto .venv
 sudo chown demisto node_modules
+sudo chown demisto /workspaces
 sudo chown -R demisto $HOME
-sudo chown -R demisto /workspaces
-
-echo "Setting up git certificate"
-
-sudo git config --system http.sslCAInfo /usr/local/share/ca-certificates/certs.crt
 
 echo "Setting up VSCode paths"
 
 cp .devcontainer/settings.json .vscode/settings.json 
 touch CommonServerUserPython.py
+touch DemistoClassApiModule.py
 path=$(printf '%s:' Packs/ApiModules/Scripts/*)
 rm -f .env
 echo "PYTHONPATH=""$path"":$PYTHONPATH" >> .env
 echo "MYPYPATH=""$path"":$MYPYPATH" >> .env
 
 echo "Setting up content dependencies"
-
-NO_HOOKS=1 .hooks/bootstrap
+poetry install
+npm install