Merge branch 'main' into users/debnatkh/issue-539-service

.github/actions/prepare/action.yaml

@@ -17,6 +17,7 @@ runs:
         sudo apt-get install -y --no-install-recommends git wget gnupg lsb-release curl xz-utils tzdata cmake \
           python3-dev python3-pip ninja-build antlr3 m4 libidn11-dev libaio1 libaio-dev make clang-14 lld-14 llvm-14 file \
           distcc strace qemu-kvm qemu-utils dpkg-dev atop
+        sudo apt-get remove -y unattended-upgrades
         sudo pip3 install conan==1.59 pytest==7.1.3 pytest-timeout pytest-xdist==3.3.1 setproctitle==1.3.2 grpcio grpcio-tools PyHamcrest tornado xmltodict pyarrow boto3 moto[server] psutil pygithub==1.59.1 pyinstaller==5.13.2 cryptography packaging six pyyaml
     - name: install ccache
       shell: bash
@@ -26,3 +27,11 @@ runs:
         curl -4 -L https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}-linux-${OS_ARCH}.tar.xz \
           | tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${CCACHE_VERSION}-linux-${OS_ARCH}/ccache
         ls -la /usr/local/bin/ccache
+    - name: add user github to kvm group if exists
+      shell: bash
+      run: |
+        id -u github 2>/dev/null && {
+        sudo usermod -a -G kvm github
+        id github
+        grep kvm /etc/group
+        } || echo "user github not found"

.github/actions/test/action.yaml

@@ -27,6 +27,12 @@ inputs:
   bazel_remote_uri:
     required: false
     description: "bazel-remote endpoint"
+  bazel_remote_username:
+    required: false
+    description: "bazel-remote username"
+  bazel_remote_password:
+    required: false
+    description: "bazel-remote password"
   cache_update:
     required: false
     description: "Use cache for tests"
@@ -104,15 +110,21 @@ runs:
           extra_params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}")
         fi
 
+        if [ ! -z "${{ inputs.bazel_remote_username }}" ]; then
+          extra_params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}")
+          extra_params+=(--bazel-remote-password "${{ inputs.bazel_remote_password }}")
+        fi
+
         if [ "${{ inputs.cache_update }}" = "true" ]; then
           extra_params+=(--cache-tests)
+          extra_params+=(--bazel-remote-put --dist-cache-evict-bins)
         fi
 
         readarray -d ',' -t test_size < <(printf "%s" "${{ inputs.test_size }}")
         readarray -d ',' -t test_type < <(printf "%s" "${{ inputs.test_type }}")
 
         echo "::group::ya-make-test"
-        sudo -E -H -u github ./ya test -k --build "${build_type}" -D'BUILD_LANGUAGES=CPP PY3 PY2 GO' \
+        sudo -E -H -u github ./ya test -k --build "${build_type}" \
           ${test_size[@]/#/--test-size=} ${test_type[@]/#/--test-type=} \
           --test-threads "${{ inputs.test_threads }}" --link-threads "${{ inputs.link_threads }}" \
           --cache-size 512G --do-not-output-stderrs -T \

.github/ansible/inventory/00-hosts.yaml

@@ -1,7 +1,7 @@
 cache:
   hosts:
     cachesrv00:
-      ansible_host: 195.242.17.155
+      ansible_host: 195.242.17.0
   vars:
     docker_install_compose: true
     docker_users:

.github/ansible/playbooks/roles/cache/tasks/main.yml

@@ -12,7 +12,7 @@
 - name: Add a remote cache user htpasswd file
   community.general.htpasswd:
     path: /etc/bazel-remote-htpasswd
-    name: cache
+    name: bazel-remote-user
     password: "{{ remote_cache_password }}"
     owner: 1000
     group: 1000

.github/config/muted_ya.txt

@@ -1,23 +1,3 @@
-cloud/blockstore/tests/fio/qemu-vhost-local-test *
-cloud/blockstore/tests/fio/qemu-vhost-null-test *
-cloud/blockstore/tests/rdma/rdma-test *
-cloud/blockstore/tests/resize-disk *
 cloud/disk_manager/internal/pkg/dataplane/snapshot/storage/tests tests.TestShallowCopySnapshotWithRandomFailure
 cloud/disk_manager/internal/pkg/dataplane/snapshot/storage/tests tests.TestShallowCopySnapshotWithRandomFailure/store_chunks_in_s3
-cloud/disk_manager/internal/pkg/dataplane/snapshot/storage/tests tests.TestShallowCopySnapshotWithRandomFailure/store_chunks_in_ydb
-cloud/filestore/tests/fio_index_migration/qemu-intrahost-migration-kikimr-nemesis-test *
-cloud/filestore/tests/fio_index_migration/qemu-intrahost-migration-kikimr-test *
-cloud/filestore/tests/fio_index_migration/qemu-intrahost-migration-local-test *
-cloud/filestore/tests/fio_index/qemu-kikimr-nemesis-test *
-cloud/filestore/tests/fio_index/qemu-kikimr-test *
-cloud/filestore/tests/fio_index/qemu-local-test *
-cloud/filestore/tests/fio_migration/qemu-intrahost-migration-kikimr-nemesis-test *
-cloud/filestore/tests/fio_migration/qemu-intrahost-migration-kikimr-test *
-cloud/filestore/tests/fio_migration/qemu-intrahost-migration-local-test *
-cloud/filestore/tests/fio/qemu-kikimr-nemesis-test *
-cloud/filestore/tests/fio/qemu-kikimr-test *
-cloud/filestore/tests/fio/qemu-local-test *
-cloud/filestore/tests/fs_posix_compliance/qemu-kikimr-nemesis-test *
-cloud/filestore/tests/fs_posix_compliance/qemu-kikimr-test *
-cloud/filestore/tests/profile_log/qemu-kikimr-test *
-cloud/filestore/tests/profile_log/qemu-local-test *
+cloud/disk_manager/internal/pkg/dataplane/snapshot/storage/tests tests.TestShallowCopySnapshotWithRandomFailure/store_chunks_in_ydb

.github/packer/github-runner.pkr.hcl

@@ -63,6 +63,7 @@ build {
       "sudo apt-get install -y --no-install-recommends git wget gnupg lsb-release curl xz-utils tzdata cmake python3-dev python3-pip ninja-build antlr3 m4 libidn11-dev libaio1 libaio-dev make clang-14 lld-14 llvm-14 file distcc s3cmd qemu-kvm dpkg-dev",
       "sudo pip3 install conan==1.59 pytest==7.1.3  pyinstaller==5.13.2 pytest-timeout pytest-xdist==3.3.1 setproctitle==1.3.2  six pyyaml packaging  cryptography grpcio grpcio-tools PyHamcrest tornado xmltodict pyarrow boto3 moto[server] psutil pygithub==1.59.1",
       "curl -L https://github.com/ccache/ccache/releases/download/v${var.CCACHE_VERSION}/ccache-${var.CCACHE_VERSION}-linux-${var.OS_ARCH}.tar.xz | sudo tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${var.CCACHE_VERSION}-linux-${var.OS_ARCH}/ccache",
+      "sudo apt-get remove -y unattended-upgrades",
 
       # Other packages
       "sudo apt-get install -y git jq tree tmux atop",

.github/scripts/vm-watcher.py

@@ -0,0 +1,123 @@
+import os
+import grpc
+import json
+import logging
+import argparse
+from github import Github, Auth as GithubAuth
+from datetime import datetime, timedelta, timezone
+from yandexcloud import SDK, RetryInterceptor
+from yandex.cloud.compute.v1.instance_service_pb2_grpc import InstanceServiceStub
+from yandex.cloud.compute.v1.instance_service_pb2 import (
+    ListInstancesRequest,
+    DeleteInstanceRequest,
+)
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s: %(levelname)s: %(message)s"
+)
+
+CACHE_VM_ID = "dp7329odurnhplpf5ff0"
+
+
+def find_workflows_containing_string(
+    client, specified_time, search_string, owner="ydb-platform", repo="nbs"
+):
+    repo = client.get_repo(f"{owner}/{repo}")
+
+    # Calculate start time for search (-10 minutes)
+    start_time = specified_time - timedelta(minutes=10)
+    end_time = specified_time + timedelta(minutes=10)
+
+    # Initialize list to keep track of matching runs
+    matching_runs_info = []
+
+    for run in repo.get_workflow_runs():
+        # Check if the run started within our time window
+        run_started_at = run.created_at
+        if start_time <= run_started_at <= end_time:
+            print("Workflow", run.name, run.created_at, run.html_url)
+            # Get jobs or the current workflow run
+            for job in run.jobs():
+
+                if "Start self-hosted runner" in job.name:
+                    print("Job", job.name)
+                    # Attempt to get logs (note: this might require additional handling for large logs)
+                    try:
+                        logs = job.get_log()
+                        if search_string in logs:
+                            matching_runs_info.append(
+                                {
+                                    "run_id": run.id,
+                                    "run_url": run.html_url,  # Link to the workflow run
+                                    "job_name": job.name,
+                                }
+                            )
+                    except Exception as e:
+                        print(
+                            f"Error fetching logs for job {job.name} in run {run.id}: {e}"
+                        )
+
+    return matching_runs_info
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--service-account-key",
+        required=True,
+        help="Path to the service account key file",
+    )
+    parser.add_argument(
+        "--folder-id",
+        required=True,
+        help="The ID of the folder to list instances in",
+        default="bjeuq5o166dq4ukv3eec",
+    )
+    parser.add_argument(
+        "--ttl", required=True, help="The TTL for the VMs", default=24, type=int
+    )
+    parser.add_argument("--apply", action="store_true", help="Apply the changes")
+
+    args = parser.parse_args()
+
+    threshold = datetime.now() - timedelta(hours=args.ttl)
+
+    interceptor = RetryInterceptor(
+        max_retry_count=5, retriable_codes=[grpc.StatusCode.UNAVAILABLE]
+    )
+
+    with open(args.service_account_key, "r") as fp:
+        sdk = SDK(
+            service_account_key=json.load(fp),
+            endpoint="api.ai.nebius.cloud",
+            interceptor=interceptor,
+        )
+
+    gh = Github(auth=GithubAuth.Token(os.environ["GITHUB_TOKEN"]))
+
+    client = sdk.client(InstanceServiceStub)
+    response = client.List(ListInstancesRequest(folder_id=args.folder_id))
+
+    for vm in response.instances:
+        if vm.id == CACHE_VM_ID:
+            logging.info(f"Skipping VM {vm.id} as it is a cache VM")
+            continue
+
+        creation_time = vm.created_at.ToDatetime()
+        if creation_time < threshold:
+            logging.info(
+                f"VM {vm.id} is older than 24 hours, deleting it, created at {creation_time}"
+            )
+
+            if args.apply:
+                client.Delete(DeleteInstanceRequest(instance_id=vm.id))
+            else:
+                runs = find_workflows_containing_string(
+                    gh, creation_time.replace(tzinfo=timezone.utc), vm.id
+                )
+                print("Runs that match this id", runs)
+
+        else:
+            logging.info(
+                f"VM {vm.id} is younger than 24 hours, keeping it, created at {creation_time}"
+            )

.github/workflows/build_and_test_on_demand.yaml

@@ -103,14 +103,16 @@ on:
 jobs:
   provide-runner:
     name: Start self-hosted runner
-    timeout-minutes: 5
+    timeout-minutes: 60
+    if: always()
     runs-on: ubuntu-latest
     outputs:
       label: ${{ steps.start-yc-runner.outputs.label }}
       instance-id: ${{ steps.start-yc-runner.outputs.instance-id }}
     steps:
       - name: Start YC runner
         id: start-yc-runner
+        if: always()
         uses: librarian/[email protected]
         timeout-minutes: 60
         with:
@@ -143,6 +145,7 @@ jobs:
         with:
           submodules: true
           ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: ${{ contains(github.event.pull_request.labels.*.name, 'rebase') && 0 || 1 }}
       - name: Rebase PR
         if: ${{ github.event.pull_request.head.sha != '' && contains(github.event.pull_request.labels.*.name, 'rebase') }}
         shell: bash

.github/workflows/build_and_test_on_demand_cmake.yaml

@@ -43,14 +43,16 @@ on:
 jobs:
   provide-runner:
     name: Start self-hosted runner
-    timeout-minutes: 5
+    timeout-minutes: 60
+    if: always()
     runs-on: ubuntu-latest
     outputs:
       label: ${{ steps.start-yc-runner.outputs.label }}
       instance-id: ${{ steps.start-yc-runner.outputs.instance-id }}
     steps:
       - name: Start YC runner
         id: start-yc-runner
+        if: always()
         uses: librarian/[email protected]
         timeout-minutes: 60
         with:
@@ -83,6 +85,7 @@ jobs:
         with:
           submodules: true
           ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: ${{ contains(github.event.pull_request.labels.*.name, 'rebase') && 0 || 1 }}
       - name: Rebase PR
         if: ${{ github.event.pull_request.head.sha != '' && contains(github.event.pull_request.labels.*.name, 'rebase') }}
         shell: bash

.github/workflows/build_and_test_ya.yaml

@@ -124,6 +124,8 @@ jobs:
         test_type: ${{ inputs.test_type }}
         cache_update: ${{ inputs.cache_update_tests }}
         bazel_remote_uri: ${{ vars.REMOTE_CACHE_URL_YA || '' }}
+        bazel_remote_username: ${{ secrets.REMOTE_CACHE_USERNAME }}
+        bazel_remote_password: ${{ secrets.REMOTE_CACHE_PASSWORD }}
         link_threads: ${{ inputs.link_threads }}
         test_threads: ${{ inputs.test_threads }}
         sync_to_s3: ${{ vars.SYNC_TO_S3 }}

.github/workflows/nightly-asan.yaml

@@ -0,0 +1,17 @@
+name: Nightly build (asan)
+on:
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build/test x86_64 using YA (asan)
+    uses: ./.github/workflows/build_and_test_on_demand.yaml
+    secrets: inherit
+    with:
+      build_preset: release-asan
+      test_type: "unittest,clang_tidy,gtest,py3test,py2test,pytest,flake8,black,py2_flake8,gofmt"
+      cache_update_build: true
+      cache_update_tests: false
+      test_threads: 6

.github/workflows/nightly-tsan.yaml

@@ -0,0 +1,17 @@
+name: Nightly build (tsan)
+on:
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build/test x86_64 using YA (tsan)
+    uses: ./.github/workflows/build_and_test_on_demand.yaml
+    secrets: inherit
+    with:
+      build_preset: release-tsan
+      test_type: "unittest,clang_tidy,gtest,py3test,py2test,pytest,flake8,black,py2_flake8,gofmt"
+      cache_update_build: true
+      cache_update_tests: false
+      test_threads: 6

.github/workflows/nightly.yaml

@@ -5,11 +5,12 @@ on:
   workflow_dispatch:
 
 jobs:
-  ya_x86_64:
-    name: Build/test x86_64 using YA
+  build:
+    name: Build/test x86_64 using YA (relwithdebinfo)
     uses: ./.github/workflows/build_and_test_on_demand.yaml
     secrets: inherit
     with:
+      build_preset: relwithdebinfo
       cache_update_build: true
       cache_update_tests: false
       test_threads: 6

.github/workflows/pr.yaml

@@ -8,6 +8,7 @@ on:
       - '.github/**'
       - 'example/**'
       - 'doc/**'
+      - '**.md'
     types:
       - 'opened'
       - 'synchronize'

CONTRIBUTING.md

@@ -3,31 +3,8 @@
 
 ## General info
 
-Hello! In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA can be found here https://yandex.ru/legal/cla/?lang=en
-
-By adopting the CLA, you state the following:
-
-* You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA,
-* You have read the terms and conditions of the CLA and agree with them in full,
-* You are legally able to provide and license your contributions as stated,
-* We may use your contributions for our open source projects and for any other our project too,
-* We rely on your assurances concerning the rights of third parties in relation to your contributions.
-
-If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you have already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA.
-
-## Provide contributions
-
-If you have already adopted terms and conditions of the CLA, you are able to provide your contributions. When you submit your pull request, please add the following information into it:
-
-```
-I hereby agree to the terms of the CLA available at: [link].
-```
-
-Replace the bracketed text as follows:
-* [link] is the link to the current version of the CLA: https://yandex.ru/legal/cla/?lang=en.
-
-It is enough to provide us such notification once.
+Anyone who wants to use and improve NBS or Filestore is welcome to make contributions. But before making a pull request, please, create an issue first and discuss the idea with the maintainers of this repo. 
 
 ## Other questions
 
-If you have any questions, please mail us at qkrorlqr@yandex-team.ru.
+If you have any questions, please mail us at thinkingwhat@yandex.ru.