diff --git a/.github/actions/prepare/action.yaml b/.github/actions/prepare/action.yaml index 47eee3307d5..eff173ba249 100644 --- a/.github/actions/prepare/action.yaml +++ b/.github/actions/prepare/action.yaml @@ -17,6 +17,7 @@ runs: sudo apt-get install -y --no-install-recommends git wget gnupg lsb-release curl xz-utils tzdata cmake \ python3-dev python3-pip ninja-build antlr3 m4 libidn11-dev libaio1 libaio-dev make clang-14 lld-14 llvm-14 file \ distcc strace qemu-kvm qemu-utils dpkg-dev atop + sudo apt-get remove -y unattended-upgrades sudo pip3 install conan==1.59 pytest==7.1.3 pytest-timeout pytest-xdist==3.3.1 setproctitle==1.3.2 grpcio grpcio-tools PyHamcrest tornado xmltodict pyarrow boto3 moto[server] psutil pygithub==1.59.1 pyinstaller==5.13.2 cryptography packaging six pyyaml - name: install ccache shell: bash @@ -26,3 +27,11 @@ runs: curl -4 -L https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}-linux-${OS_ARCH}.tar.xz \ | tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${CCACHE_VERSION}-linux-${OS_ARCH}/ccache ls -la /usr/local/bin/ccache + - name: add user github to kvm group if exists + shell: bash + run: | + id -u github 2>/dev/null && { + sudo usermod -a -G kvm github + id github + grep kvm /etc/group + } || echo "user github not found" diff --git a/.github/actions/test/action.yaml b/.github/actions/test/action.yaml index 1a3782625ae..23b0d815437 100644 --- a/.github/actions/test/action.yaml +++ b/.github/actions/test/action.yaml @@ -27,6 +27,12 @@ inputs: bazel_remote_uri: required: false description: "bazel-remote endpoint" + bazel_remote_username: + required: false + description: "bazel-remote username" + bazel_remote_password: + required: false + description: "bazel-remote password" cache_update: required: false description: "Use cache for tests" @@ -104,15 +110,21 @@ runs: extra_params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}") fi + if [ ! -z "${{ inputs.bazel_remote_username }}" ]; then + extra_params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}") + extra_params+=(--bazel-remote-password "${{ inputs.bazel_remote_password }}") + fi + if [ "${{ inputs.cache_update }}" = "true" ]; then extra_params+=(--cache-tests) + extra_params+=(--bazel-remote-put --dist-cache-evict-bins) fi readarray -d ',' -t test_size < <(printf "%s" "${{ inputs.test_size }}") readarray -d ',' -t test_type < <(printf "%s" "${{ inputs.test_type }}") echo "::group::ya-make-test" - sudo -E -H -u github ./ya test -k --build "${build_type}" -D'BUILD_LANGUAGES=CPP PY3 PY2 GO' \ + sudo -E -H -u github ./ya test -k --build "${build_type}" \ ${test_size[@]/#/--test-size=} ${test_type[@]/#/--test-type=} \ --test-threads "${{ inputs.test_threads }}" --link-threads "${{ inputs.link_threads }}" \ --cache-size 512G --do-not-output-stderrs -T \ diff --git a/.github/ansible/inventory/00-hosts.yaml b/.github/ansible/inventory/00-hosts.yaml index f5c6319dd7a..bb4345acab9 100644 --- a/.github/ansible/inventory/00-hosts.yaml +++ b/.github/ansible/inventory/00-hosts.yaml @@ -1,7 +1,7 @@ cache: hosts: cachesrv00: - ansible_host: 195.242.17.155 + ansible_host: 195.242.17.0 vars: docker_install_compose: true docker_users: diff --git a/.github/ansible/playbooks/roles/cache/tasks/main.yml b/.github/ansible/playbooks/roles/cache/tasks/main.yml index 6a636369091..609d50720a3 100644 --- a/.github/ansible/playbooks/roles/cache/tasks/main.yml +++ b/.github/ansible/playbooks/roles/cache/tasks/main.yml @@ -12,7 +12,7 @@ - name: Add a remote cache user htpasswd file community.general.htpasswd: path: /etc/bazel-remote-htpasswd - name: cache + name: bazel-remote-user password: "{{ remote_cache_password }}" owner: 1000 group: 1000 diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index ef65d8bd05d..ee5d9631965 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -1,23 +1,3 @@ -cloud/blockstore/tests/fio/qemu-vhost-local-test * -cloud/blockstore/tests/fio/qemu-vhost-null-test * -cloud/blockstore/tests/rdma/rdma-test * -cloud/blockstore/tests/resize-disk * cloud/disk_manager/internal/pkg/dataplane/snapshot/storage/tests tests.TestShallowCopySnapshotWithRandomFailure cloud/disk_manager/internal/pkg/dataplane/snapshot/storage/tests tests.TestShallowCopySnapshotWithRandomFailure/store_chunks_in_s3 -cloud/disk_manager/internal/pkg/dataplane/snapshot/storage/tests tests.TestShallowCopySnapshotWithRandomFailure/store_chunks_in_ydb -cloud/filestore/tests/fio_index_migration/qemu-intrahost-migration-kikimr-nemesis-test * -cloud/filestore/tests/fio_index_migration/qemu-intrahost-migration-kikimr-test * -cloud/filestore/tests/fio_index_migration/qemu-intrahost-migration-local-test * -cloud/filestore/tests/fio_index/qemu-kikimr-nemesis-test * -cloud/filestore/tests/fio_index/qemu-kikimr-test * -cloud/filestore/tests/fio_index/qemu-local-test * -cloud/filestore/tests/fio_migration/qemu-intrahost-migration-kikimr-nemesis-test * -cloud/filestore/tests/fio_migration/qemu-intrahost-migration-kikimr-test * -cloud/filestore/tests/fio_migration/qemu-intrahost-migration-local-test * -cloud/filestore/tests/fio/qemu-kikimr-nemesis-test * -cloud/filestore/tests/fio/qemu-kikimr-test * -cloud/filestore/tests/fio/qemu-local-test * -cloud/filestore/tests/fs_posix_compliance/qemu-kikimr-nemesis-test * -cloud/filestore/tests/fs_posix_compliance/qemu-kikimr-test * -cloud/filestore/tests/profile_log/qemu-kikimr-test * -cloud/filestore/tests/profile_log/qemu-local-test * +cloud/disk_manager/internal/pkg/dataplane/snapshot/storage/tests tests.TestShallowCopySnapshotWithRandomFailure/store_chunks_in_ydb \ No newline at end of file diff --git a/.github/packer/github-runner.pkr.hcl b/.github/packer/github-runner.pkr.hcl index f0b23d67f8a..6f49c88094a 100644 --- a/.github/packer/github-runner.pkr.hcl +++ b/.github/packer/github-runner.pkr.hcl @@ -63,6 +63,7 @@ build { "sudo apt-get install -y --no-install-recommends git wget gnupg lsb-release curl xz-utils tzdata cmake python3-dev python3-pip ninja-build antlr3 m4 libidn11-dev libaio1 libaio-dev make clang-14 lld-14 llvm-14 file distcc s3cmd qemu-kvm dpkg-dev", "sudo pip3 install conan==1.59 pytest==7.1.3 pyinstaller==5.13.2 pytest-timeout pytest-xdist==3.3.1 setproctitle==1.3.2 six pyyaml packaging cryptography grpcio grpcio-tools PyHamcrest tornado xmltodict pyarrow boto3 moto[server] psutil pygithub==1.59.1", "curl -L https://github.com/ccache/ccache/releases/download/v${var.CCACHE_VERSION}/ccache-${var.CCACHE_VERSION}-linux-${var.OS_ARCH}.tar.xz | sudo tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${var.CCACHE_VERSION}-linux-${var.OS_ARCH}/ccache", + "sudo apt-get remove -y unattended-upgrades", # Other packages "sudo apt-get install -y git jq tree tmux atop", diff --git a/.github/scripts/vm-watcher.py b/.github/scripts/vm-watcher.py new file mode 100644 index 00000000000..36e3cd158a4 --- /dev/null +++ b/.github/scripts/vm-watcher.py @@ -0,0 +1,123 @@ +import os +import grpc +import json +import logging +import argparse +from github import Github, Auth as GithubAuth +from datetime import datetime, timedelta, timezone +from yandexcloud import SDK, RetryInterceptor +from yandex.cloud.compute.v1.instance_service_pb2_grpc import InstanceServiceStub +from yandex.cloud.compute.v1.instance_service_pb2 import ( + ListInstancesRequest, + DeleteInstanceRequest, +) + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s: %(levelname)s: %(message)s" +) + +CACHE_VM_ID = "dp7329odurnhplpf5ff0" + + +def find_workflows_containing_string( + client, specified_time, search_string, owner="ydb-platform", repo="nbs" +): + repo = client.get_repo(f"{owner}/{repo}") + + # Calculate start time for search (-10 minutes) + start_time = specified_time - timedelta(minutes=10) + end_time = specified_time + timedelta(minutes=10) + + # Initialize list to keep track of matching runs + matching_runs_info = [] + + for run in repo.get_workflow_runs(): + # Check if the run started within our time window + run_started_at = run.created_at + if start_time <= run_started_at <= end_time: + print("Workflow", run.name, run.created_at, run.html_url) + # Get jobs or the current workflow run + for job in run.jobs(): + + if "Start self-hosted runner" in job.name: + print("Job", job.name) + # Attempt to get logs (note: this might require additional handling for large logs) + try: + logs = job.get_log() + if search_string in logs: + matching_runs_info.append( + { + "run_id": run.id, + "run_url": run.html_url, # Link to the workflow run + "job_name": job.name, + } + ) + except Exception as e: + print( + f"Error fetching logs for job {job.name} in run {run.id}: {e}" + ) + + return matching_runs_info + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--service-account-key", + required=True, + help="Path to the service account key file", + ) + parser.add_argument( + "--folder-id", + required=True, + help="The ID of the folder to list instances in", + default="bjeuq5o166dq4ukv3eec", + ) + parser.add_argument( + "--ttl", required=True, help="The TTL for the VMs", default=24, type=int + ) + parser.add_argument("--apply", action="store_true", help="Apply the changes") + + args = parser.parse_args() + + threshold = datetime.now() - timedelta(hours=args.ttl) + + interceptor = RetryInterceptor( + max_retry_count=5, retriable_codes=[grpc.StatusCode.UNAVAILABLE] + ) + + with open(args.service_account_key, "r") as fp: + sdk = SDK( + service_account_key=json.load(fp), + endpoint="api.ai.nebius.cloud", + interceptor=interceptor, + ) + + gh = Github(auth=GithubAuth.Token(os.environ["GITHUB_TOKEN"])) + + client = sdk.client(InstanceServiceStub) + response = client.List(ListInstancesRequest(folder_id=args.folder_id)) + + for vm in response.instances: + if vm.id == CACHE_VM_ID: + logging.info(f"Skipping VM {vm.id} as it is a cache VM") + continue + + creation_time = vm.created_at.ToDatetime() + if creation_time < threshold: + logging.info( + f"VM {vm.id} is older than 24 hours, deleting it, created at {creation_time}" + ) + + if args.apply: + client.Delete(DeleteInstanceRequest(instance_id=vm.id)) + else: + runs = find_workflows_containing_string( + gh, creation_time.replace(tzinfo=timezone.utc), vm.id + ) + print("Runs that match this id", runs) + + else: + logging.info( + f"VM {vm.id} is younger than 24 hours, keeping it, created at {creation_time}" + ) diff --git a/.github/workflows/build_and_test_on_demand.yaml b/.github/workflows/build_and_test_on_demand.yaml index ae6d6b8da29..ae49db8ebb6 100644 --- a/.github/workflows/build_and_test_on_demand.yaml +++ b/.github/workflows/build_and_test_on_demand.yaml @@ -103,7 +103,8 @@ on: jobs: provide-runner: name: Start self-hosted runner - timeout-minutes: 5 + timeout-minutes: 60 + if: always() runs-on: ubuntu-latest outputs: label: ${{ steps.start-yc-runner.outputs.label }} @@ -111,6 +112,7 @@ jobs: steps: - name: Start YC runner id: start-yc-runner + if: always() uses: librarian/yc-github-runner@0.0.15 timeout-minutes: 60 with: @@ -143,6 +145,7 @@ jobs: with: submodules: true ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: ${{ contains(github.event.pull_request.labels.*.name, 'rebase') && 0 || 1 }} - name: Rebase PR if: ${{ github.event.pull_request.head.sha != '' && contains(github.event.pull_request.labels.*.name, 'rebase') }} shell: bash diff --git a/.github/workflows/build_and_test_on_demand_cmake.yaml b/.github/workflows/build_and_test_on_demand_cmake.yaml index 19da58ca87f..3b38d7301ac 100644 --- a/.github/workflows/build_and_test_on_demand_cmake.yaml +++ b/.github/workflows/build_and_test_on_demand_cmake.yaml @@ -43,7 +43,8 @@ on: jobs: provide-runner: name: Start self-hosted runner - timeout-minutes: 5 + timeout-minutes: 60 + if: always() runs-on: ubuntu-latest outputs: label: ${{ steps.start-yc-runner.outputs.label }} @@ -51,6 +52,7 @@ jobs: steps: - name: Start YC runner id: start-yc-runner + if: always() uses: librarian/yc-github-runner@0.0.15 timeout-minutes: 60 with: @@ -83,6 +85,7 @@ jobs: with: submodules: true ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: ${{ contains(github.event.pull_request.labels.*.name, 'rebase') && 0 || 1 }} - name: Rebase PR if: ${{ github.event.pull_request.head.sha != '' && contains(github.event.pull_request.labels.*.name, 'rebase') }} shell: bash diff --git a/.github/workflows/build_and_test_ya.yaml b/.github/workflows/build_and_test_ya.yaml index c81de202f6f..8abcdabef3f 100644 --- a/.github/workflows/build_and_test_ya.yaml +++ b/.github/workflows/build_and_test_ya.yaml @@ -124,6 +124,8 @@ jobs: test_type: ${{ inputs.test_type }} cache_update: ${{ inputs.cache_update_tests }} bazel_remote_uri: ${{ vars.REMOTE_CACHE_URL_YA || '' }} + bazel_remote_username: ${{ secrets.REMOTE_CACHE_USERNAME }} + bazel_remote_password: ${{ secrets.REMOTE_CACHE_PASSWORD }} link_threads: ${{ inputs.link_threads }} test_threads: ${{ inputs.test_threads }} sync_to_s3: ${{ vars.SYNC_TO_S3 }} diff --git a/.github/workflows/nightly-asan.yaml b/.github/workflows/nightly-asan.yaml new file mode 100644 index 00000000000..8d1e5adbc97 --- /dev/null +++ b/.github/workflows/nightly-asan.yaml @@ -0,0 +1,17 @@ +name: Nightly build (asan) +on: + schedule: + - cron: "0 1 * * *" + workflow_dispatch: + +jobs: + build: + name: Build/test x86_64 using YA (asan) + uses: ./.github/workflows/build_and_test_on_demand.yaml + secrets: inherit + with: + build_preset: release-asan + test_type: "unittest,clang_tidy,gtest,py3test,py2test,pytest,flake8,black,py2_flake8,gofmt" + cache_update_build: true + cache_update_tests: false + test_threads: 6 diff --git a/.github/workflows/nightly-tsan.yaml b/.github/workflows/nightly-tsan.yaml new file mode 100644 index 00000000000..eff6892096c --- /dev/null +++ b/.github/workflows/nightly-tsan.yaml @@ -0,0 +1,17 @@ +name: Nightly build (tsan) +on: + schedule: + - cron: "0 1 * * *" + workflow_dispatch: + +jobs: + build: + name: Build/test x86_64 using YA (tsan) + uses: ./.github/workflows/build_and_test_on_demand.yaml + secrets: inherit + with: + build_preset: release-tsan + test_type: "unittest,clang_tidy,gtest,py3test,py2test,pytest,flake8,black,py2_flake8,gofmt" + cache_update_build: true + cache_update_tests: false + test_threads: 6 diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 1e7aa0b7e80..84489173481 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -5,11 +5,12 @@ on: workflow_dispatch: jobs: - ya_x86_64: - name: Build/test x86_64 using YA + build: + name: Build/test x86_64 using YA (relwithdebinfo) uses: ./.github/workflows/build_and_test_on_demand.yaml secrets: inherit with: + build_preset: relwithdebinfo cache_update_build: true cache_update_tests: false test_threads: 6 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ed2a01b6355..04a1bf92a75 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -8,6 +8,7 @@ on: - '.github/**' - 'example/**' - 'doc/**' + - '**.md' types: - 'opened' - 'synchronize' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f16aa1f341f..58689a2ed4e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,31 +3,8 @@ ## General info -Hello! In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA can be found here https://yandex.ru/legal/cla/?lang=en - -By adopting the CLA, you state the following: - -* You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA, -* You have read the terms and conditions of the CLA and agree with them in full, -* You are legally able to provide and license your contributions as stated, -* We may use your contributions for our open source projects and for any other our project too, -* We rely on your assurances concerning the rights of third parties in relation to your contributions. - -If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you have already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA. - -## Provide contributions - -If you have already adopted terms and conditions of the CLA, you are able to provide your contributions. When you submit your pull request, please add the following information into it: - -``` -I hereby agree to the terms of the CLA available at: [link]. -``` - -Replace the bracketed text as follows: -* [link] is the link to the current version of the CLA: https://yandex.ru/legal/cla/?lang=en. - -It is enough to provide us such notification once. +Anyone who wants to use and improve NBS or Filestore is welcome to make contributions. But before making a pull request, please, create an issue first and discuss the idea with the maintainers of this repo. ## Other questions -If you have any questions, please mail us at qkrorlqr@yandex-team.ru. +If you have any questions, please mail us at thinkingwhat@yandex.ru. diff --git a/GITHUB.md b/GITHUB.md new file mode 100644 index 00000000000..3409018075f --- /dev/null +++ b/GITHUB.md @@ -0,0 +1,37 @@ +Checks are launched without labels only for [organization members](https://github.com/orgs/ydb-platform/people). Later we plan to limit this only to NBS teams. + +If PR is opened not by a team member they will receive a message that the team member needs to label their PR with an `ok-to-test` label. Beware of RCE. + +There is also a list of labels that slightly alters how and which tests are run: + +1. `large-tests` to launch large tests in PR. By default, we launch small and medium. +2. `blockstore`, `filestore`, `disk_manager`, and `tasks` to launch test ONLY for specified projects. You can specify more than one label. +3. `sleep` to add 7200s (2 hours) sleep to your run, if you want to debug it. + +Also, you can launch [ya make](https://github.com/ydb-platform/nbs/actions/workflows/build_and_test_on_demand.yaml) or [cmake](https://github.com/ydb-platform/nbs/actions/workflows/build_and_test_on_demand_cmake.yaml) builds on your branch with any timeout you want (but please do not do more than 12h, VMs are expensive). You can find the IP of the VM inside of the jobs. The first occurrence happens in the `Prepare runner` job in the `Configure NCP` step, later IP is set in the header of jobs. You must use your GitHub key for it. User `github`. Feature not available for non-members. + +All build and test workflows provide some level of debug info available on our s3 website. + +Example URL for the top-level directory is like this: https://github-actions-s3.website.nemax.nebius.cloud/ydb-platform/nbs/PR-check/8103221450/1/nebius-x86-64/ + +* `ydb-platform` - name of the organisation +* `nbs` - name of the repo +* `PR-check` - ID of the workflow +* `8103221450` - ID of the workflow run (you can look it up in the `Actions` tab in your PR) +* `1` - number of the runs, if you restart workflow this number will increase. +* `nebius-x86-64` - nebius is the prefix and x86-64 an architecture, there also can be one of the suffixes: `-debug`, `-asan`,`-tsan` if you choose to build with debugging symbols or sanitizers. + +For new runs, we generate index.html files only for the top level directories, not for directories lower. And every night we regenerate index files for the whole s3 bucket. + +On the top level, you can expect adirectory structure like this: + +* `build_logs/` - well, ya make build logs +* `logs/` - log of the tests, short version +* `test_logs/` - logs of ya test +* `test_reports/` - junit report, debug data so you can more or less debug what happened during preparation of summary report. +* `summary/` - ya-test.html with results of the test run +* `test_data/` - all test data except binaries, VM images, and everything like that, that take a lot of space on S3. Stored only for a week, while other directories are stored for a month. + +Files in `test_data` are synced only for failed tests and the list of folders to sync is determined by [script fail-checker.py](https://github.com/ydb-platform/nbs/blob/01c51c0da8168c7111a03c8d34cd220b9b87eaec/.github/actions/test/action.yaml#L191) + +Availability of other logs in the summary report is dependent on what ya make decide to add in junit report. diff --git a/LICENSE b/LICENSE index ba760ac61d9..4aef279215a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,181 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2023 YANDEX LLC Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/README.md b/README.md index f86938719fd..a7d15517017 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Follow the instructions [here](example/README.md) to build and run NBS on your m Follow the instructions [here](CLANG-FORMAT.md) to install clang-format for formatting the code. +Additional information about features of our [Github Actions](GITHUB.md) (labels, test results and so on) + ### How to Deploy TODO diff --git a/build/ext_mapping.conf.json b/build/ext_mapping.conf.json index afba187648d..e14cb655a4c 100644 --- a/build/ext_mapping.conf.json +++ b/build/ext_mapping.conf.json @@ -12,7 +12,6 @@ "2513045473": "https://storage.nemax.nebius.cloud/nbs-oss-resources/flock", "2951476475": "https://storage.ai.nebius.cloud/nbs-oss-resources/ubuntu-18.04-minimal-cloudimg-amd64.img", "3064742393": "https://storage.ai.nebius.cloud/nbs-oss-resources/ubuntu1604-ci-stable", - "5274078903": "https://storage.ai.nebius.cloud/nbs-oss-resources/panic.img", "4709742882": "https://storage.ai.nebius.cloud/nbs-oss-resources/ubuntu-22.04-jammy-server-cloudimg-amd64.vmdk", "4915490540": "https://storage.ai.nebius.cloud/nbs-oss-resources/root.squashfs", "1268336763": "https://storage.ai.nebius.cloud/nbs-oss-resources/1268336763.rootfs.squashfs", @@ -32,7 +31,6 @@ "2513045473": "cloud/filestore/tools/testing/fs_posix_compliance/suite/bin", "2951476475": "cloud/disk_manager/internal/pkg/dataplane/url/qcow2/tests|cloud/disk_manager/internal/pkg/facade/image_service_test|cloud/disk_manager/internal/pkg/facade/image_service_nemesis_test", "3064742393": "cloud/disk_manager/internal/pkg/dataplane/url/qcow2/tests", - "5274078903": "cloud/disk_manager/internal/pkg/dataplane/url/tests|cloud/disk_manager/internal/pkg/facade/image_service_test|cloud/disk_manager/internal/pkg/facade/image_service_nemesis_test", "4709742882": "cloud/disk_manager/internal/pkg/dataplane/url/tests|cloud/disk_manager/internal/pkg/dataplane/url/vmdk/tests", "4915490540": "cloud/disk_manager/internal/pkg/facade/image_service_test|cloud/disk_manager/internal/pkg/facade/image_service_nemesis_test", "1268336763": "cloud/disk_manager/internal/pkg/logging/journald_tests", diff --git a/cloud/blockstore/apps/client/lib/start_endpoint.cpp b/cloud/blockstore/apps/client/lib/start_endpoint.cpp index 1a6f7fa542e..63a393efa81 100644 --- a/cloud/blockstore/apps/client/lib/start_endpoint.cpp +++ b/cloud/blockstore/apps/client/lib/start_endpoint.cpp @@ -109,6 +109,7 @@ class TStartEndpointCommand final TString EncryptionKeyPath; TString EncryptionKeyHash; bool Persistent = false; + TString NbdDeviceFile; public: TStartEndpointCommand(IBlockStorePtr client) @@ -180,6 +181,10 @@ class TStartEndpointCommand final Opts.AddLongOption("persistent", "add endpoint to keyring") .NoArgument() .SetFlag(&Persistent); + + Opts.AddLongOption("nbd-device", "nbd device file which nbd-client connected to") + .RequiredArgument("STR") + .StoreResult(&NbdDeviceFile); } protected: @@ -225,6 +230,7 @@ class TStartEndpointCommand final EncryptionKeyPath, EncryptionKeyHash)); request->SetPersistent(Persistent); + request->SetNbdDeviceFile(NbdDeviceFile); } STORAGE_DEBUG("Sending StartEndpoint request"); diff --git a/cloud/blockstore/apps/disk_agent/ya.make b/cloud/blockstore/apps/disk_agent/ya.make index e0ebc056f2e..a1f81505f6e 100644 --- a/cloud/blockstore/apps/disk_agent/ya.make +++ b/cloud/blockstore/apps/disk_agent/ya.make @@ -13,7 +13,6 @@ PEERDIR( cloud/storage/core/libs/daemon - contrib/ydb/core/driver_lib/run contrib/ydb/core/security library/cpp/getopt diff --git a/cloud/blockstore/apps/server/ya.make b/cloud/blockstore/apps/server/ya.make index 0f47b1735d1..032b3bb3523 100644 --- a/cloud/blockstore/apps/server/ya.make +++ b/cloud/blockstore/apps/server/ya.make @@ -18,7 +18,6 @@ PEERDIR( cloud/storage/core/libs/daemon cloud/storage/core/libs/iam/iface - contrib/ydb/core/driver_lib/run contrib/ydb/core/security library/cpp/getopt diff --git a/cloud/blockstore/config/disk.proto b/cloud/blockstore/config/disk.proto index 47604a98e90..16af38fa6d1 100644 --- a/cloud/blockstore/config/disk.proto +++ b/cloud/blockstore/config/disk.proto @@ -107,7 +107,14 @@ message TStorageDiscoveryConfig message TPoolConfig { optional string PoolName = 1; + + // The minimum allowed size of a device. + // If MinSize == 0 and Layout is specified then the minimum allowed size + // is Layout.HeaderSize + Layout.DeviceSize. optional uint64 MinSize = 2; + + // The maximum allowed size of a device. If not specified, then the + // maximum size of the device is not limited. optional uint64 MaxSize = 3; optional TLayout Layout = 4; diff --git a/cloud/blockstore/config/notify.proto b/cloud/blockstore/config/notify.proto index 949c5a36b4c..af856ec56b2 100644 --- a/cloud/blockstore/config/notify.proto +++ b/cloud/blockstore/config/notify.proto @@ -5,7 +5,7 @@ package NCloud.NBlockStore.NProto; option go_package = "github.com/ydb-platform/nbs/cloud/blockstore/config"; //////////////////////////////////////////////////////////////////////////////// -// Config for Notify service (https://github.yandex-team.ru/data-ui/notify). +// Config for Notify service. message TNotifyConfig { diff --git a/cloud/blockstore/config/rdma.proto b/cloud/blockstore/config/rdma.proto index 30f76bf9522..bc131a80e71 100644 --- a/cloud/blockstore/config/rdma.proto +++ b/cloud/blockstore/config/rdma.proto @@ -43,3 +43,14 @@ message TRdmaTarget TRdmaEndpoint Endpoint = 1; TRdmaServer Server = 2; } + +message TRdmaConfig +{ + bool ClientEnabled = 1; + + TRdmaClient Client = 2; + + bool ServerEnabled = 3; + + TRdmaServer Server = 4; +} diff --git a/cloud/blockstore/config/server.proto b/cloud/blockstore/config/server.proto index ddf0b0435a8..167ed93b3aa 100644 --- a/cloud/blockstore/config/server.proto +++ b/cloud/blockstore/config/server.proto @@ -168,6 +168,9 @@ message TServerConfig // Path to vhost server executable. optional string VhostServerPath = 108; + + // Prefix for nbd device paths, e.g. "/dev/nbd" + optional string NbdDevicePrefix = 109; } //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/blockstore/config/storage.proto b/cloud/blockstore/config/storage.proto index a566bbd10d8..0357703a6df 100644 --- a/cloud/blockstore/config/storage.proto +++ b/cloud/blockstore/config/storage.proto @@ -448,7 +448,7 @@ message TStorageServiceConfig optional uint32 CompactionGarbageBlockLimit = 174; // User data may be dumped on some monitoring pages if this flag is set. - // See https://st.yandex-team.ru/NBS-1726#5fd39472aa62c1459c7287cb + // See NBS-1726 optional bool UserDataDebugDumpAllowed = 175; // Fresh channels configuration. @@ -920,4 +920,32 @@ message TStorageServiceConfig // Limits unconfirmed (and confirmed but not added to the index) blob count. optional uint32 UnconfirmedBlobCountHardLimit = 349; + + // These are duplicates of "CachedConfigPath" and "CachedSessionsPath" + // fields in the "TDiskAgentConfig" and here just to ease deployment of the + // caches. Disk agent config has higher priority than storage config. + optional string CachedDiskAgentConfigPath = 350; + optional string CachedDiskAgentSessionsPath = 351; + + // Max bandwidth used for shadow disk fill in MiB/s (actual bandwidth is x2 + // due to the need to read and write). + optional uint32 MaxShadowDiskFillBandwidth = 352; + // Minimum delay between shadow disk acquire attempts when writes to the + // source disk are blocked (in ms). + optional uint32 MinAcquireShadowDiskRetryDelayWhenBlocked = 353; + // Maximum delay between shadow disk acquire attempts when writes to the + // source disk are blocked (in ms). + optional uint32 MaxAcquireShadowDiskRetryDelayWhenBlocked = 354; + // Minimum delay between shadow disk acquire attempts when writes to the + // source disk are not blocked (in ms). + optional uint32 MinAcquireShadowDiskRetryDelayWhenNonBlocked = 355; + // Maximum delay between shadow disk acquire attempts when writes to the + // source disk are not blocked (in ms). + optional uint32 MaxAcquireShadowDiskRetryDelayWhenNonBlocked = 356; + // Timeout for attempts to acquire the shadow disk when writes to the source + // disk are blocked (in ms). + optional uint32 MaxAcquireShadowDiskTotalTimeoutWhenBlocked = 357; + // Timeout for attempts to acquire the shadow disk when writes to the source + // disk are not blocked (in ms). + optional uint32 MaxAcquireShadowDiskTotalTimeoutWhenNonBlocked = 358; } diff --git a/cloud/blockstore/libs/client/durable.cpp b/cloud/blockstore/libs/client/durable.cpp index 9c8bbf6769c..9115bb66092 100644 --- a/cloud/blockstore/libs/client/durable.cpp +++ b/cloud/blockstore/libs/client/durable.cpp @@ -121,7 +121,7 @@ using TRequestStatePtr = TIntrusivePtr>; //////////////////////////////////////////////////////////////////////////////// -class TDurableClient +class TDurableClient final : public IBlockStore , public std::enable_shared_from_this { diff --git a/cloud/blockstore/libs/client/tsan.supp b/cloud/blockstore/libs/client/tsan.supp new file mode 100644 index 00000000000..0ad5266fd0b --- /dev/null +++ b/cloud/blockstore/libs/client/tsan.supp @@ -0,0 +1,3 @@ +# There is a race between tc_on_alarm & on_writable NBS-3614 +race:tc_on_alarm +race:on_writable diff --git a/cloud/blockstore/libs/client/ya.make b/cloud/blockstore/libs/client/ya.make index 674bed308b3..5524a4169c0 100644 --- a/cloud/blockstore/libs/client/ya.make +++ b/cloud/blockstore/libs/client/ya.make @@ -39,7 +39,7 @@ ENDIF() END() -# Until https://st.yandex-team.ru/DEVTOOLSSUPPORT-25698 is not solved. +# Until DEVTOOLSSUPPORT-25698 is not solved. IF (SANITIZER_TYPE == "address" OR SANITIZER_TYPE == "memory") RECURSE_FOR_TESTS( ut diff --git a/cloud/blockstore/libs/daemon/common/bootstrap.cpp b/cloud/blockstore/libs/daemon/common/bootstrap.cpp index ef29d91757f..ca6d13c75df 100644 --- a/cloud/blockstore/libs/daemon/common/bootstrap.cpp +++ b/cloud/blockstore/libs/daemon/common/bootstrap.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -198,6 +199,7 @@ void TBootstrapBase::Init() BootstrapLogging = CreateLoggingService("console", logSettings); Log = BootstrapLogging->CreateLog("BLOCKSTORE_SERVER"); + SetCriticalEventsLog(Log); Configs->Log = Log; STORAGE_INFO("NBS server version: " << GetFullVersionString()); @@ -477,17 +479,6 @@ void TBootstrapBase::Init() STORAGE_INFO("RDMA EndpointListener initialized"); } - auto endpointManager = CreateEndpointManager( - Logging, - ServerStats, - Executor, - EndpointEventHandler, - std::move(sessionManager), - std::move(endpointListeners), - Configs->ServerConfig->GetNbdSocketSuffix()); - - STORAGE_INFO("EndpointManager initialized"); - IEndpointStoragePtr endpointStorage; switch (Configs->ServerConfig->GetEndpointStorageType()) { case NCloud::NProto::ENDPOINT_STORAGE_DEFAULT: @@ -507,8 +498,13 @@ void TBootstrapBase::Init() } STORAGE_INFO("EndpointStorage initialized"); - EndpointService = CreateMultipleEndpointService( - std::move(Service), + TEndpointManagerOptions endpointManagerOptions = { + .ClientConfig = Configs->EndpointConfig->GetClientConfig(), + .NbdSocketSuffix = Configs->ServerConfig->GetNbdSocketSuffix(), + .NbdDevicePrefix = Configs->ServerConfig->GetNbdDevicePrefix(), + }; + + EndpointManager = CreateEndpointManager( Timer, Scheduler, Logging, @@ -516,10 +512,20 @@ void TBootstrapBase::Init() VolumeStats, ServerStats, Executor, + EndpointEventHandler, + std::move(sessionManager), std::move(endpointStorage), - std::move(endpointManager), - Configs->EndpointConfig->GetClientConfig()); - Service = EndpointService; + std::move(endpointListeners), + NBD::CreateDeviceConnectionFactory(Logging, TDuration::Days(1)), + std::move(endpointManagerOptions)); + + STORAGE_INFO("EndpointManager initialized"); + + Service = CreateMultipleEndpointService( + std::move(Service), + Timer, + Scheduler, + EndpointManager); STORAGE_INFO("MultipleEndpointService initialized"); @@ -585,7 +591,7 @@ void TBootstrapBase::Init() TVector requestProviders = { Server, - EndpointService + EndpointManager }; if (NbdServer) { @@ -628,6 +634,9 @@ void TBootstrapBase::InitDbgConfigs() Configs->InitEndpointConfig(); Configs->InitHostPerformanceProfile(); Configs->InitDiskAgentConfig(); + // InitRdmaConfig should be called after InitDiskAgentConfig and + // InitServerConfig to backport legacy RDMA config + Configs->InitRdmaConfig(); Configs->InitDiskRegistryProxyConfig(); Configs->InitDiagnosticsConfig(); Configs->InitDiscoveryConfig(); @@ -799,6 +808,7 @@ void TBootstrapBase::Start() START_COMMON_COMPONENT(Spdk); START_KIKIMR_COMPONENT(ActorSystem); START_COMMON_COMPONENT(FileIOService); + START_COMMON_COMPONENT(EndpointManager); START_COMMON_COMPONENT(Service); START_COMMON_COMPONENT(VhostServer); START_COMMON_COMPONENT(NbdServer); @@ -821,7 +831,7 @@ void TBootstrapBase::Start() STORAGE_INFO("Process memory locked"); } - auto restoreFuture = EndpointService->RestoreEndpoints(); + auto restoreFuture = EndpointManager->RestoreEndpoints(); if (!Configs->Options->TemporaryServer) { auto balancerSwitch = VolumeBalancerSwitch; restoreFuture.Subscribe([=] (const auto& future) { @@ -864,6 +874,7 @@ void TBootstrapBase::Stop() STOP_COMMON_COMPONENT(NbdServer); STOP_COMMON_COMPONENT(VhostServer); STOP_COMMON_COMPONENT(Service); + STOP_COMMON_COMPONENT(EndpointManager); STOP_COMMON_COMPONENT(FileIOService); STOP_KIKIMR_COMPONENT(ActorSystem); STOP_COMMON_COMPONENT(Spdk); diff --git a/cloud/blockstore/libs/daemon/common/bootstrap.h b/cloud/blockstore/libs/daemon/common/bootstrap.h index 71c4cabada1..ac8c6e8d33a 100644 --- a/cloud/blockstore/libs/daemon/common/bootstrap.h +++ b/cloud/blockstore/libs/daemon/common/bootstrap.h @@ -68,7 +68,7 @@ class TBootstrapBase NSpdk::ISpdkEnvPtr Spdk; ICachingAllocatorPtr Allocator; IStorageProviderPtr AioStorageProvider; - IEndpointServicePtr EndpointService; + IEndpointManagerPtr EndpointManager; IEndpointEventProxyPtr EndpointEventHandler; NRdma::IServerPtr RdmaServer; NRdma::IClientPtr RdmaClient; diff --git a/cloud/blockstore/libs/daemon/common/config_initializer.cpp b/cloud/blockstore/libs/daemon/common/config_initializer.cpp index ed987273e98..cde897ab583 100644 --- a/cloud/blockstore/libs/daemon/common/config_initializer.cpp +++ b/cloud/blockstore/libs/daemon/common/config_initializer.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -87,6 +88,31 @@ void TConfigInitializerCommon::InitDiskAgentConfig() ); } +void TConfigInitializerCommon::InitRdmaConfig() +{ + NProto::TRdmaConfig rdmaConfig; + + if (Options->RdmaConfig) { + ParseProtoTextFromFileRobust(Options->RdmaConfig, rdmaConfig); + } else { + // no rdma config file is given fallback to legacy config + if (DiskAgentConfig->DeprecatedHasRdmaTarget()) { + rdmaConfig.SetServerEnabled(true); + const auto& rdmaTarget = DiskAgentConfig->DeprecatedGetRdmaTarget(); + rdmaConfig.MutableServer()->CopyFrom(rdmaTarget.GetServer()); + } + + if (ServerConfig->DeprecatedGetRdmaClientEnabled()) { + rdmaConfig.SetClientEnabled(true); + rdmaConfig.MutableClient()->CopyFrom( + ServerConfig->DeprecatedGetRdmaClientConfig()); + } + } + + RdmaConfig = + std::make_shared(rdmaConfig); +} + void TConfigInitializerCommon::InitDiskRegistryProxyConfig() { NProto::TDiskRegistryProxyConfig config; diff --git a/cloud/blockstore/libs/daemon/common/config_initializer.h b/cloud/blockstore/libs/daemon/common/config_initializer.h index 749d8338226..f229d4d5818 100644 --- a/cloud/blockstore/libs/daemon/common/config_initializer.h +++ b/cloud/blockstore/libs/daemon/common/config_initializer.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ struct TConfigInitializerCommon TDiagnosticsConfigPtr DiagnosticsConfig; NSpdk::TSpdkEnvConfigPtr SpdkEnvConfig; NClient::THostPerformanceProfile HostPerformanceProfile; + NRdma::TRdmaConfigPtr RdmaConfig; TString Rack; TLog Log; @@ -49,6 +51,7 @@ struct TConfigInitializerCommon void InitHostPerformanceProfile(); void InitServerConfig(); void InitSpdkEnvConfig(); + void InitRdmaConfig(); virtual bool GetUseNonreplicatedRdmaActor() const = 0; virtual TDuration GetInactiveClientsTimeout() const = 0; diff --git a/cloud/blockstore/libs/daemon/common/options.cpp b/cloud/blockstore/libs/daemon/common/options.cpp index 5fd39bd87a8..c4578897f32 100644 --- a/cloud/blockstore/libs/daemon/common/options.cpp +++ b/cloud/blockstore/libs/daemon/common/options.cpp @@ -47,6 +47,11 @@ TOptionsCommon::TOptionsCommon() .RequiredArgument("FILE") .StoreResult(&DiscoveryConfig); + Opts.AddLongOption("rdma-file") + .RequiredArgument("FILE") + .DefaultValue("") + .StoreResult(&RdmaConfig); + Opts.AddLongOption("temporary-server", "run temporary server for blue-green deployment") .NoArgument() .StoreTrue(&TemporaryServer); diff --git a/cloud/blockstore/libs/daemon/common/options.h b/cloud/blockstore/libs/daemon/common/options.h index 4b2b82e9ecc..efdd36667da 100644 --- a/cloud/blockstore/libs/daemon/common/options.h +++ b/cloud/blockstore/libs/daemon/common/options.h @@ -26,6 +26,7 @@ struct TOptionsCommon TString DiskAgentConfig; TString DiskRegistryProxyConfig; TString EndpointConfig; + TString RdmaConfig; enum class EServiceKind { Null /* "null" */ , diff --git a/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp b/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp index d303a66606e..f77a096fe14 100644 --- a/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp +++ b/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -80,15 +81,10 @@ namespace { //////////////////////////////////////////////////////////////////////////////// -NRdma::TClientConfigPtr CreateRdmaClientConfig(const TServerAppConfigPtr app) +NRdma::TClientConfigPtr CreateRdmaClientConfig( + const NRdma::TRdmaConfigPtr config) { - const auto& server = app->GetServerConfig(); - - if (server->HasRdmaClientConfig()) { - return std::make_shared(server->GetRdmaClientConfig()); - }; - - return std::make_shared(); + return std::make_shared(config->GetClient()); } } // namespace @@ -183,11 +179,11 @@ void TBootstrapYdb::InitSpdk() void TBootstrapYdb::InitRdmaClient() { try { - if (Configs->ServerConfig->GetRdmaClientEnabled()) { + if (Configs->RdmaConfig->GetClientEnabled()) { RdmaClient = ServerModuleFactories->RdmaClientFactory( Logging, Monitoring, - CreateRdmaClientConfig(Configs->ServerConfig)); + CreateRdmaClientConfig(Configs->RdmaConfig)); STORAGE_INFO("RDMA client initialized"); } @@ -243,6 +239,9 @@ void TBootstrapYdb::InitKikimrService() } Configs->InitDiskAgentConfig(); + // InitRdmaConfig should be called after InitDiskAgentConfig and + // InitServerConfig to backport legacy RDMA config + Configs->InitRdmaConfig(); STORAGE_INFO("Configs initialized"); @@ -495,6 +494,7 @@ void TBootstrapYdb::InitKikimrService() args.DiagnosticsConfig = Configs->DiagnosticsConfig; args.StorageConfig = Configs->StorageConfig; args.DiskAgentConfig = Configs->DiskAgentConfig; + args.RdmaConfig = Configs->RdmaConfig; args.DiskRegistryProxyConfig = Configs->DiskRegistryProxyConfig; args.AsyncLogger = AsyncLogger; args.StatsAggregator = StatsAggregator; diff --git a/cloud/blockstore/libs/diagnostics/critical_events.h b/cloud/blockstore/libs/diagnostics/critical_events.h index e556f8a0769..9034826debc 100644 --- a/cloud/blockstore/libs/diagnostics/critical_events.h +++ b/cloud/blockstore/libs/diagnostics/critical_events.h @@ -82,6 +82,7 @@ namespace NCloud::NBlockStore { xxx(MonitoringSvgTemplatesNotFound) \ xxx(DiskRegistryUnexpectedAffectedDisks) \ xxx(ReadBlockCountMismatch) \ + xxx(CancelRoutineIsNotSet) \ // BLOCKSTORE_IMPOSSIBLE_EVENTS //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/blockstore/libs/diagnostics/hostname.cpp b/cloud/blockstore/libs/diagnostics/hostname.cpp index 80e01a7ce84..98ce5961809 100644 --- a/cloud/blockstore/libs/diagnostics/hostname.cpp +++ b/cloud/blockstore/libs/diagnostics/hostname.cpp @@ -112,7 +112,7 @@ TString GetMonitoringNBSOverviewToTVUrl(const TDiagnosticsConfig& config) << data.MonitoringUrl << "/projects/" << data.MonitoringProject << "/dashboards/" << data.MonitoringNBSTVDashboard << "?from=now-1d&to=now&refresh=60000&p.cluster=" - << data.MonitoringClusterName << "&p.host=cluster"; + << data.MonitoringClusterName << "&p.host=" << GetShortHostName(); } TString GetMonitoringYDBGroupUrl( @@ -120,17 +120,20 @@ TString GetMonitoringYDBGroupUrl( ui32 groupId, const TString& storagePool) { - TMonitoringUrlData data = config.GetMonitoringUrlData(); - return TStringBuilder() - << data.MonitoringUrl - << "/projects/kikimr/explorer/" - "queries?q.0.s=histogram_percentile(99, {project=\"kikimr" - << "\", cluster=\"*\", storagePool=\"" << storagePool - << "\", group=\"" << groupId << "\", host=\"" << GetShortHostName() - << "\", service=\"vdisks\", " - "subsystem=\"latency_histo\", " - "handleclass=\"GetFast\"})&q.0.name=A&from=now-1d&to=now&refresh=" - "60000"; + constexpr TStringBuf GetFast = + R"(q.0.s=histogram_percentile(99, {project="kikimr", cluster="*", storagePool="%s", group="%)" PRIu32 + R"(", host="*", service="vdisks", subsystem="latency_histo", handleclass="GetFast"})&q.0.name=A)"; + constexpr TStringBuf PutUserData = + R"(q.1.s=histogram_percentile(99, {project="kikimr", cluster="*", storagePool="%s", group="%)" PRIu32 + R"(", host="*", service="vdisks", subsystem="latency_histo", handleclass="PutUserData"})&q.1.name=B)"; + constexpr TStringBuf Url = + "%s/projects/kikimr/explorer/" + "queries?%s&%s&from=now-1d&to=now&refresh=60000"; + return Sprintf( + Url.data(), + config.GetMonitoringUrlData().MonitoringUrl.c_str(), + Sprintf(GetFast.data(), storagePool.c_str(), groupId).c_str(), + Sprintf(PutUserData.data(), storagePool.c_str(), groupId).c_str()); } } // namespace NCloud::NBlockStore diff --git a/cloud/blockstore/libs/disk_agent/bootstrap.cpp b/cloud/blockstore/libs/disk_agent/bootstrap.cpp index c772e753833..445dde2bc82 100644 --- a/cloud/blockstore/libs/disk_agent/bootstrap.cpp +++ b/cloud/blockstore/libs/disk_agent/bootstrap.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -167,15 +168,9 @@ class TMonitoringProxy final } }; -NRdma::TServerConfigPtr CreateRdmaServerConfig( - NStorage::TDiskAgentConfig& agent) +NRdma::TServerConfigPtr CreateRdmaServerConfig(NRdma::TRdmaConfig& config) { - const auto& target = agent.GetRdmaTarget(); - - if (target.HasServer()) { - return std::make_shared(target.GetServer()); - } - return std::make_shared(); + return std::make_shared(config.GetServer()); } } // namespace @@ -267,10 +262,10 @@ void TBootstrap::InitProfileLog() } } -void TBootstrap::InitRdmaServer(NStorage::TDiskAgentConfig& config) +void TBootstrap::InitRdmaServer(NRdma::TRdmaConfig& config) { try { - if (config.HasRdmaTarget()) { + if (config.GetServerEnabled()) { RdmaServer = ServerModuleFactories->RdmaServerFactory( Logging, Monitoring, @@ -321,6 +316,9 @@ void TBootstrap::InitKikimrService() } Configs->InitDiskAgentConfig(); + // InitRdmaConfig should be called after InitDiskAgentConfig + // to backport legacy RDMA config + Configs->InitRdmaConfig(); STORAGE_INFO("Configs initialized"); @@ -378,7 +376,9 @@ void TBootstrap::InitKikimrService() break; } - InitRdmaServer(config); + if (Configs->RdmaConfig) { + InitRdmaServer(*Configs->RdmaConfig); + } } Allocator = CreateCachingAllocator( @@ -409,6 +409,7 @@ void TBootstrap::InitKikimrService() args.AppConfig = Configs->KikimrConfig; args.StorageConfig = Configs->StorageConfig; args.DiskAgentConfig = Configs->DiskAgentConfig; + args.RdmaConfig = Configs->RdmaConfig; args.DiskRegistryProxyConfig = Configs->DiskRegistryProxyConfig; args.AsyncLogger = AsyncLogger; args.Spdk = Spdk; @@ -522,8 +523,8 @@ void TBootstrap::Start() START_COMPONENT(TraceProcessor); START_COMPONENT(Spdk); START_COMPONENT(RdmaServer); - START_COMPONENT(ActorSystem); START_COMPONENT(FileIOService); + START_COMPONENT(ActorSystem); // we need to start scheduler after all other components for 2 reasons: // 1) any component can schedule a task that uses a dependency that hasn't @@ -553,8 +554,11 @@ void TBootstrap::Stop() // scheduled tasks and shutting down of component dependencies STOP_COMPONENT(Scheduler); - STOP_COMPONENT(FileIOService); STOP_COMPONENT(ActorSystem); + // stop FileIOService after ActorSystem to ensure that there are no + // in-flight I/O requests from TDiskAgentActor + STOP_COMPONENT(FileIOService); + STOP_COMPONENT(Spdk); STOP_COMPONENT(RdmaServer); STOP_COMPONENT(TraceProcessor); diff --git a/cloud/blockstore/libs/disk_agent/bootstrap.h b/cloud/blockstore/libs/disk_agent/bootstrap.h index 5e5a27a442e..e93790338e1 100644 --- a/cloud/blockstore/libs/disk_agent/bootstrap.h +++ b/cloud/blockstore/libs/disk_agent/bootstrap.h @@ -93,7 +93,7 @@ class TBootstrap void InitProfileLog(); void InitKikimrService(); - void InitRdmaServer(NStorage::TDiskAgentConfig& config); + void InitRdmaServer(NRdma::TRdmaConfig& config); }; } // namespace NCloud::NBlockStore::NServer diff --git a/cloud/blockstore/libs/disk_agent/config_initializer.cpp b/cloud/blockstore/libs/disk_agent/config_initializer.cpp index 4498b37e02b..537dbfb069d 100644 --- a/cloud/blockstore/libs/disk_agent/config_initializer.cpp +++ b/cloud/blockstore/libs/disk_agent/config_initializer.cpp @@ -3,6 +3,7 @@ #include "options.h" #include +#include #include #include #include @@ -204,6 +205,25 @@ void TConfigInitializer::InitFeaturesConfig() std::make_shared(featuresConfig); } +void TConfigInitializer::InitRdmaConfig() +{ + NProto::TRdmaConfig rdmaConfig; + + if (Options->RdmaConfig) { + ParseProtoTextFromFileRobust(Options->RdmaConfig, rdmaConfig); + } else { + // no rdma config file is given fallback to legacy config + if (DiskAgentConfig->DeprecatedHasRdmaTarget()) { + rdmaConfig.SetServerEnabled(true); + const auto& rdmaTarget = DiskAgentConfig->DeprecatedGetRdmaTarget(); + rdmaConfig.MutableServer()->CopyFrom(rdmaTarget.GetServer()); + } + } + + RdmaConfig = + std::make_shared(rdmaConfig); +} + NKikimrConfig::TLogConfig TConfigInitializer::GetLogConfig() const { // TODO: move to custom config diff --git a/cloud/blockstore/libs/disk_agent/config_initializer.h b/cloud/blockstore/libs/disk_agent/config_initializer.h index 276bd60b513..e5dba610314 100644 --- a/cloud/blockstore/libs/disk_agent/config_initializer.h +++ b/cloud/blockstore/libs/disk_agent/config_initializer.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ struct TConfigInitializer TDiagnosticsConfigPtr DiagnosticsConfig; NSpdk::TSpdkEnvConfigPtr SpdkEnvConfig; NFeatures::TFeaturesConfigPtr FeaturesConfig; + NRdma::TRdmaConfigPtr RdmaConfig; TString Rack; @@ -52,6 +54,7 @@ struct TConfigInitializer void InitServerConfig(); void InitSpdkEnvConfig(); void InitFeaturesConfig(); + void InitRdmaConfig(); NKikimrConfig::TLogConfig GetLogConfig() const; NKikimrConfig::TMonitoringConfig GetMonitoringConfig() const; diff --git a/cloud/blockstore/libs/disk_agent/options.cpp b/cloud/blockstore/libs/disk_agent/options.cpp index 07c0b90ffbb..c00e99d0bd5 100644 --- a/cloud/blockstore/libs/disk_agent/options.cpp +++ b/cloud/blockstore/libs/disk_agent/options.cpp @@ -24,6 +24,11 @@ TOptions::TOptions() .RequiredArgument("FILE") .StoreResult(&FeaturesConfig); + Opts.AddLongOption("rdma-file") + .RequiredArgument("FILE") + .DefaultValue("") + .StoreResult(&RdmaConfig); + Opts.AddLongOption("syslog-service") .RequiredArgument("STR") .StoreResult(&SysLogService); diff --git a/cloud/blockstore/libs/disk_agent/options.h b/cloud/blockstore/libs/disk_agent/options.h index 934a5867406..8d080bcde24 100644 --- a/cloud/blockstore/libs/disk_agent/options.h +++ b/cloud/blockstore/libs/disk_agent/options.h @@ -19,6 +19,7 @@ struct TOptions TString DiskAgentConfig; TString DiskRegistryProxyConfig; TString FeaturesConfig; + TString RdmaConfig; TString SysLogService; diff --git a/cloud/blockstore/libs/encryption/tsan.supp b/cloud/blockstore/libs/encryption/tsan.supp new file mode 100644 index 00000000000..1b0427525d9 --- /dev/null +++ b/cloud/blockstore/libs/encryption/tsan.supp @@ -0,0 +1,3 @@ +# There is a race between reading and writing 'allow_customize' on initialization +# https://github.com/ydb-platform/nbs/blob/main/contrib/libs/openssl/crypto/mem.c#L204-L211 +race:CRYPTO_malloc diff --git a/cloud/blockstore/libs/endpoints/endpoint_manager.cpp b/cloud/blockstore/libs/endpoints/endpoint_manager.cpp index 691dd16432c..6d23eff2aef 100644 --- a/cloud/blockstore/libs/endpoints/endpoint_manager.cpp +++ b/cloud/blockstore/libs/endpoints/endpoint_manager.cpp @@ -5,26 +5,35 @@ #include "session_manager.h" #include +#include +#include #include #include #include +#include +#include #include #include +#include #include #include #include #include +#include +#include #include #include +#include #include +#include +#include namespace NCloud::NBlockStore::NServer { +using namespace NClient; using namespace NThreading; -using namespace NCloud::NBlockStore::NClient; - namespace { //////////////////////////////////////////////////////////////////////////////// @@ -103,7 +112,7 @@ bool CompareRequests( const NProto::TStartEndpointRequest& left, const NProto::TStartEndpointRequest& right) { - Y_DEBUG_ABORT_UNLESS(24 == GetFieldCount()); + Y_DEBUG_ABORT_UNLESS(26 == GetFieldCount()); return left.GetUnixSocketPath() == right.GetUnixSocketPath() && left.GetDiskId() == right.GetDiskId() && left.GetInstanceId() == right.GetInstanceId() @@ -130,11 +139,171 @@ bool CompareRequests( left.GetClientCGroups().end(), right.GetClientCGroups().begin(), right.GetClientCGroups().end()) - && left.GetPersistent() == right.GetPersistent(); + && left.GetPersistent() == right.GetPersistent() + && left.GetNbdDeviceFile() == right.GetNbdDeviceFile() + && left.GetUseFreeNbdDeviceFile() == right.GetUseFreeNbdDeviceFile(); +} + +bool CompareRequests( + const NProto::TStopEndpointRequest& left, + const NProto::TStopEndpointRequest& right) +{ + Y_DEBUG_ABORT_UNLESS(2 == GetFieldCount()); + return left.GetUnixSocketPath() == right.GetUnixSocketPath(); +} + +bool CompareRequests( + const NProto::TRefreshEndpointRequest& left, + const NProto::TRefreshEndpointRequest& right) +{ + Y_DEBUG_ABORT_UNLESS(2 == GetFieldCount()); + return left.GetUnixSocketPath() == right.GetUnixSocketPath(); } //////////////////////////////////////////////////////////////////////////////// +class TDeviceManager +{ +private: + const TString DevicePrefix; + TVector BusyDevices; + +public: + TDeviceManager(TString devicePrefix) + : DevicePrefix(std::move(devicePrefix)) + { + size_t num = 0; + while (NFs::Exists(GetDeviceName(num))) { + ++num; + } + BusyDevices.resize(num, false); + } + + NProto::TError AcquireDevice(const TString& device) + { + size_t num = 0; + if (!GetDeviceNum(device, num)) { + return MakeError(E_ARGUMENT, TStringBuilder() + << "Couldn't parse nbd device file: " << device); + } + + if (!NFs::Exists(device)) { + return MakeError(E_INVALID_STATE, TStringBuilder() + << "No file for nbd device: " << device); + } + + if (num < BusyDevices.size() && BusyDevices[num]) { + return MakeError(E_INVALID_STATE, TStringBuilder() + << "Nbd device file doesn't exist: " << device); + } + + if (num >= BusyDevices.size()) { + BusyDevices.resize(num + 1, false); + } + + BusyDevices[num] = true; + return {}; + } + + void ReleaseDevice(const TString& device) + { + if (device.empty()) { + return; + } + + size_t num = 0; + bool res = GetDeviceNum(device, num); + Y_ENSURE(res && num < BusyDevices.size() && BusyDevices[num]); + + BusyDevices[num] = false; + } + + TString GetFreeDevice() + { + size_t num = 0; + for (; num < BusyDevices.size(); ++num) { + if (!BusyDevices[num]) { + break; + } + } + + return GetDeviceName(num); + } + +private: + bool GetDeviceNum(const TString& device, size_t& num) + { + if (!device.StartsWith(DevicePrefix)) { + return false; + } + + auto numStr = device.substr(DevicePrefix.size()); + return TryFromString(numStr, num); + } + + TString GetDeviceName(size_t num) + { + return DevicePrefix + ToString(num); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TBlockStoreNotImplemented + : public IBlockStore +{ +public: + void Start() override + {} + + void Stop() override + {} + + TStorageBuffer AllocateBuffer(size_t bytesCount) override + { + Y_UNUSED(bytesCount); + return nullptr; + } + +#define BLOCKSTORE_IMPLEMENT_METHOD(name, ...) \ + TFuture name( \ + TCallContextPtr ctx, \ + std::shared_ptr request) override \ + { \ + Y_UNUSED(ctx); \ + Y_UNUSED(request); \ + return MakeFuture(TErrorResponse( \ + E_NOT_IMPLEMENTED, "Unsupported request")); \ + } \ +// BLOCKSTORE_IMPLEMENT_METHOD + + BLOCKSTORE_SERVICE(BLOCKSTORE_IMPLEMENT_METHOD) + +#undef BLOCKSTORE_IMPLEMENT_METHOD +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TEndpointManager; + +class TRestoringClient final + : public TBlockStoreNotImplemented +{ +private: + TEndpointManager& EndpointManager; + +public: + TRestoringClient(TEndpointManager& endpointManager) + : EndpointManager(endpointManager) + {} + + TFuture StartEndpoint( + TCallContextPtr ctx, + std::shared_ptr req) override; +}; + +//////////////////////////////////////////////////////////////////////////////// + #define BLOCKSTORE_DECLARE_METHOD(name, ...) \ struct T##name##Method \ { \ @@ -147,6 +316,8 @@ bool CompareRequests( #undef BLOCKSTORE_DECLARE_METHOD +//////////////////////////////////////////////////////////////////////////////// + class TSwitchEndpointRequest { private: @@ -184,9 +355,21 @@ class TSwitchEndpointRequest { Reason = reason; } - }; +//////////////////////////////////////////////////////////////////////////////// + +bool CompareRequests( + const TSwitchEndpointRequest& left, + const TSwitchEndpointRequest& right) +{ + return left.GetDiskId() == right.GetDiskId() + && left.GetUnixSocketPath() == right.GetUnixSocketPath() + && left.GetReason() == right.GetReason(); +} + +//////////////////////////////////////////////////////////////////////////////// + struct TSwitchEndpointMethod { using TRequest = TSwitchEndpointRequest; @@ -204,17 +387,31 @@ struct TRequestState //////////////////////////////////////////////////////////////////////////////// +struct TEndpoint +{ + std::shared_ptr Request; + NBD::IDeviceConnectionPtr Device; + NProto::TVolume Volume; +}; + +//////////////////////////////////////////////////////////////////////////////// + class TEndpointManager final : public IEndpointManager , public IEndpointEventHandler + , public std::enable_shared_from_this { private: + const ILoggingServicePtr Logging; const IServerStatsPtr ServerStats; const TExecutorPtr Executor; const ISessionManagerPtr SessionManager; + const IEndpointStoragePtr EndpointStorage; const THashMap EndpointListeners; + const NBD::IDeviceConnectionFactoryPtr NbdDeviceFactory; const TString NbdSocketSuffix; + TDeviceManager NbdDeviceManager; TLog Log; using TRequestStateVariant = std::variant< @@ -225,23 +422,87 @@ class TEndpointManager final >; THashMap ProcessingSockets; - THashMap> Requests; + THashMap Endpoints; + + NClient::IMetricClientPtr RestoringClient; + TSet RestoringEndpoints; + + enum { + WaitingForRestoring = 0, + ReadingStorage = 1, + StartingEndpoints = 2, + Completed = 3, + }; + + TAtomic RestoringStage = WaitingForRestoring; public: TEndpointManager( + ITimerPtr timer, + ISchedulerPtr scheduler, ILoggingServicePtr logging, + IRequestStatsPtr requestStats, + IVolumeStatsPtr volumeStats, IServerStatsPtr serverStats, TExecutorPtr executor, ISessionManagerPtr sessionManager, + IEndpointStoragePtr endpointStorage, THashMap listeners, - TString nbdSocketSuffix) - : ServerStats(std::move(serverStats)) + NBD::IDeviceConnectionFactoryPtr nbdDeviceFactory, + TEndpointManagerOptions options) + : Logging(std::move(logging)) + , ServerStats(std::move(serverStats)) , Executor(std::move(executor)) , SessionManager(std::move(sessionManager)) + , EndpointStorage(std::move(endpointStorage)) , EndpointListeners(std::move(listeners)) - , NbdSocketSuffix(std::move(nbdSocketSuffix)) + , NbdDeviceFactory(std::move(nbdDeviceFactory)) + , NbdSocketSuffix(options.NbdSocketSuffix) + , NbdDeviceManager(options.NbdDevicePrefix) + { + Log = Logging->CreateLog("BLOCKSTORE_SERVER"); + + IBlockStorePtr client = std::make_shared(*this); + + NProto::TClientAppConfig config; + *config.MutableClientConfig() = options.ClientConfig; + auto appConfig = std::make_shared(std::move(config)); + auto retryPolicy = CreateRetryPolicy(appConfig); + + client = CreateDurableClient( + std::move(appConfig), + std::move(client), + std::move(retryPolicy), + Logging, + std::move(timer), + std::move(scheduler), + std::move(requestStats), + std::move(volumeStats)); + + RestoringClient = NClient::CreateMetricClient( + std::move(client), + Logging, + ServerStats); + } + + void Start() override + { + RestoringClient->Start(); + } + + void Stop() override + { + RestoringClient->Stop(); + + for (auto& [socket, endpoint]: Endpoints) { + endpoint.Device->Stop(); + } + } + + size_t CollectRequests( + const TIncompleteRequestsCollector& collector) override { - Log = logging->CreateLog("BLOCKSTORE_SERVER"); + return RestoringClient->CollectRequests(collector); } #define ENDPOINT_IMPLEMENT_METHOD(name, specifier, ...) \ @@ -263,11 +524,7 @@ class TEndpointManager final std::shared_ptr req); \ // ENDPOINT_IMPLEMENT_METHOD - ENDPOINT_IMPLEMENT_METHOD(StartEndpoint, override) - ENDPOINT_IMPLEMENT_METHOD(StopEndpoint, override) - ENDPOINT_IMPLEMENT_METHOD(ListEndpoints, override) - ENDPOINT_IMPLEMENT_METHOD(DescribeEndpoint, override) - ENDPOINT_IMPLEMENT_METHOD(RefreshEndpoint, override) + BLOCKSTORE_ENDPOINT_SERVICE(ENDPOINT_IMPLEMENT_METHOD, override) ENDPOINT_IMPLEMENT_METHOD(SwitchEndpoint, ) #undef ENDPOINT_IMPLEMENT_METHOD @@ -276,10 +533,37 @@ class TEndpointManager final const TString& diskId, const TString& reason) override; + TFuture RestoreEndpoints() override + { + AtomicSet(RestoringStage, ReadingStorage); + return Executor->Execute([this] () mutable { + auto future = DoRestoreEndpoints(); + AtomicSet(RestoringStage, StartingEndpoints); + Executor->WaitFor(future); + AtomicSet(RestoringStage, Completed); + }); + } + + TFuture RestoreSingleEndpoint( + TCallContextPtr ctx, + std::shared_ptr request); + private: + bool IsEndpointRestoring(const TString& socket) + { + switch (AtomicGet(RestoringStage)) { + case WaitingForRestoring: return false; + case ReadingStorage: return true; + case StartingEndpoints: return RestoringEndpoints.contains(socket); + case Completed: return false; + } + return false; + } + NProto::TStartEndpointResponse StartEndpointImpl( TCallContextPtr ctx, - std::shared_ptr request); + std::shared_ptr request, + bool restoring); NProto::TStopEndpointResponse StopEndpointImpl( TCallContextPtr ctx, @@ -293,12 +577,16 @@ class TEndpointManager final TCallContextPtr ctx, std::shared_ptr request); - NProto::TStartEndpointResponse AlterEndpoint( + NProto::TError AlterEndpoint( + TCallContextPtr ctx, + NProto::TStartEndpointRequest newReq, + NProto::TStartEndpointRequest oldReq); + + NProto::TError RestartListenerEndpoint( TCallContextPtr ctx, - const NProto::TStartEndpointRequest& newRequest, - const NProto::TStartEndpointRequest& oldRequest); + const NProto::TStartEndpointRequest& request); - NProto::TError OpenEndpointSockets( + NProto::TError OpenAllEndpointSockets( const NProto::TStartEndpointRequest& request, const TSessionInfo& sessionInfo); @@ -306,23 +594,89 @@ class TEndpointManager final const NProto::TStartEndpointRequest& request, const TSessionInfo& sessionInfo); - TVector> CloseEndpointSockets( - const NProto::TStartEndpointRequest& startRequest); - - TFuture CloseEndpointSocket( - const NProto::TStartEndpointRequest& startRequest); + void CloseAllEndpointSockets(const NProto::TStartEndpointRequest& request); + void CloseEndpointSocket(const NProto::TStartEndpointRequest& request); std::shared_ptr CreateNbdStartEndpointRequest( const NProto::TStartEndpointRequest& request); + TResultOrError StartNbdDevice( + std::shared_ptr request, + bool restoring); + + void ReleaseNbdDevice(const TString& device, bool restoring); + + void DetachFileDevice(const TString& device); + + template + void RemoveSession(TCallContextPtr ctx, const T& request) + { + auto future = SessionManager->RemoveSession( + std::move(ctx), + request.GetUnixSocketPath(), + request.GetHeaders()); + + auto error = Executor->WaitFor(future); + if (HasError(error)) { + STORAGE_ERROR("Failed to remove session: " << FormatError(error)); + } + } + + TFuture DoRestoreEndpoints(); + + void HandleRestoredEndpoint( + const TString& socket, + const NProto::TError& error); + + NProto::TError AddEndpointToStorage( + const NProto::TStartEndpointRequest& request) + { + if (!request.GetPersistent()) { + return {}; + } + + auto [data, error] = SerializeEndpoint(request); + if (HasError(error)) { + return error; + } + + return EndpointStorage->AddEndpoint(request.GetUnixSocketPath(), data); + } + template TPromise AddProcessingSocket( const typename TMethod::TRequest& request) { auto promise = NewPromise(); + const auto& socketPath = request.GetUnixSocketPath(); + + auto it = ProcessingSockets.find(socketPath); + if (it != ProcessingSockets.end()) { + const auto& st = it->second; + auto* state = std::get_if>(&st); + if (!state) { + auto response = TErrorResponse(E_REJECTED, TStringBuilder() + << "endpoint " << socketPath.Quote() + << " is " << GetProcessName(st) << " now"); + promise.SetValue(response); + return promise; + } + + if (!CompareRequests(request, state->Request)) { + auto response = TErrorResponse(E_REJECTED, TStringBuilder() + << "endpoint " << socketPath.Quote() + << " is " << GetProcessName(st) << " now with other args"); + promise.SetValue(response); + return promise; + } + + auto response = Executor->WaitFor(state->Result); + promise.SetValue(response); + return promise; + } auto [_, inserted] = ProcessingSockets.emplace( - request.GetUnixSocketPath(), + socketPath, TRequestState{request, promise.GetFuture()}); Y_ABORT_UNLESS(inserted); @@ -334,11 +688,9 @@ class TEndpointManager final ProcessingSockets.erase(socketPath); } - TErrorResponse MakeBusySocketError( - const TString& socketPath, - const TRequestStateVariant& st) + TString GetProcessName(const TRequestStateVariant& st) { - auto processName = std::visit(TOverloaded{ + return std::visit(TOverloaded{ [] (const TRequestState&) { return "starting"; }, @@ -355,41 +707,35 @@ class TEndpointManager final return "busy (undefined process)"; } }, st); - - return TErrorResponse(E_REJECTED, TStringBuilder() - << "endpoint " << socketPath.Quote() - << " is " << processName << " now"); } }; //////////////////////////////////////////////////////////////////////////////// +TFuture TEndpointManager::RestoreSingleEndpoint( + TCallContextPtr ctx, + std::shared_ptr request) +{ + return Executor->Execute([this, ctx, request] () mutable { + return StartEndpointImpl(std::move(ctx), std::move(request), true); + }); +} + NProto::TStartEndpointResponse TEndpointManager::DoStartEndpoint( TCallContextPtr ctx, std::shared_ptr request) { auto socketPath = request->GetUnixSocketPath(); - - auto it = ProcessingSockets.find(socketPath); - if (it != ProcessingSockets.end()) { - const auto& st = it->second; - auto* state = std::get_if>(&st); - if (!state) { - return MakeBusySocketError(socketPath, st); - } - - if (!AreSameStartEndpointRequests(*request, state->Request)) { - return TErrorResponse(E_REJECTED, TStringBuilder() - << "endpoint " << socketPath.Quote() - << " is starting now with other args"); - } - - return Executor->WaitFor(state->Result); + if (IsEndpointRestoring(socketPath)) { + return TErrorResponse(E_REJECTED, "endpoint is restoring now"); } auto promise = AddProcessingSocket(*request); + if (promise.HasValue()) { + return promise.ExtractValue(); + } - auto response = StartEndpointImpl(std::move(ctx), std::move(request)); + auto response = StartEndpointImpl(std::move(ctx), std::move(request), false); promise.SetValue(response); RemoveProcessingSocket(socketPath); @@ -398,13 +744,38 @@ NProto::TStartEndpointResponse TEndpointManager::DoStartEndpoint( NProto::TStartEndpointResponse TEndpointManager::StartEndpointImpl( TCallContextPtr ctx, - std::shared_ptr request) + std::shared_ptr request, + bool restoring) { - auto socketPath = request->GetUnixSocketPath(); + const auto& socketPath = request->GetUnixSocketPath(); + + auto it = Endpoints.find(socketPath); + if (it != Endpoints.end()) { + auto endpoint = it->second; + + if (!NFs::Exists(socketPath)) { + // restart listener endpoint to recreate the socket + auto error = RestartListenerEndpoint(ctx, *endpoint.Request); + if (HasError(error)) { + return TErrorResponse(error); + } + + if (!NFs::Exists(socketPath)) { + return TErrorResponse(E_INVALID_STATE, TStringBuilder() + << "failed to recreate socket: " << socketPath.Quote()); + } + } + + auto error = AlterEndpoint(std::move(ctx), *request, *endpoint.Request); + if (HasError(error)) { + return TErrorResponse(error); + } - auto it = Requests.find(socketPath); - if (it != Requests.end()) { - return AlterEndpoint(std::move(ctx), *request, *it->second); + NProto::TStartEndpointResponse response; + response.MutableError()->CopyFrom(error); + response.MutableVolume()->CopyFrom(endpoint.Volume); + response.SetNbdDeviceFile(endpoint.Request->GetNbdDeviceFile()); + return response; } auto future = SessionManager->CreateSession(ctx, *request); @@ -413,68 +784,89 @@ NProto::TStartEndpointResponse TEndpointManager::StartEndpointImpl( return TErrorResponse(error); } - error = OpenEndpointSockets(*request, sessionInfo); + error = OpenAllEndpointSockets(*request, sessionInfo); if (HasError(error)) { - auto future = SessionManager->RemoveSession( - std::move(ctx), - socketPath, - request->GetHeaders()); - Executor->WaitFor(future); + RemoveSession(std::move(ctx), *request); + return TErrorResponse(error); + } + + auto deviceOrError = StartNbdDevice(request, restoring); + error = deviceOrError.GetError(); + if (HasError(error)) { + CloseAllEndpointSockets(*request); + RemoveSession(std::move(ctx), *request); return TErrorResponse(error); } + auto device = deviceOrError.ExtractResult(); + + if (!restoring) { + error = AddEndpointToStorage(*request); + if (HasError(error)) { + device->Stop(); + ReleaseNbdDevice(request->GetNbdDeviceFile(), restoring); + CloseAllEndpointSockets(*request); + RemoveSession(std::move(ctx), *request); + return TErrorResponse(error); + } + } + + TEndpoint endpoint = { + .Request = request, + .Device = device, + .Volume = sessionInfo.Volume, + }; if (auto c = ServerStats->GetEndpointCounter(request->GetIpcType())) { c->Inc(); } - auto [_, inserted] = Requests.emplace(socketPath, std::move(request)); + auto [_, inserted] = Endpoints.emplace(socketPath, std::move(endpoint)); STORAGE_VERIFY(inserted, TWellKnownEntityTypes::ENDPOINT, socketPath); NProto::TStartEndpointResponse response; response.MutableVolume()->CopyFrom(sessionInfo.Volume); + response.SetNbdDeviceFile(request->GetNbdDeviceFile()); return response; } -NProto::TStartEndpointResponse TEndpointManager::AlterEndpoint( +NProto::TError TEndpointManager::AlterEndpoint( TCallContextPtr ctx, - const NProto::TStartEndpointRequest& newRequest, - const NProto::TStartEndpointRequest& oldRequest) + NProto::TStartEndpointRequest newReq, + NProto::TStartEndpointRequest oldReq) { - const auto& socketPath = newRequest.GetUnixSocketPath(); - - auto startedEndpoint = oldRequest; + const auto& socketPath = newReq.GetUnixSocketPath(); // NBS-3018 - if (!CompareRequests( - oldRequest.GetClientProfile(), - newRequest.GetClientProfile())) - { + if (!CompareRequests(oldReq.GetClientProfile(), newReq.GetClientProfile())) { STORAGE_WARN("Modified ClientProfile will be ignored for endpoint: " << socketPath.Quote()); - startedEndpoint.MutableClientProfile()->CopyFrom( - newRequest.GetClientProfile()); + oldReq.MutableClientProfile()->CopyFrom(newReq.GetClientProfile()); } // CLOUD-98154 - if (oldRequest.GetDeviceName() != newRequest.GetDeviceName()) { + if (oldReq.GetDeviceName() != newReq.GetDeviceName()) { STORAGE_WARN("Modified DeviceName will be ignored for endpoint: " << socketPath.Quote()); - startedEndpoint.SetDeviceName(newRequest.GetDeviceName()); + oldReq.SetDeviceName(newReq.GetDeviceName()); + } + + if (newReq.GetUseFreeNbdDeviceFile() && oldReq.GetNbdDeviceFile()) { + newReq.SetNbdDeviceFile(oldReq.GetNbdDeviceFile()); } - if (CompareRequests(newRequest, startedEndpoint)) { - return TErrorResponse(S_ALREADY, TStringBuilder() + if (CompareRequests(newReq, oldReq)) { + return MakeError(S_ALREADY, TStringBuilder() << "endpoint " << socketPath.Quote() << " has already been started"); } - startedEndpoint.SetVolumeAccessMode(newRequest.GetVolumeAccessMode()); - startedEndpoint.SetVolumeMountMode(newRequest.GetVolumeMountMode()); - startedEndpoint.SetMountSeqNumber(newRequest.GetMountSeqNumber()); + oldReq.SetVolumeAccessMode(newReq.GetVolumeAccessMode()); + oldReq.SetVolumeMountMode(newReq.GetVolumeMountMode()); + oldReq.SetMountSeqNumber(newReq.GetMountSeqNumber()); - if (!CompareRequests(newRequest, startedEndpoint)) { - return TErrorResponse(E_INVALID_STATE, TStringBuilder() + if (!CompareRequests(newReq, oldReq)) { + return MakeError(E_INVALID_STATE, TStringBuilder() << "endpoint " << socketPath.Quote() << " has already been started with other args"); } @@ -482,25 +874,25 @@ NProto::TStartEndpointResponse TEndpointManager::AlterEndpoint( auto future = SessionManager->AlterSession( ctx, socketPath, - newRequest.GetVolumeAccessMode(), - newRequest.GetVolumeMountMode(), - newRequest.GetMountSeqNumber(), - newRequest.GetHeaders()); + newReq.GetVolumeAccessMode(), + newReq.GetVolumeMountMode(), + newReq.GetMountSeqNumber(), + newReq.GetHeaders()); if (auto error = Executor->WaitFor(future); HasError(error)) { - return TErrorResponse(error); + return error; } auto [sessionInfo, error] = Executor->WaitFor(SessionManager->GetSession( ctx, socketPath, - newRequest.GetHeaders())); + newReq.GetHeaders())); if (HasError(error)) { - return TErrorResponse(error); + return error; } - auto listenerIt = EndpointListeners.find(startedEndpoint.GetIpcType()); + auto listenerIt = EndpointListeners.find(oldReq.GetIpcType()); STORAGE_VERIFY( listenerIt != EndpointListeners.end(), TWellKnownEntityTypes::ENDPOINT, @@ -509,11 +901,55 @@ NProto::TStartEndpointResponse TEndpointManager::AlterEndpoint( auto& listener = listenerIt->second; auto alterFuture = listener->AlterEndpoint( - startedEndpoint, + oldReq, + sessionInfo.Volume, + sessionInfo.Session); + + return Executor->WaitFor(alterFuture); +} + +NProto::TError TEndpointManager::RestartListenerEndpoint( + TCallContextPtr ctx, + const NProto::TStartEndpointRequest& request) +{ + STORAGE_INFO("Restart listener endpoint: " << request); + + auto sessionFuture = SessionManager->GetSession( + ctx, + request.GetUnixSocketPath(), + request.GetHeaders()); + + auto [sessionInfo, error] = Executor->WaitFor(sessionFuture); + if (HasError(error)) { + return error; + } + + auto listenerIt = EndpointListeners.find(request.GetIpcType()); + STORAGE_VERIFY( + listenerIt != EndpointListeners.end(), + TWellKnownEntityTypes::ENDPOINT, + request.GetUnixSocketPath()); + + auto& listener = listenerIt->second; + + auto future = listener->StopEndpoint(request.GetUnixSocketPath()); + error = Executor->WaitFor(future); + if (HasError(error)) { + STORAGE_ERROR("Failed to stop endpoint while restarting it: " + << FormatError(error)); + } + + future = listener->StartEndpoint( + request, sessionInfo.Volume, sessionInfo.Session); + error = Executor->WaitFor(future); + if (HasError(error)) { + STORAGE_ERROR("Failed to start endpoint while recreating it: " + << FormatError(error)); + } - return TErrorResponse(Executor->WaitFor(alterFuture)); + return error; } NProto::TStopEndpointResponse TEndpointManager::DoStopEndpoint( @@ -521,19 +957,14 @@ NProto::TStopEndpointResponse TEndpointManager::DoStopEndpoint( std::shared_ptr request) { auto socketPath = request->GetUnixSocketPath(); - - auto it = ProcessingSockets.find(socketPath); - if (it != ProcessingSockets.end()) { - const auto& st = it->second; - auto* state = std::get_if>(&st); - if (!state) { - return MakeBusySocketError(socketPath, st); - } - - return Executor->WaitFor(state->Result); + if (IsEndpointRestoring(socketPath)) { + return TErrorResponse(E_REJECTED, "endpoint is restoring now"); } auto promise = AddProcessingSocket(*request); + if (promise.HasValue()) { + return promise.ExtractValue(); + } auto response = StopEndpointImpl(std::move(ctx), std::move(request)); promise.SetValue(response); @@ -546,36 +977,33 @@ NProto::TStopEndpointResponse TEndpointManager::StopEndpointImpl( TCallContextPtr ctx, std::shared_ptr request) { - auto socketPath = request->GetUnixSocketPath(); + const auto& socketPath = request->GetUnixSocketPath(); - auto it = Requests.find(socketPath); - if (it == Requests.end()) { + auto it = Endpoints.find(socketPath); + if (it == Endpoints.end()) { return TErrorResponse(S_FALSE, TStringBuilder() << "endpoint " << socketPath.Quote() << " hasn't been started yet"); } - auto startRequest = std::move(it->second); - Requests.erase(it); - if (auto c = ServerStats->GetEndpointCounter(startRequest->GetIpcType())) { + auto endpoint = std::move(it->second); + Endpoints.erase(it); + if (auto c = ServerStats->GetEndpointCounter(endpoint.Request->GetIpcType())) { c->Dec(); } - auto futures = CloseEndpointSockets(*startRequest); - auto future = SessionManager->RemoveSession( - std::move(ctx), - socketPath, - request->GetHeaders()); - futures.push_back(future); + endpoint.Device->Stop(); + ReleaseNbdDevice(endpoint.Request->GetNbdDeviceFile(), false); + CloseAllEndpointSockets(*endpoint.Request); + RemoveSession(std::move(ctx), *request); - NProto::TError result; - for (const auto& future: futures) { - auto error = Executor->WaitFor(future); - if (HasError(error)) { - result = error; - } + auto error = EndpointStorage->RemoveEndpoint(socketPath); + if (HasError(error)) { + STORAGE_ERROR("Failed to remove endpoint from storage: " + << FormatError(error)); } - return TErrorResponse(result); + + return {}; } NProto::TListEndpointsResponse TEndpointManager::DoListEndpoints( @@ -586,12 +1014,82 @@ NProto::TListEndpointsResponse TEndpointManager::DoListEndpoints( Y_UNUSED(request); NProto::TListEndpointsResponse response; + auto& responseEndpoints = *response.MutableEndpoints(); + responseEndpoints.Reserve(Endpoints.size()); + + for (auto [_, endpoint]: Endpoints) { + responseEndpoints.Add()->CopyFrom(*endpoint.Request); + } + + response.SetEndpointsWereRestored( + AtomicGet(RestoringStage) == Completed); + return response; +} + +NProto::TKickEndpointResponse TEndpointManager::DoKickEndpoint( + TCallContextPtr ctx, + std::shared_ptr request) +{ + auto keyringId = ToString(request->GetKeyringId()); + auto [str, error] = EndpointStorage->GetEndpoint(keyringId); + if (HasError(error)) { + return TErrorResponse(error); + } + + auto startReq = DeserializeEndpoint(str); + if (!startReq) { + return TErrorResponse(E_INVALID_STATE, TStringBuilder() + << "Failed to deserialize endpoint with key " + << request->GetKeyringId()); + } + + startReq->MutableHeaders()->MergeFrom(request->GetHeaders()); + startReq->SetPersistent(false); + + STORAGE_INFO("Kick StartEndpoint request: " << *startReq); + auto response = DoStartEndpoint( + std::move(ctx), + std::move(startReq)); + + return TErrorResponse(response.GetError()); +} + +NProto::TListKeyringsResponse TEndpointManager::DoListKeyrings( + TCallContextPtr ctx, + std::shared_ptr request) +{ + Y_UNUSED(ctx); + Y_UNUSED(request); + + auto [storedIds, error] = EndpointStorage->GetEndpointIds(); + if (HasError(error)) { + return TErrorResponse(error); + } + + NProto::TListKeyringsResponse response; auto& endpoints = *response.MutableEndpoints(); - endpoints.Reserve(Requests.size()); - for (auto it: Requests) { + endpoints.Reserve(storedIds.size()); + + for (auto keyringId: storedIds) { auto& endpoint = *endpoints.Add(); - endpoint.CopyFrom(*it.second); + endpoint.SetKeyringId(keyringId); + + auto [str, error] = EndpointStorage->GetEndpoint(keyringId); + if (HasError(error)) { + STORAGE_WARN("Failed to get endpoint from storage, ID: " + << keyringId << ", error: " << FormatError(error)); + continue; + } + + auto req = DeserializeEndpoint(str); + if (!req) { + STORAGE_WARN("Failed to deserialize endpoint from storage, ID: " + << keyringId); + continue; + } + + endpoint.MutableRequest()->CopyFrom(*req); } return response; @@ -599,13 +1097,18 @@ NProto::TListEndpointsResponse TEndpointManager::DoListEndpoints( NProto::TDescribeEndpointResponse TEndpointManager::DoDescribeEndpoint( TCallContextPtr ctx, - std::shared_ptr req) + std::shared_ptr request) { Y_UNUSED(ctx); + auto socketPath = request->GetUnixSocketPath(); + if (IsEndpointRestoring(socketPath)) { + return TErrorResponse(E_REJECTED, "endpoint is restoring now"); + } + NProto::TDescribeEndpointResponse response; - auto [profile, err] = SessionManager->GetProfile(req->GetUnixSocketPath()); + auto [profile, err] = SessionManager->GetProfile(socketPath); if (HasError(err)) { response.MutableError()->CopyFrom(err); } else { @@ -620,19 +1123,14 @@ NProto::TRefreshEndpointResponse TEndpointManager::DoRefreshEndpoint( std::shared_ptr request) { auto socketPath = request->GetUnixSocketPath(); - - auto it = ProcessingSockets.find(socketPath); - if (it != ProcessingSockets.end()) { - const auto& st = it->second; - auto* state = std::get_if>(&st); - if (!state) { - return MakeBusySocketError(socketPath, st); - } - - return Executor->WaitFor(state->Result); + if (IsEndpointRestoring(socketPath)) { + return TErrorResponse(E_REJECTED, "endpoint is restoring now"); } auto promise = AddProcessingSocket(*request); + if (promise.HasValue()) { + return promise.ExtractValue(); + } auto response = RefreshEndpointImpl(std::move(ctx), std::move(request)); promise.SetValue(response); @@ -648,13 +1146,13 @@ NProto::TRefreshEndpointResponse TEndpointManager::RefreshEndpointImpl( const auto& socketPath = request->GetUnixSocketPath(); const auto& headers = request->GetHeaders(); - auto it = Requests.find(socketPath); - if (it == Requests.end()) { + auto it = Endpoints.find(socketPath); + if (it == Endpoints.end()) { return TErrorResponse(S_FALSE, TStringBuilder() << "endpoint " << socketPath.Quote() << " not started"); } - auto ipcType = it->second->GetIpcType(); + auto ipcType = it->second.Request->GetIpcType(); auto listenerIt = EndpointListeners.find(ipcType); STORAGE_VERIFY( listenerIt != EndpointListeners.end(), @@ -672,7 +1170,7 @@ NProto::TRefreshEndpointResponse TEndpointManager::RefreshEndpointImpl( return TErrorResponse(error); } -NProto::TError TEndpointManager::OpenEndpointSockets( +NProto::TError TEndpointManager::OpenAllEndpointSockets( const NProto::TStartEndpointRequest& request, const TSessionInfo& sessionInfo) { @@ -687,8 +1185,7 @@ NProto::TError TEndpointManager::OpenEndpointSockets( auto error = OpenEndpointSocket(*nbdRequest, sessionInfo); if (HasError(error)) { - auto closeFuture = CloseEndpointSocket(request); - Executor->WaitFor(closeFuture); + CloseEndpointSocket(request); return error; } } @@ -702,13 +1199,13 @@ NProto::TError TEndpointManager::OpenEndpointSocket( auto ipcType = request.GetIpcType(); auto listenerIt = EndpointListeners.find(ipcType); if (listenerIt == EndpointListeners.end()) { - return TErrorResponse(E_ARGUMENT, TStringBuilder() + return MakeError(E_ARGUMENT, TStringBuilder() << "unsupported endpoint type: " << static_cast(ipcType)); } auto listener = listenerIt->second; if (request.GetUnixSocketPath().size() > UnixSocketPathLengthLimit) { - return TErrorResponse(E_ARGUMENT, TStringBuilder() + return MakeError(E_ARGUMENT, TStringBuilder() << "Length of socket path should not be more than " << UnixSocketPathLengthLimit); } @@ -721,36 +1218,38 @@ NProto::TError TEndpointManager::OpenEndpointSocket( return Executor->WaitFor(future); } -TVector> TEndpointManager::CloseEndpointSockets( - const NProto::TStartEndpointRequest& startRequest) +void TEndpointManager::CloseAllEndpointSockets( + const NProto::TStartEndpointRequest& request) { - TVector> futures; - - auto future = CloseEndpointSocket(startRequest); - futures.push_back(future); + CloseEndpointSocket(request); - auto nbdRequest = CreateNbdStartEndpointRequest(startRequest); + auto nbdRequest = CreateNbdStartEndpointRequest(request); if (nbdRequest) { STORAGE_INFO("Stop additional endpoint: " << nbdRequest->GetUnixSocketPath().Quote()); - auto future = CloseEndpointSocket(*nbdRequest); - futures.push_back(future); + CloseEndpointSocket(*nbdRequest); } - return futures; } -TFuture TEndpointManager::CloseEndpointSocket( - const NProto::TStartEndpointRequest& startRequest) +void TEndpointManager::CloseEndpointSocket( + const NProto::TStartEndpointRequest& request) { - auto ipcType = startRequest.GetIpcType(); + auto ipcType = request.GetIpcType(); + const auto& socketPath = request.GetUnixSocketPath(); + auto listenerIt = EndpointListeners.find(ipcType); STORAGE_VERIFY( listenerIt != EndpointListeners.end(), TWellKnownEntityTypes::ENDPOINT, - startRequest.GetUnixSocketPath()); + socketPath); const auto& listener = listenerIt->second; - return listener->StopEndpoint(startRequest.GetUnixSocketPath()); + auto future = listener->StopEndpoint(socketPath); + auto error = Executor->WaitFor(future); + if (HasError(error)) { + STORAGE_ERROR("Failed to close socket " << socketPath.Quote() + << ", error: " << FormatError(error)); + } } using TStartEndpointRequestPtr = std::shared_ptr; @@ -778,31 +1277,27 @@ NProto::TError TEndpointManager::DoSwitchEndpoint( { const auto& diskId = request->GetDiskId(); - auto reqIt = FindIf(Requests, [&] (auto& v) { - const auto& [_, req] = v; - return req->GetDiskId() == diskId - && req->GetIpcType() == NProto::IPC_VHOST; + auto it = FindIf(Endpoints, [&] (auto& v) { + const auto& [_, endpoint] = v; + return endpoint.Request->GetDiskId() == diskId + && endpoint.Request->GetIpcType() == NProto::IPC_VHOST; }); - if (reqIt == Requests.end()) { - return TErrorResponse(S_OK); + if (it == Endpoints.end()) { + return MakeError(S_OK); } - const auto& socketPath = reqIt->first; - - auto sockIt = ProcessingSockets.find(socketPath); - if (sockIt != ProcessingSockets.end()) { - const auto& st = sockIt->second; - auto* state = std::get_if>(&st); - if (!state) { - return MakeBusySocketError(socketPath, st); - } - - return Executor->WaitFor(state->Result); + auto socketPath = it->first; + if (IsEndpointRestoring(socketPath)) { + return MakeError(E_REJECTED, "endpoint is restoring now"); } request->SetUnixSocketPath(socketPath); + auto promise = AddProcessingSocket(*request); + if (promise.HasValue()) { + return promise.ExtractValue(); + } auto response = SwitchEndpointImpl(std::move(ctx), std::move(request)); promise.SetValue(response); @@ -817,13 +1312,13 @@ NProto::TError TEndpointManager::SwitchEndpointImpl( { const auto& socketPath = request->GetUnixSocketPath(); - auto it = Requests.find(socketPath); - if (it == Requests.end()) { - return TErrorResponse(S_FALSE, TStringBuilder() + auto it = Endpoints.find(socketPath); + if (it == Endpoints.end()) { + return MakeError(S_FALSE, TStringBuilder() << "endpoint " << socketPath.Quote() << " not started"); } - auto& startRequest = it->second; + auto startRequest = it->second.Request; auto listenerIt = EndpointListeners.find(startRequest->GetIpcType()); STORAGE_VERIFY( listenerIt != EndpointListeners.end(), @@ -858,7 +1353,61 @@ NProto::TError TEndpointManager::SwitchEndpointImpl( << ", " << error.GetMessage()); } - return TErrorResponse(error); + return error; +} + +TResultOrError TEndpointManager::StartNbdDevice( + std::shared_ptr request, + bool restoring) +{ + if (request->GetIpcType() != NProto::IPC_NBD) { + return NBD::CreateDeviceConnectionStub(); + } + + if (request->HasUseFreeNbdDeviceFile() && + request->GetUseFreeNbdDeviceFile()) + { + if (restoring) { + return MakeError(E_ARGUMENT, + "Forbidden 'FreeNbdDeviceFile' flag in restoring endpoints"); + } + + auto nbdDevice = NbdDeviceManager.GetFreeDevice(); + request->SetUseFreeNbdDeviceFile(false); + request->SetNbdDeviceFile(nbdDevice); + } + + if (!request->HasNbdDeviceFile() || !request->GetNbdDeviceFile()) { + return NBD::CreateDeviceConnectionStub(); + } + + if (!restoring) { + if (!request->GetPersistent()) { + return MakeError(E_ARGUMENT, + "Only persistent endpoints can connect to nbd device"); + } + + const auto& nbdDevice = request->GetNbdDeviceFile(); + auto error = NbdDeviceManager.AcquireDevice(nbdDevice); + if (HasError(error)) { + return error; + } + } + + NBD::IDeviceConnectionPtr device; + try { + // Release file descriptor lock by removing loopback device + DetachFileDevice(request->GetNbdDeviceFile()); + + TNetworkAddress address(TUnixSocketPath(request->GetUnixSocketPath())); + device = NbdDeviceFactory->Create(address, request->GetNbdDeviceFile()); + device->Start(); + } catch (...) { + ReleaseNbdDevice(request->GetNbdDeviceFile(), restoring); + return MakeError(E_FAIL, CurrentExceptionMessage()); + } + + return device; } TFuture TEndpointManager::SwitchEndpointIfNeeded( @@ -873,26 +1422,174 @@ TFuture TEndpointManager::SwitchEndpointIfNeeded( return SwitchEndpoint(std::move(ctx), std::move(request)); } +TFuture TEndpointManager::DoRestoreEndpoints() +{ + auto [storedIds, error] = EndpointStorage->GetEndpointIds(); + if (HasError(error)) { + STORAGE_ERROR("Failed to get endpoints from storage: " + << FormatError(error)); + ReportEndpointRestoringError(); + return MakeFuture(); + } + + STORAGE_INFO("Found " << storedIds.size() << " endpoints in storage"); + + TString clientId = CreateGuidAsString() + "_bootstrap"; + + TVector> futures; + + for (auto keyringId: storedIds) { + auto [str, error] = EndpointStorage->GetEndpoint(keyringId); + if (HasError(error)) { + // NBS-3678 + STORAGE_WARN("Failed to restore endpoint. ID: " << keyringId + << ", error: " << FormatError(error)); + continue; + } + + auto request = DeserializeEndpoint(str); + + if (!request) { + ReportEndpointRestoringError(); + STORAGE_ERROR("Failed to deserialize request. ID: " << keyringId); + continue; + } + + if (request->HasNbdDeviceFile() && request->GetNbdDeviceFile()) { + const auto& nbdDevice = request->GetNbdDeviceFile(); + auto error = NbdDeviceManager.AcquireDevice(nbdDevice); + + if (HasError(error)) { + ReportEndpointRestoringError(); + STORAGE_ERROR("Failed to acquire nbd device" + << ", endpoint: " << request->GetUnixSocketPath().Quote() + << ", error: " << FormatError(error)); + continue; + } + } + + auto& headers = *request->MutableHeaders(); + + if (!headers.GetClientId()) { + headers.SetClientId(clientId); + } + + auto requestId = headers.GetRequestId(); + if (!requestId) { + headers.SetRequestId(CreateRequestId()); + } + + auto socketPath = request->GetUnixSocketPath(); + RestoringEndpoints.insert(socketPath); + + auto future = RestoringClient->StartEndpoint( + MakeIntrusive(requestId), + std::move(request)); + + auto weakPtr = weak_from_this(); + future.Subscribe([weakPtr, socketPath] (const auto& f) { + const auto& response = f.GetValue(); + if (HasError(response)) { + ReportEndpointRestoringError(); + } + + if (auto ptr = weakPtr.lock()) { + ptr->HandleRestoredEndpoint(socketPath, response.GetError()); + } + }); + + futures.push_back(future.IgnoreResult()); + } + + return WaitAll(futures); +} + +void TEndpointManager::HandleRestoredEndpoint( + const TString& socketPath, + const NProto::TError& error) +{ + if (HasError(error)) { + STORAGE_ERROR("Failed to start endpoint " << socketPath.Quote() + << ", error:" << FormatError(error)); + } + + Executor->Execute([socketPath, this] () mutable { + RestoringEndpoints.erase(socketPath); + }); +} + +void TEndpointManager::ReleaseNbdDevice(const TString& device, bool restoring) +{ + if (restoring) { + return; + } + + NbdDeviceManager.ReleaseDevice(device); +} + +void TEndpointManager::DetachFileDevice(const TString& device) +{ + STORAGE_DEBUG("Detach file device " << device); + + auto mountedFiles = NBD::FindMountedFiles(device); + STORAGE_DEBUG("find mounted files: " << JoinSeq(",", mountedFiles)); + + auto loopbackDevices = NBD::FindLoopbackDevices(mountedFiles); + for (const auto& loopbackDevice: loopbackDevices) { + STORAGE_INFO("Remove loopback device " << loopbackDevice.Quote() + << " to unlock block device " << device.Quote()); + + int result = NBD::RemoveLoopbackDevice(loopbackDevice); + if (result != 0) { + throw TServiceError(E_FAIL) << "failed to remove loopback device " + << loopbackDevice.Quote() << " with error: " << result; + } + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TFuture TRestoringClient::StartEndpoint( + TCallContextPtr ctx, + std::shared_ptr req) +{ + return EndpointManager.RestoreSingleEndpoint( + std::move(ctx), + std::move(req)); +} + } // namespace //////////////////////////////////////////////////////////////////////////////// IEndpointManagerPtr CreateEndpointManager( + ITimerPtr timer, + ISchedulerPtr scheduler, ILoggingServicePtr logging, + IRequestStatsPtr requestStats, + IVolumeStatsPtr volumeStats, IServerStatsPtr serverStats, TExecutorPtr executor, IEndpointEventProxyPtr eventProxy, ISessionManagerPtr sessionManager, + IEndpointStoragePtr endpointStorage, THashMap listeners, - TString nbdSocketSuffix) + NBD::IDeviceConnectionFactoryPtr nbdDeviceFactory, + TEndpointManagerOptions options) { auto manager = std::make_shared( + std::move(timer), + std::move(scheduler), std::move(logging), + std::move(requestStats), + std::move(volumeStats), std::move(serverStats), std::move(executor), std::move(sessionManager), + std::move(endpointStorage), std::move(listeners), - std::move(nbdSocketSuffix)); + std::move(nbdDeviceFactory), + std::move(options)); eventProxy->Register(manager); return manager; } diff --git a/cloud/blockstore/libs/endpoints/endpoint_manager.h b/cloud/blockstore/libs/endpoints/endpoint_manager.h index 353cad16550..4ed18abd7c9 100644 --- a/cloud/blockstore/libs/endpoints/endpoint_manager.h +++ b/cloud/blockstore/libs/endpoints/endpoint_manager.h @@ -2,12 +2,17 @@ #include "public.h" +#include #include #include -#include +#include +#include +#include #include +#include #include +#include #include @@ -16,40 +21,49 @@ namespace NCloud::NBlockStore::NServer { //////////////////////////////////////////////////////////////////////////////// struct IEndpointManager + : public IStartable + , public IIncompleteRequestProvider { virtual ~IEndpointManager() = default; - virtual NThreading::TFuture StartEndpoint( - TCallContextPtr ctx, - std::shared_ptr request) = 0; +#define ENDPOINT_DECLARE_METHOD(name, ...) \ + virtual NThreading::TFuture name( \ + TCallContextPtr ctx, \ + std::shared_ptr req) = 0; \ +// ENDPOINT_DECLARE_METHOD - virtual NThreading::TFuture StopEndpoint( - TCallContextPtr ctx, - std::shared_ptr request) = 0; + BLOCKSTORE_ENDPOINT_SERVICE(ENDPOINT_DECLARE_METHOD) - virtual NThreading::TFuture ListEndpoints( - TCallContextPtr ctx, - std::shared_ptr request) = 0; +#undef ENDPOINT_DECLARE_METHOD - virtual NThreading::TFuture DescribeEndpoint( - TCallContextPtr ctx, - std::shared_ptr request) = 0; + virtual NThreading::TFuture RestoreEndpoints() = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// - virtual NThreading::TFuture RefreshEndpoint( - TCallContextPtr ctx, - std::shared_ptr request) = 0; +struct TEndpointManagerOptions +{ + NProto::TClientConfig ClientConfig; + TString NbdSocketSuffix; + TString NbdDevicePrefix = "/dev/nbd"; }; //////////////////////////////////////////////////////////////////////////////// IEndpointManagerPtr CreateEndpointManager( + ITimerPtr timer, + ISchedulerPtr scheduler, ILoggingServicePtr logging, + IRequestStatsPtr requestStats, + IVolumeStatsPtr volumeStats, IServerStatsPtr serverStats, TExecutorPtr executor, IEndpointEventProxyPtr eventProxy, ISessionManagerPtr sessionManager, + IEndpointStoragePtr endpointStorage, THashMap listeners, - TString nbdSocketSuffix); + NBD::IDeviceConnectionFactoryPtr nbdDeviceFactory, + TEndpointManagerOptions options); bool AreSameStartEndpointRequests( const NProto::TStartEndpointRequest& left, diff --git a/cloud/blockstore/libs/endpoints/endpoint_manager_ut.cpp b/cloud/blockstore/libs/endpoints/endpoint_manager_ut.cpp index a6f7f6e82a3..47f1ebbdbac 100644 --- a/cloud/blockstore/libs/endpoints/endpoint_manager_ut.cpp +++ b/cloud/blockstore/libs/endpoints/endpoint_manager_ut.cpp @@ -9,29 +9,37 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include +#include #include #include #include #include +#include #include #include #include #include #include +#include +#include #include #include +#include #include +#include #include namespace NCloud::NBlockStore::NServer { @@ -120,6 +128,23 @@ struct TTestSessionManager final //////////////////////////////////////////////////////////////////////////////// +struct TTestDeviceFactory + : public NBD::IDeviceConnectionFactory +{ + TVector Devices; + + NBD::IDeviceConnectionPtr Create( + TNetworkAddress connectAddress, + TString deviceName) override + { + Y_UNUSED(connectAddress); + Devices.push_back(deviceName); + return NBD::CreateDeviceConnectionStub(); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + struct TTestEndpoint { NProto::TStartEndpointRequest Request; @@ -140,6 +165,10 @@ class TTestEndpointListener final ui32 AlterEndpointCounter = 0; ui32 SwitchEndpointCounter = 0; + using TStartEndpointHandler = std::function( + const NProto::TStartEndpointRequest& request, + NClient::ISessionPtr session)>; + public: TTestEndpointListener( TFuture result = MakeFuture()) @@ -152,11 +181,16 @@ class TTestEndpointListener final NClient::ISessionPtr session) override { Y_UNUSED(volume); + return StartEndpointHandler(request, session); + } + TStartEndpointHandler StartEndpointHandler = [&] ( + const NProto::TStartEndpointRequest& request, + NClient::ISessionPtr session) + { UNIT_ASSERT(!Endpoints.contains(request.GetUnixSocketPath())); - TTestEndpoint endpoint; - endpoint.Request = request; + TFsPath(request.GetUnixSocketPath()).Touch(); Endpoints.emplace( request.GetUnixSocketPath(), @@ -166,7 +200,7 @@ class TTestEndpointListener final }); return Result; - } + }; TFuture AlterEndpoint( const NProto::TStartEndpointRequest& request, @@ -183,7 +217,7 @@ class TTestEndpointListener final TFuture StopEndpoint(const TString& socketPath) override { Endpoints.erase(socketPath); - + TFsPath(socketPath).DeleteIfExists(); return Result; } @@ -220,17 +254,34 @@ class TTestEndpointListener final struct TBootstrap { - const ILoggingServicePtr Logging = CreateLoggingService("console"); - const IBlockStorePtr Service; - const TExecutorPtr Executor = TExecutor::Create("TestService"); - - TBootstrap(IBlockStorePtr service) - : Service(std::move(service)) - {} + const TString DirPath = "./" + CreateGuidAsString(); + ILoggingServicePtr Logging = CreateLoggingService("console"); + ITimerPtr Timer = CreateWallClockTimer(); + std::shared_ptr Scheduler = std::make_shared(); + IBlockStorePtr Service = std::make_shared(); + TExecutorPtr Executor = TExecutor::Create("TestService"); + IRequestStatsPtr RequestStats = CreateRequestStatsStub(); + IVolumeStatsPtr VolumeStats = CreateVolumeStatsStub(); + IServerStatsPtr ServerStats = CreateServerStatsStub(); + ISessionManagerPtr SessionManager; + IEndpointStoragePtr EndpointStorage = CreateFileEndpointStorage(DirPath); + IMutableEndpointStoragePtr MutableStorage = CreateFileMutableEndpointStorage(DirPath); + THashMap EndpointListeners; + NBD::IDeviceConnectionFactoryPtr NbdDeviceFactory; + IEndpointEventProxyPtr EndpointEventHandler = CreateEndpointEventProxy(); + TEndpointManagerOptions Options; + IEndpointManagerPtr EndpointManager; + + TBootstrap() + { + MutableStorage->Init(); + } ~TBootstrap() { Stop(); + + MutableStorage->Remove(); } void Start() @@ -246,10 +297,18 @@ struct TBootstrap if (Executor) { Executor->Start(); } + + if (EndpointManager) { + EndpointManager->Start(); + } } void Stop() { + if (EndpointManager) { + EndpointManager->Stop(); + } + if (Executor) { Executor->Stop(); } @@ -365,43 +424,48 @@ std::shared_ptr CreateTestService( //////////////////////////////////////////////////////////////////////////////// -IEndpointManagerPtr CreateEndpointManager( - TBootstrap& bootstrap, - THashMap endpointListeners, - IServerStatsPtr serverStats = CreateServerStatsStub(), - TString nbdSocketSuffix = "", - IEndpointEventProxyPtr endpointEventHandler = CreateEndpointEventProxy()) +IEndpointManagerPtr CreateEndpointManager(TBootstrap& bootstrap) { - TSessionManagerOptions sessionManagerOptions; - sessionManagerOptions.DefaultClientConfig.SetRequestTimeout( - TestRequestTimeout.MilliSeconds()); + if (!bootstrap.SessionManager) { + TSessionManagerOptions sessionManagerOptions; + sessionManagerOptions.DefaultClientConfig.SetRequestTimeout( + TestRequestTimeout.MilliSeconds()); - auto encryptionClientFactory = CreateEncryptionClientFactory( - bootstrap.Logging, - CreateDefaultEncryptionKeyProvider()); + auto encryptionClientFactory = CreateEncryptionClientFactory( + bootstrap.Logging, + CreateDefaultEncryptionKeyProvider()); - auto sessionManager = CreateSessionManager( - CreateWallClockTimer(), - CreateSchedulerStub(), - bootstrap.Logging, - CreateMonitoringServiceStub(), - CreateRequestStatsStub(), - CreateVolumeStatsStub(), - serverStats, - bootstrap.Service, - CreateDefaultStorageProvider(bootstrap.Service), - encryptionClientFactory, - bootstrap.Executor, - sessionManagerOptions); + bootstrap.SessionManager = CreateSessionManager( + bootstrap.Timer, + bootstrap.Scheduler, + bootstrap.Logging, + CreateMonitoringServiceStub(), + bootstrap.RequestStats, + bootstrap.VolumeStats, + bootstrap.ServerStats, + bootstrap.Service, + CreateDefaultStorageProvider(bootstrap.Service), + std::move(encryptionClientFactory), + bootstrap.Executor, + std::move(sessionManagerOptions)); + } - return NServer::CreateEndpointManager( + bootstrap.EndpointManager = NServer::CreateEndpointManager( + bootstrap.Timer, + bootstrap.Scheduler, bootstrap.Logging, - serverStats, + bootstrap.RequestStats, + bootstrap.VolumeStats, + bootstrap.ServerStats, bootstrap.Executor, - std::move(endpointEventHandler), - std::move(sessionManager), - std::move(endpointListeners), - std::move(nbdSocketSuffix)); + bootstrap.EndpointEventHandler, + bootstrap.SessionManager, + bootstrap.EndpointStorage, + bootstrap.EndpointListeners, + bootstrap.NbdDeviceFactory, + bootstrap.Options); + + return bootstrap.EndpointManager; } //////////////////////////////////////////////////////////////////////////////// @@ -443,19 +507,23 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) { Y_UNIT_TEST(ShouldHandleStartStopEndpoint) { - TString unixSocket = "testSocket"; + TTempDir dir; + TString unixSocket = (dir.Path() / "testSocket").GetPath(); TString diskId = "testDiskId"; auto ipcType = NProto::IPC_GRPC; + TBootstrap bootstrap; TMap mountedVolumes; - TBootstrap bootstrap(CreateTestService(mountedVolumes)); + bootstrap.Service = CreateTestService(mountedVolumes); auto listener = std::make_shared(); - auto manager = CreateEndpointManager( - bootstrap, - {{ ipcType, listener }}); + bootstrap.EndpointListeners = {{ ipcType, listener }}; + auto manager = CreateEndpointManager(bootstrap); bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; { NProto::TStartEndpointRequest request; @@ -485,19 +553,23 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) Y_UNIT_TEST(ShouldChangeMountModesUsingStartEndpoint) { - auto unixSocket = "testSocket"; + TTempDir dir; + TString unixSocket = (dir.Path() / "testSocket").GetPath(); auto ipcType = NProto::IPC_GRPC; TString diskId = "testDiskId"; + TBootstrap bootstrap; TMap mountedVolumes; - TBootstrap bootstrap(CreateTestService(mountedVolumes)); + bootstrap.Service = CreateTestService(mountedVolumes); auto listener = std::make_shared(); - auto manager = CreateEndpointManager( - bootstrap, - {{ ipcType, listener }}); + bootstrap.EndpointListeners = {{ ipcType, listener }}; + auto manager = CreateEndpointManager(bootstrap); bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; UNIT_ASSERT_VALUES_EQUAL(0, listener->AlterEndpointCounter); @@ -568,28 +640,33 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) Y_UNIT_TEST(ShouldHandleListEndpoints) { + TBootstrap bootstrap; TMap mountedVolumes; - TBootstrap bootstrap(CreateTestService(mountedVolumes)); + bootstrap.Service = CreateTestService(mountedVolumes); - auto manager = CreateEndpointManager( - bootstrap, - { - { NProto::IPC_GRPC, std::make_shared() }, - { NProto::IPC_NBD, std::make_shared() }, - }); + bootstrap.EndpointListeners = { + { NProto::IPC_GRPC, std::make_shared() }, + { NProto::IPC_NBD, std::make_shared() }, + }; + auto manager = CreateEndpointManager(bootstrap); bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; + + TTempDir dir; NProto::TStartEndpointRequest request1; SetDefaultHeaders(request1); - request1.SetUnixSocketPath("testSocket1"); + request1.SetUnixSocketPath((dir.Path() / "testSocket1").GetPath()); request1.SetDiskId("testDiskId1"); request1.SetClientId(TestClientId); request1.SetIpcType(NProto::IPC_GRPC); NProto::TStartEndpointRequest request2; SetDefaultHeaders(request2); - request2.SetUnixSocketPath("testSocket2"); + request2.SetUnixSocketPath((dir.Path() / "testSocket2").GetPath()); request2.SetDiskId("testDiskId2"); request2.SetClientId(TestClientId); request2.SetIpcType(NProto::IPC_NBD); @@ -632,17 +709,21 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) Y_UNIT_TEST(ShouldNotStartStopEndpointTwice) { + TBootstrap bootstrap; TMap mountedVolumes; - TBootstrap bootstrap(CreateTestService(mountedVolumes)); + bootstrap.Service = CreateTestService(mountedVolumes); auto listener = std::make_shared(); - auto manager = CreateEndpointManager( - bootstrap, - {{ NProto::IPC_GRPC, listener }}); + bootstrap.EndpointListeners = {{ NProto::IPC_GRPC, listener }}; + auto manager = CreateEndpointManager(bootstrap); bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; - auto socketPath = "testSocketPath"; + TTempDir dir; + auto socketPath = (dir.Path() / "testSocket").GetPath(); auto diskId = "testDiskId"; NProto::TStartEndpointRequest startRequest; @@ -661,7 +742,7 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) { auto future = StartEndpoint(*manager, startRequest); auto response = future.GetValue(TDuration::Seconds(5)); - UNIT_ASSERT(response.GetError().GetCode() == S_ALREADY); + UNIT_ASSERT_C(response.GetError().GetCode() == S_ALREADY, response.GetError()); } UNIT_ASSERT(mountedVolumes.contains(diskId)); @@ -685,22 +766,25 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) Y_UNIT_TEST(ShouldNotStartBusyEndpoint) { + TBootstrap bootstrap; TMap mountedVolumes; - TBootstrap bootstrap(CreateTestService(mountedVolumes)); + bootstrap.Service = CreateTestService(mountedVolumes); auto grpcListener = std::make_shared(); auto nbdListener = std::make_shared(); + bootstrap.EndpointListeners = { + { NProto::IPC_GRPC, grpcListener }, + { NProto::IPC_NBD, nbdListener }, + }; - auto manager = CreateEndpointManager( - bootstrap, - { - { NProto::IPC_GRPC, grpcListener }, - { NProto::IPC_NBD, nbdListener }, - }); - + auto manager = CreateEndpointManager(bootstrap); bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; - auto socketPath = "testSocketPath"; + TTempDir dir; + auto socketPath = (dir.Path() / "testSocket").GetPath(); { NProto::TStartEndpointRequest request; @@ -735,22 +819,26 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) Y_UNIT_TEST(ShouldNotMountDiskWhenStartEndpointFailed) { + TBootstrap bootstrap; TMap mountedVolumes; - TBootstrap bootstrap(CreateTestService(mountedVolumes)); + bootstrap.Service = CreateTestService(mountedVolumes); auto error = TErrorResponse(E_FAIL, "Endpoint listener is broken"); auto listener = std::make_shared( MakeFuture(error)); + bootstrap.EndpointListeners = {{ NProto::IPC_GRPC, listener }}; - auto manager = CreateEndpointManager( - bootstrap, - {{ NProto::IPC_GRPC, listener }}); - + auto manager = CreateEndpointManager(bootstrap); bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; + + TTempDir dir; NProto::TStartEndpointRequest request; SetDefaultHeaders(request); - request.SetUnixSocketPath("testSocket"); + request.SetUnixSocketPath((dir.Path() / "testSocket").GetPath()); request.SetDiskId("testDiskId"); request.SetClientId(TestClientId); request.SetIpcType(NProto::IPC_GRPC); @@ -762,17 +850,21 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) Y_UNIT_TEST(ShouldHandleLocalRequests) { + TBootstrap bootstrap; TMap mountedVolumes; - TBootstrap bootstrap(CreateTestService(mountedVolumes)); + bootstrap.Service = CreateTestService(mountedVolumes); auto listener = std::make_shared(); - auto manager = CreateEndpointManager( - bootstrap, - {{ NProto::IPC_GRPC, listener }}); + bootstrap.EndpointListeners = {{ NProto::IPC_GRPC, listener }}; + auto manager = CreateEndpointManager(bootstrap); bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; - auto unixSocket = "testSocket"; + TTempDir dir; + TString unixSocket = (dir.Path() / "testSocket").GetPath(); { NProto::TStartEndpointRequest request; @@ -874,18 +966,22 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) ++unmountCounter; }; + TBootstrap bootstrap; TMap mountedVolumes; - TBootstrap bootstrap(CreateTestService(mountedVolumes)); + bootstrap.Service = CreateTestService(mountedVolumes); auto listener = std::make_shared(); - auto manager = CreateEndpointManager( - bootstrap, - {{ NProto::IPC_VHOST, listener }}, - serverStats); + bootstrap.EndpointListeners = {{ NProto::IPC_VHOST, listener }}; + bootstrap.ServerStats = serverStats; + auto manager = CreateEndpointManager(bootstrap); bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; - auto unixSocket = "testSocket"; + TTempDir dir; + TString unixSocket = (dir.Path() / "testSocket").GetPath(); { NProto::TStartEndpointRequest request; @@ -911,17 +1007,19 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) Y_UNIT_TEST(ShouldNotStartEndpointWithSocketPathLongerThanLimit) { + TBootstrap bootstrap; TMap mountedVolumes; - TBootstrap bootstrap(CreateTestService(mountedVolumes)); + bootstrap.Service = CreateTestService(mountedVolumes); auto grpcListener = CreateSocketEndpointListener(bootstrap.Logging, 16); grpcListener->SetClientStorageFactory(CreateClientStorageFactoryStub()); + bootstrap.EndpointListeners = {{ NProto::IPC_GRPC, grpcListener }}; - auto manager = CreateEndpointManager( - bootstrap, - {{ NProto::IPC_GRPC, grpcListener }}); - + auto manager = CreateEndpointManager(bootstrap); bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; TString maxSocketPath(UnixSocketPathLengthLimit, 'x'); @@ -959,26 +1057,28 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) Y_UNIT_TEST(ShouldStartStopNbdEndpointWithGrpcEndpoint) { - TString unixSocket = "testSocket"; + TTempDir dir; + TString unixSocket = (dir.Path() / "testSocket").GetPath(); TString diskId = "testDiskId"; TString nbdSocketSuffix = "_nbd"; + TBootstrap bootstrap; TMap mountedVolumes; - TBootstrap bootstrap(CreateTestService(mountedVolumes)); + bootstrap.Service = CreateTestService(mountedVolumes); auto grpcListener = std::make_shared(); auto nbdListener = std::make_shared(); + bootstrap.EndpointListeners = { + { NProto::IPC_GRPC, grpcListener }, + { NProto::IPC_NBD, nbdListener }, + }; + bootstrap.Options.NbdSocketSuffix = nbdSocketSuffix; - auto manager = CreateEndpointManager( - bootstrap, - { - { NProto::IPC_GRPC, grpcListener }, - { NProto::IPC_NBD, nbdListener }, - }, - CreateServerStatsStub(), - nbdSocketSuffix); - + auto manager = CreateEndpointManager(bootstrap); bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; { NProto::TStartEndpointRequest request; @@ -1022,28 +1122,29 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) Y_UNIT_TEST(ShouldIgnoreInstanceIdWhenCompareStartEndpointRequests) { + TBootstrap bootstrap; TMap mountedVolumes; - TBootstrap bootstrap(CreateTestService(mountedVolumes)); + bootstrap.Service = CreateTestService(mountedVolumes); auto sessionManager = std::make_shared(); - auto manager = NServer::CreateEndpointManager( - bootstrap.Logging, - CreateServerStatsStub(), - bootstrap.Executor, - CreateEndpointEventProxy(), - sessionManager, - {{ NProto::IPC_GRPC, std::make_shared() }}, - "" // NbdSocketSuffix - ); + bootstrap.SessionManager = sessionManager; + + auto listener = std::make_shared(); + bootstrap.EndpointListeners = {{ NProto::IPC_GRPC, listener }}; + auto manager = CreateEndpointManager(bootstrap); bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; + TTempDir dir; size_t requestId = 42; NProto::TStartEndpointRequest request; SetDefaultHeaders(request); request.MutableHeaders()->SetRequestId(++requestId); - request.SetUnixSocketPath("testSocket"); + request.SetUnixSocketPath((dir.Path() / "testSocket").GetPath()); request.SetDiskId("testDiskId"); request.SetClientId(TestClientId); request.SetInstanceId("testInstanceId"); @@ -1121,8 +1222,11 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) Y_UNIT_TEST(ShouldCompareStartEndpointRequestsWithoutHeaders) { + TTempDir dir; + TString unixSocket = (dir.Path() / "testSocket").GetPath(); + NProto::TStartEndpointRequest request1; - request1.SetUnixSocketPath("testUnixSocketPath"); + request1.SetUnixSocketPath(unixSocket); request1.SetDiskId("testDiskId"); request1.SetInstanceId("testInstanceId"); request1.SetClientId("testClientId"); @@ -1145,17 +1249,21 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) // NBS-3018, CLOUD-98154 Y_UNIT_TEST(ShouldIgnoreSomeArgsWhenStartEndpointTwice) { + TBootstrap bootstrap; TMap mountedVolumes; - TBootstrap bootstrap(CreateTestService(mountedVolumes)); + bootstrap.Service = CreateTestService(mountedVolumes); auto listener = std::make_shared(); - auto manager = CreateEndpointManager( - bootstrap, - {{ NProto::IPC_GRPC, listener }}); + bootstrap.EndpointListeners = {{ NProto::IPC_GRPC, listener }}; + auto manager = CreateEndpointManager(bootstrap); bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; - auto socketPath = "testSocketPath"; + TTempDir dir; + auto socketPath = (dir.Path() / "testSocket").GetPath(); auto diskId = "testDiskId"; NProto::TStartEndpointRequest startRequest; @@ -1189,26 +1297,26 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) Y_UNIT_TEST(ShouldSwitchEndpointWhenEndpointStarted) { + TBootstrap bootstrap; TMap mountedVolumes; - TBootstrap bootstrap(CreateTestService(mountedVolumes)); + bootstrap.Service = CreateTestService(mountedVolumes); - auto endpointEventHandler = CreateEndpointEventProxy(); auto listener = std::make_shared(); - auto manager = CreateEndpointManager( - bootstrap, - {{ NProto::IPC_VHOST, listener }}, - CreateServerStatsStub(), - "", - endpointEventHandler); + bootstrap.EndpointListeners = {{ NProto::IPC_VHOST, listener }}; + auto manager = CreateEndpointManager(bootstrap); bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; - auto socketPath = "testSocketPath"; + TTempDir dir; + auto socketPath = (dir.Path() / "testSocket").GetPath(); auto diskId = "testDiskId"; { // without started endpoint SwitchEndpointIfNeeded is ignored - auto future = endpointEventHandler->SwitchEndpointIfNeeded( + auto future = bootstrap.EndpointEventHandler->SwitchEndpointIfNeeded( diskId, "test"); auto error = future.GetValue(TDuration::Seconds(5)); UNIT_ASSERT_VALUES_EQUAL_C( @@ -1237,7 +1345,7 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) { // with started endpoint SwitchEndpointIfNeeded leads to // SwitchEndpoint call - auto future = endpointEventHandler->SwitchEndpointIfNeeded( + auto future = bootstrap.EndpointEventHandler->SwitchEndpointIfNeeded( diskId, "test"); auto error = future.GetValue(TDuration::Seconds(5)); @@ -1248,6 +1356,481 @@ Y_UNIT_TEST_SUITE(TEndpointManagerTest) UNIT_ASSERT_VALUES_EQUAL(1, listener->SwitchEndpointCounter); } } + + Y_UNIT_TEST(ShouldStartEndpointWithNbdDevice) + { + TString nbdDevPrefix = CreateGuidAsString() + "_nbd"; + int deviceCount = 6; + for (int i = 0; i < deviceCount; ++i) { + TFsPath(nbdDevPrefix + ToString(i)).Touch(); + } + Y_DEFER { + for (int i = 0; i < deviceCount; ++i) { + TFsPath(nbdDevPrefix + ToString(i)).DeleteIfExists(); + } + }; + + TBootstrap bootstrap; + TMap mountedVolumes; + bootstrap.Service = CreateTestService(mountedVolumes); + + auto listener = std::make_shared(); + bootstrap.EndpointListeners = {{ NProto::IPC_NBD, listener }}; + + auto deviceFactory = std::make_shared(); + bootstrap.NbdDeviceFactory = deviceFactory; + + bootstrap.Options.NbdDevicePrefix = nbdDevPrefix; + + auto manager = CreateEndpointManager(bootstrap); + bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; + + auto& storage = *bootstrap.EndpointStorage; + google::protobuf::util::MessageDifferencer comparator; + + TTempDir dir; + TString unixSocket = (dir.Path() / "testSocket").GetPath(); + TString diskId = "testDiskId"; + TString nbdDevFile = nbdDevPrefix + "0"; + + NProto::TStartEndpointRequest baseRequest; + SetDefaultHeaders(baseRequest); + baseRequest.SetUnixSocketPath(unixSocket); + baseRequest.SetDiskId(diskId); + baseRequest.SetClientId(TestClientId); + baseRequest.SetIpcType(NProto::IPC_NBD); + baseRequest.SetNbdDeviceFile(nbdDevFile); + baseRequest.SetPersistent(true); + + { + auto request = baseRequest; + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(!HasError(response), response.GetError()); + + UNIT_ASSERT(mountedVolumes.contains(diskId)); + UNIT_ASSERT(listener->GetEndpoints().contains(unixSocket)); + + UNIT_ASSERT_VALUES_EQUAL(1, deviceFactory->Devices.size()); + UNIT_ASSERT_VALUES_EQUAL(nbdDevFile, deviceFactory->Devices[0]); + + auto [str, error] = storage.GetEndpoint(request.GetUnixSocketPath()); + UNIT_ASSERT(!HasError(error)); + auto req = DeserializeEndpoint(str); + UNIT_ASSERT(req); + UNIT_ASSERT(comparator.Equals(request, *req)); + } + + { + auto request = baseRequest; + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(response.GetError().GetCode() == S_ALREADY); + UNIT_ASSERT_VALUES_EQUAL(nbdDevFile, response.GetNbdDeviceFile()); + } + + { + auto request = baseRequest; + request.SetUseFreeNbdDeviceFile(true); + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(response.GetError().GetCode() == S_ALREADY); + UNIT_ASSERT_VALUES_EQUAL(nbdDevFile, response.GetNbdDeviceFile()); + } + + { + auto request = baseRequest; + request.SetUnixSocketPath(unixSocket + "other"); + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(response.GetError().GetCode() == E_INVALID_STATE); + } + + { + auto request = baseRequest; + request.SetNbdDeviceFile(nbdDevPrefix + "1"); + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(response.GetError().GetCode() == E_INVALID_STATE); + } + + { + auto request = baseRequest; + request.SetNbdDeviceFile(""); + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(response.GetError().GetCode() == E_INVALID_STATE); + } + + { + auto future = StopEndpoint(*manager, unixSocket); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(!HasError(response)); + + UNIT_ASSERT(mountedVolumes.empty()); + UNIT_ASSERT(listener->GetEndpoints().empty()); + } + + // + + { + auto request = baseRequest; + request.SetNbdDeviceFile(""); + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(!HasError(response), response.GetError()); + } + + { + auto request = baseRequest; + request.SetNbdDeviceFile(""); + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(response.GetError().GetCode() == S_ALREADY); + UNIT_ASSERT_VALUES_EQUAL("", response.GetNbdDeviceFile()); + } + + { + auto request = baseRequest; + request.SetUseFreeNbdDeviceFile(true); + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(response.GetError().GetCode() == E_INVALID_STATE); + } + + { + auto request = baseRequest; + request.SetNbdDeviceFile(nbdDevPrefix + "1"); + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(response.GetError().GetCode() == E_INVALID_STATE); + } + + { + auto future = StopEndpoint(*manager, unixSocket); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(!HasError(response)); + + UNIT_ASSERT(mountedVolumes.empty()); + UNIT_ASSERT(listener->GetEndpoints().empty()); + } + + // + + { + auto request = baseRequest; + request.SetNbdDeviceFile("blabla3"); + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(response.GetError().GetCode() == E_ARGUMENT); + } + + { + auto request = baseRequest; + request.SetPersistent(false); + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(response.GetError().GetCode() == E_ARGUMENT); + } + + // + + int num = 0; + deviceFactory->Devices.clear(); + + for (int i: {0, 1, 3}) { + auto request = baseRequest; + request.SetUnixSocketPath(unixSocket + ToString(num++)); + request.SetNbdDeviceFile(nbdDevPrefix + ToString(i)); + + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(!HasError(response), response.GetError()); + UNIT_ASSERT_VALUES_EQUAL( + nbdDevPrefix + ToString(i), + response.GetNbdDeviceFile()); + + UNIT_ASSERT_VALUES_EQUAL(num, deviceFactory->Devices.size()); + UNIT_ASSERT_VALUES_EQUAL( + nbdDevPrefix + ToString(i), + deviceFactory->Devices[num - 1]); + + auto [str, error] = storage.GetEndpoint(request.GetUnixSocketPath()); + UNIT_ASSERT(!HasError(error)); + auto req = DeserializeEndpoint(str); + UNIT_ASSERT(req); + UNIT_ASSERT(comparator.Equals(request, *req)); + } + + for (int i: {2, 4, 5}) { + auto request = baseRequest; + request.SetUnixSocketPath(unixSocket + ToString(num++)); + request.SetUseFreeNbdDeviceFile(true); + + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(!HasError(response), response.GetError()); + UNIT_ASSERT_VALUES_EQUAL( + nbdDevPrefix + ToString(i), + response.GetNbdDeviceFile()); + + UNIT_ASSERT_VALUES_EQUAL(num, deviceFactory->Devices.size()); + UNIT_ASSERT_VALUES_EQUAL( + nbdDevPrefix + ToString(i), + deviceFactory->Devices[num - 1]); + + auto [str, error] = storage.GetEndpoint(request.GetUnixSocketPath()); + UNIT_ASSERT(!HasError(error)); + auto req = DeserializeEndpoint(str); + UNIT_ASSERT(req); + UNIT_ASSERT(!comparator.Equals(request, *req)); + UNIT_ASSERT_VALUES_EQUAL( + nbdDevPrefix + ToString(i), + req->GetNbdDeviceFile()); + } + + { + auto request = baseRequest; + request.SetUnixSocketPath(unixSocket + ToString(num++)); + request.SetUseFreeNbdDeviceFile(true); + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(response.GetError().GetCode() == E_INVALID_STATE); + } + + for (int i = 0; i < deviceCount; ++i) { + auto future = StopEndpoint(*manager, unixSocket + ToString(i)); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(!HasError(response)); + } + + UNIT_ASSERT(mountedVolumes.empty()); + UNIT_ASSERT(listener->GetEndpoints().empty()); + } + + Y_UNIT_TEST(ShouldRestoreEndpointWithNbdDevice) + { + TString nbdDevPrefix = CreateGuidAsString() + "_nbd"; + int deviceCount = 10; + for (int i = 0; i < deviceCount; ++i) { + TFsPath(nbdDevPrefix + ToString(i)).Touch(); + } + Y_DEFER { + for (int i = 0; i < deviceCount; ++i) { + TFsPath(nbdDevPrefix + ToString(i)).DeleteIfExists(); + } + }; + + TBootstrap bootstrap; + TMap mountedVolumes; + bootstrap.Service = CreateTestService(mountedVolumes); + + auto listener = std::make_shared(); + bootstrap.EndpointListeners = {{ NProto::IPC_NBD, listener }}; + + auto deviceFactory = std::make_shared(); + bootstrap.NbdDeviceFactory = deviceFactory; + + bootstrap.Options.NbdDevicePrefix = nbdDevPrefix; + + auto manager = CreateEndpointManager(bootstrap); + bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; + + TTempDir dir; + TString unixSocket = (dir.Path() / "testSocket").GetPath(); + TString diskId = "testDiskId"; + + NProto::TStartEndpointRequest request; + SetDefaultHeaders(request); + request.SetDiskId(diskId); + request.SetClientId(TestClientId); + request.SetIpcType(NProto::IPC_NBD); + + size_t correctCount = 5; + size_t wrongCount = 3; + + for (size_t i = 0; i < wrongCount + correctCount; ++i) { + request.SetUnixSocketPath(unixSocket + ToString(i)); + + if (i < wrongCount) { + request.SetUseFreeNbdDeviceFile(true); + } else { + request.SetNbdDeviceFile(nbdDevPrefix + ToString(i)); + } + + auto [str, error] = SerializeEndpoint(request); + UNIT_ASSERT_C(!HasError(error), error); + + auto keyOrError = bootstrap.MutableStorage->AddEndpoint( + request.GetUnixSocketPath(), + str); + UNIT_ASSERT_C(!HasError(keyOrError), keyOrError.GetError()); + } + + NMonitoring::TDynamicCountersPtr counters = new NMonitoring::TDynamicCounters(); + InitCriticalEventsCounter(counters); + auto configCounter = counters->GetCounter("AppCriticalEvents/EndpointRestoringError", true); + UNIT_ASSERT_VALUES_EQUAL(0, static_cast(*configCounter)); + + manager->RestoreEndpoints().Wait(); + + UNIT_ASSERT(wrongCount != correctCount); + UNIT_ASSERT_VALUES_EQUAL(wrongCount, static_cast(*configCounter)); + } + + Y_UNIT_TEST(ShouldNotUseRestoringNbdDevices) + { + TString nbdDevPrefix = CreateGuidAsString() + "_nbd"; + int deviceCount = 6; + for (int i = 0; i < deviceCount; ++i) { + TFsPath(nbdDevPrefix + ToString(i)).Touch(); + } + Y_DEFER { + for (int i = 0; i < deviceCount; ++i) { + TFsPath(nbdDevPrefix + ToString(i)).DeleteIfExists(); + } + }; + + TTempDir dir; + TString unixSocket = (dir.Path() / "testSocket").GetPath(); + TString diskId = "testDiskId"; + TString nbdDevFile = nbdDevPrefix + "0"; + + TBootstrap bootstrap; + TMap mountedVolumes; + bootstrap.Service = CreateTestService(mountedVolumes); + + auto rejected = MakeFuture(MakeError(E_REJECTED)); + auto listener = std::make_shared(rejected); + listener->StartEndpointHandler = [&] ( + const NProto::TStartEndpointRequest& request, + NClient::ISessionPtr session) + { + UNIT_ASSERT(session); + + if (unixSocket == request.GetUnixSocketPath()) { + return MakeFuture(MakeError(E_REJECTED)); + } + return MakeFuture(); + }; + + bootstrap.EndpointListeners = {{ NProto::IPC_NBD, listener }}; + + auto deviceFactory = std::make_shared(); + bootstrap.NbdDeviceFactory = deviceFactory; + + bootstrap.Options.NbdDevicePrefix = nbdDevPrefix; + + auto manager = CreateEndpointManager(bootstrap); + bootstrap.Start(); + Y_DEFER { + bootstrap.Stop(); + }; + + NProto::TStartEndpointRequest request; + SetDefaultHeaders(request); + request.SetUnixSocketPath(unixSocket); + request.SetDiskId(diskId); + request.SetClientId(TestClientId); + request.SetIpcType(NProto::IPC_NBD); + request.SetNbdDeviceFile(nbdDevFile); + request.SetPersistent(true); + + { + auto [str, error] = SerializeEndpoint(request); + UNIT_ASSERT_C(!HasError(error), error); + + auto keyOrError = bootstrap.MutableStorage->AddEndpoint( + request.GetUnixSocketPath(), + str); + UNIT_ASSERT_C(!HasError(keyOrError), keyOrError.GetError()); + } + + NMonitoring::TDynamicCountersPtr counters = new NMonitoring::TDynamicCounters(); + InitCriticalEventsCounter(counters); + auto configCounter = counters->GetCounter("AppCriticalEvents/EndpointRestoringError", true); + UNIT_ASSERT_VALUES_EQUAL(0, static_cast(*configCounter)); + + manager->RestoreEndpoints(); + bootstrap.Scheduler->RunAllScheduledTasks(); + + UNIT_ASSERT_VALUES_EQUAL(0, static_cast(*configCounter)); + + { + request.SetUnixSocketPath(unixSocket + "other"); + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(response.GetError().GetCode() == E_INVALID_STATE); + } + + { + request.SetUnixSocketPath(unixSocket + "other"); + request.SetUseFreeNbdDeviceFile(true); + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(!HasError(response)); + UNIT_ASSERT(response.GetNbdDeviceFile() != nbdDevFile); + } + } + + Y_UNIT_TEST(ShouldRecreateSocketWhenRestartEndpoint) + { + TTempDir dir; + auto socketPath = dir.Path() / "testSocket"; + TString diskId = "testDiskId"; + auto ipcType = NProto::IPC_GRPC; + + TBootstrap bootstrap; + TMap mountedVolumes; + bootstrap.Service = CreateTestService(mountedVolumes); + + auto grpcListener = CreateSocketEndpointListener(bootstrap.Logging, 16); + grpcListener->SetClientStorageFactory(CreateClientStorageFactoryStub()); + bootstrap.EndpointListeners = {{ NProto::IPC_GRPC, grpcListener }}; + + auto manager = CreateEndpointManager(bootstrap); + bootstrap.Start(); + + NProto::TStartEndpointRequest request; + SetDefaultHeaders(request); + request.SetUnixSocketPath(socketPath.GetPath()); + request.SetDiskId(diskId); + request.SetClientId(TestClientId); + request.SetIpcType(ipcType); + + socketPath.DeleteIfExists(); + UNIT_ASSERT(!socketPath.Exists()); + + { + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(!HasError(response), response.GetError()); + } + + UNIT_ASSERT(socketPath.Exists()); + socketPath.DeleteIfExists(); + UNIT_ASSERT(!socketPath.Exists()); + + { + auto future = StartEndpoint(*manager, request); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(!HasError(response)); + } + + UNIT_ASSERT(socketPath.Exists()); + + { + auto future = StopEndpoint(*manager, socketPath.GetPath()); + auto response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT(!HasError(response)); + } + } } } // namespace NCloud::NBlockStore::NServer diff --git a/cloud/blockstore/libs/endpoints/public.h b/cloud/blockstore/libs/endpoints/public.h index c98d99208ee..4005f12a3be 100644 --- a/cloud/blockstore/libs/endpoints/public.h +++ b/cloud/blockstore/libs/endpoints/public.h @@ -22,9 +22,6 @@ using IEndpointManagerPtr = std::shared_ptr; struct IEndpointListener; using IEndpointListenerPtr = std::shared_ptr; -struct IEndpointService; -using IEndpointServicePtr = std::shared_ptr; - struct IEndpointEventHandler; using IEndpointEventHandlerPtr = std::shared_ptr; diff --git a/cloud/blockstore/libs/endpoints/service_endpoint.cpp b/cloud/blockstore/libs/endpoints/service_endpoint.cpp index f42a432850a..4f6e6a7bb3e 100644 --- a/cloud/blockstore/libs/endpoints/service_endpoint.cpp +++ b/cloud/blockstore/libs/endpoints/service_endpoint.cpp @@ -2,176 +2,49 @@ #include "endpoint_manager.h" -#include -#include -#include -#include #include -#include #include #include #include #include -#include -#include -#include -#include - -#include -#include -#include -#include namespace NCloud::NBlockStore::NServer { +using namespace NClient; using namespace NThreading; namespace { //////////////////////////////////////////////////////////////////////////////// -class TEndpointClient final - : public IBlockStore -{ -private: - IEndpointManagerPtr EndpointManager; - -public: - TEndpointClient(IEndpointManagerPtr endpointManager) - : EndpointManager(std::move(endpointManager)) - {} - - void Start() override - {} - - void Stop() override - {} - - TStorageBuffer AllocateBuffer(size_t bytesCount) override - { - Y_UNUSED(bytesCount); - return nullptr; - } - -#define BLOCKSTORE_IMPLEMENT_METHOD(name, ...) \ - TFuture name( \ - TCallContextPtr ctx, \ - std::shared_ptr request) override \ - { \ - Y_UNUSED(ctx); \ - Y_UNUSED(request); \ - return MakeFuture(TErrorResponse( \ - E_NOT_IMPLEMENTED, "Unsupported request")); \ - } \ -// BLOCKSTORE_IMPLEMENT_METHOD - - BLOCKSTORE_STORAGE_SERVICE(BLOCKSTORE_IMPLEMENT_METHOD) - BLOCKSTORE_IMPLEMENT_METHOD(KickEndpoint) - BLOCKSTORE_IMPLEMENT_METHOD(ListKeyrings) - -#undef BLOCKSTORE_IMPLEMENT_METHOD - -#define ENDPOINT_IMPLEMENT_METHOD(name, ...) \ - TFuture name( \ - TCallContextPtr ctx, \ - std::shared_ptr req) override \ - { \ - return EndpointManager->name(std::move(ctx), std::move(req)); \ - } \ -// ENDPOINT_IMPLEMENT_METHOD - - ENDPOINT_IMPLEMENT_METHOD(StartEndpoint) - ENDPOINT_IMPLEMENT_METHOD(StopEndpoint) - ENDPOINT_IMPLEMENT_METHOD(ListEndpoints) - ENDPOINT_IMPLEMENT_METHOD(DescribeEndpoint) - ENDPOINT_IMPLEMENT_METHOD(RefreshEndpoint) - -#undef ENDPOINT_IMPLEMENT_METHOD -}; - -//////////////////////////////////////////////////////////////////////////////// - class TEndpointService final - : public IEndpointService - , public std::enable_shared_from_this + : public IBlockStore { private: const IBlockStorePtr Service; const ITimerPtr Timer; const ISchedulerPtr Scheduler; - const TExecutorPtr Executor; - const IEndpointStoragePtr EndpointStorage; const IEndpointManagerPtr EndpointManager; - NClient::IMetricClientPtr RestoringClient; - TSet RestoringEndpoints; - - TLog Log; - - enum { - WaitingForRestoring = 0, - ReadingStorage = 1, - StartingEndpoints = 2, - Completed = 3, - }; - - TAtomic RestoringStage = WaitingForRestoring; - public: TEndpointService( IBlockStorePtr service, ITimerPtr timer, ISchedulerPtr scheduler, - ILoggingServicePtr logging, - IRequestStatsPtr requestStats, - IVolumeStatsPtr volumeStats, - IServerStatsPtr serverStats, - TExecutorPtr executor, - IEndpointStoragePtr endpointStorage, - IEndpointManagerPtr endpointManager, - NProto::TClientConfig clientConfig) + IEndpointManagerPtr endpointManager) : Service(std::move(service)) , Timer(std::move(timer)) , Scheduler(std::move(scheduler)) - , Executor(std::move(executor)) - , EndpointStorage(std::move(endpointStorage)) , EndpointManager(std::move(endpointManager)) - { - Log = logging->CreateLog("BLOCKSTORE_SERVER"); - - NProto::TClientAppConfig clientAppConfig; - *clientAppConfig.MutableClientConfig() = std::move(clientConfig); - auto appConfig = std::make_shared( - std::move(clientAppConfig)); - - IBlockStorePtr client = std::make_shared( - EndpointManager); - - client = CreateDurableClient( - appConfig, - std::move(client), - CreateRetryPolicy(appConfig), - logging, - Timer, - Scheduler, - std::move(requestStats), - std::move(volumeStats)); - - RestoringClient = NClient::CreateMetricClient( - std::move(client), - std::move(logging), - std::move(serverStats)); - } + {} void Start() override { Service->Start(); - RestoringClient->Start(); } void Stop() override { - RestoringClient->Stop(); Service->Stop(); } @@ -180,12 +53,6 @@ class TEndpointService final return Service->AllocateBuffer(bytesCount); } - size_t CollectRequests( - const TIncompleteRequestsCollector& collector) override - { - return RestoringClient->CollectRequests(collector); - } - #define STORAGE_IMPLEMENT_METHOD(name, ...) \ TFuture name( \ TCallContextPtr ctx, \ @@ -201,17 +68,11 @@ class TEndpointService final #define ENDPOINT_IMPLEMENT_METHOD(name, ...) \ TFuture name( \ - TCallContextPtr callContext, \ - std::shared_ptr request) override \ + TCallContextPtr ctx, \ + std::shared_ptr req) override \ { \ - auto timeout = request->GetHeaders().GetRequestTimeout(); \ - auto future = Executor->Execute([ \ - ctx = std::move(callContext), \ - req = std::move(request), \ - this] () mutable \ - { \ - return Do##name(std::move(ctx), std::move(req)); \ - }); \ + auto timeout = req->GetHeaders().GetRequestTimeout(); \ + auto future = EndpointManager->name(std::move(ctx), std::move(req)); \ return CreateTimeoutFuture(future, TDuration::MilliSeconds(timeout)); \ } \ // ENDPOINT_IMPLEMENT_METHOD @@ -220,220 +81,7 @@ class TEndpointService final #undef ENDPOINT_IMPLEMENT_METHOD - TFuture RestoreEndpoints() override - { - AtomicSet(RestoringStage, ReadingStorage); - return Executor->Execute([this] () mutable { - auto future = DoRestoreEndpoints(); - AtomicSet(RestoringStage, StartingEndpoints); - Executor->WaitFor(future); - AtomicSet(RestoringStage, Completed); - }); - } - private: - bool IsEndpointRestoring(const TString& socket) - { - switch (AtomicGet(RestoringStage)) { - case WaitingForRestoring: return false; - case ReadingStorage: return true; - case StartingEndpoints: return RestoringEndpoints.contains(socket); - case Completed: return false; - } - return false; - } - - NProto::TStartEndpointResponse DoStartEndpoint( - TCallContextPtr ctx, - std::shared_ptr request) - { - if (IsEndpointRestoring(request->GetUnixSocketPath())) { - return TErrorResponse(E_REJECTED, "endpoint is restoring now"); - } - - auto req = *request; - auto future = EndpointManager->StartEndpoint( - std::move(ctx), - std::move(request)); - - auto response = Executor->WaitFor(future); - if (HasError(response)) { - return response; - } - - if (req.GetPersistent()) { - AddEndpointToStorage(req); - } - - return response; - } - - NProto::TStopEndpointResponse DoStopEndpoint( - TCallContextPtr ctx, - std::shared_ptr request) - { - if (IsEndpointRestoring(request->GetUnixSocketPath())) { - return TErrorResponse(E_REJECTED, "endpoint is restoring now"); - } - - auto socketPath = request->GetUnixSocketPath(); - auto future = EndpointManager->StopEndpoint( - std::move(ctx), - std::move(request)); - - auto response = Executor->WaitFor(future); - if (HasError(response)) { - return response; - } - - EndpointStorage->RemoveEndpoint(socketPath); - return response; - } - - NProto::TListEndpointsResponse DoListEndpoints( - TCallContextPtr ctx, - std::shared_ptr request) - { - auto future = EndpointManager->ListEndpoints( - std::move(ctx), - std::move(request)); - - auto response = Executor->WaitFor(future); - response.SetEndpointsWereRestored( - AtomicGet(RestoringStage) == Completed); - return response; - } - - NProto::TKickEndpointResponse DoKickEndpoint( - TCallContextPtr ctx, - std::shared_ptr request) - { - auto [str, error] = EndpointStorage->GetEndpoint( - ToString(request->GetKeyringId())); - - if (HasError(error)) { - return TErrorResponse(error); - } - - auto startReq = DeserializeEndpoint(str); - - if (!startReq) { - return TErrorResponse(E_INVALID_STATE, TStringBuilder() - << "Failed to deserialize endpoint with key " - << request->GetKeyringId()); - } - - startReq->MutableHeaders()->MergeFrom(request->GetHeaders()); - - STORAGE_INFO("Kick StartEndpoint request: " << *startReq); - auto response = DoStartEndpoint( - std::move(ctx), - std::move(startReq)); - - return TErrorResponse(response.GetError()); - } - - NProto::TListKeyringsResponse DoListKeyrings( - TCallContextPtr ctx, - std::shared_ptr request) - { - Y_UNUSED(ctx); - Y_UNUSED(request); - - auto [storedIds, error] = EndpointStorage->GetEndpointIds(); - if (HasError(error)) { - return TErrorResponse(error); - } - - NProto::TListKeyringsResponse response; - auto& endpoints = *response.MutableEndpoints(); - - endpoints.Reserve(storedIds.size()); - - for (auto keyringId: storedIds) { - auto& endpoint = *endpoints.Add(); - endpoint.SetKeyringId(keyringId); - - auto [str, error] = EndpointStorage->GetEndpoint(keyringId); - if (HasError(error)) { - STORAGE_WARN("Failed to get endpoint from storage, ID: " - << keyringId << ", error: " << FormatError(error)); - continue; - } - - auto req = DeserializeEndpoint(str); - if (!req) { - STORAGE_WARN("Failed to deserialize endpoint from storage, ID: " - << keyringId); - continue; - } - - endpoint.MutableRequest()->CopyFrom(*req); - } - - return response; - } - - NProto::TDescribeEndpointResponse DoDescribeEndpoint( - TCallContextPtr ctx, - std::shared_ptr request) - { - if (IsEndpointRestoring(request->GetUnixSocketPath())) { - return TErrorResponse(E_REJECTED, "endpoint is restoring now"); - } - - auto future = EndpointManager->DescribeEndpoint( - std::move(ctx), - std::move(request)); - return Executor->WaitFor(future); - } - - NProto::TRefreshEndpointResponse DoRefreshEndpoint( - TCallContextPtr ctx, - std::shared_ptr request) - { - if (IsEndpointRestoring(request->GetUnixSocketPath())) { - return TErrorResponse(E_REJECTED, "endpoint is restoring now"); - } - - auto future = EndpointManager->RefreshEndpoint( - std::move(ctx), - std::move(request)); - return Executor->WaitFor(future); - } - - TFuture DoRestoreEndpoints(); - - NProto::TError AddEndpointToStorage( - const NProto::TStartEndpointRequest& request) - { - auto [data, error] = SerializeEndpoint(request); - if (HasError(error)) { - return error; - } - - error = EndpointStorage->AddEndpoint(request.GetUnixSocketPath(), data); - if (HasError(error)) { - return error; - } - - return {}; - } - - void HandleRestoredEndpoint( - const TString& socketPath, - const NProto::TError& error) - { - if (HasError(error)) { - STORAGE_ERROR("Failed to start endpoint " << socketPath.Quote() - << ", error:" << FormatError(error)); - } - - Executor->Execute([socketPath, this] () mutable { - RestoringEndpoints.erase(socketPath); - }); - } - template TFuture CreateTimeoutFuture(const TFuture& future, TDuration timeout) { @@ -455,106 +103,21 @@ class TEndpointService final } }; -//////////////////////////////////////////////////////////////////////////////// - -TFuture TEndpointService::DoRestoreEndpoints() -{ - auto [storedIds, error] = EndpointStorage->GetEndpointIds(); - if (HasError(error)) { - STORAGE_ERROR("Failed to get endpoints from storage: " - << FormatError(error)); - ReportEndpointRestoringError(); - return MakeFuture(); - } - - STORAGE_INFO("Found " << storedIds.size() << " endpoints in storage"); - - TString clientId = CreateGuidAsString() + "_bootstrap"; - - TVector> futures; - - for (auto keyringId: storedIds) { - auto [str, error] = EndpointStorage->GetEndpoint(keyringId); - if (HasError(error)) { - // NBS-3678 - STORAGE_WARN("Failed to restore endpoint. ID: " << keyringId - << ", error: " << FormatError(error)); - continue; - } - - auto request = DeserializeEndpoint(str); - - if (!request) { - ReportEndpointRestoringError(); - STORAGE_ERROR("Failed to deserialize request. ID: " << keyringId); - continue; - } - - auto& headers = *request->MutableHeaders(); - - if (!headers.GetClientId()) { - headers.SetClientId(clientId); - } - - auto requestId = headers.GetRequestId(); - if (!requestId) { - headers.SetRequestId(CreateRequestId()); - } - - auto socketPath = request->GetUnixSocketPath(); - RestoringEndpoints.insert(socketPath); - - auto future = RestoringClient->StartEndpoint( - MakeIntrusive(requestId), - std::move(request)); - - auto weakPtr = weak_from_this(); - future.Subscribe([weakPtr, socketPath] (const auto& f) { - const auto& response = f.GetValue(); - if (HasError(response)) { - ReportEndpointRestoringError(); - } - - if (auto ptr = weakPtr.lock()) { - ptr->HandleRestoredEndpoint(socketPath, response.GetError()); - } - }); - - futures.push_back(future.IgnoreResult()); - } - - return WaitAll(futures); -} - } // namespace //////////////////////////////////////////////////////////////////////////////// -IEndpointServicePtr CreateMultipleEndpointService( +IBlockStorePtr CreateMultipleEndpointService( IBlockStorePtr service, ITimerPtr timer, ISchedulerPtr scheduler, - ILoggingServicePtr logging, - IRequestStatsPtr requestStats, - IVolumeStatsPtr volumeStats, - IServerStatsPtr serverStats, - TExecutorPtr executor, - IEndpointStoragePtr endpointStorage, - IEndpointManagerPtr endpointManager, - NProto::TClientConfig clientConfig) + IEndpointManagerPtr endpointManager) { return std::make_shared( std::move(service), std::move(timer), std::move(scheduler), - std::move(logging), - std::move(requestStats), - std::move(volumeStats), - std::move(serverStats), - std::move(executor), - std::move(endpointStorage), - std::move(endpointManager), - std::move(clientConfig)); + std::move(endpointManager)); } } // namespace NCloud::NBlockStore::NServer diff --git a/cloud/blockstore/libs/endpoints/service_endpoint.h b/cloud/blockstore/libs/endpoints/service_endpoint.h index 26b0057498a..c3296610d33 100644 --- a/cloud/blockstore/libs/endpoints/service_endpoint.h +++ b/cloud/blockstore/libs/endpoints/service_endpoint.h @@ -5,37 +5,16 @@ #include #include -#include -#include -#include - -#include -#include +#include namespace NCloud::NBlockStore::NServer { //////////////////////////////////////////////////////////////////////////////// -struct IEndpointService - : public IBlockStore - , public IIncompleteRequestProvider -{ - virtual NThreading::TFuture RestoreEndpoints() = 0; -}; - -//////////////////////////////////////////////////////////////////////////////// - -IEndpointServicePtr CreateMultipleEndpointService( +IBlockStorePtr CreateMultipleEndpointService( IBlockStorePtr service, ITimerPtr timer, ISchedulerPtr scheduler, - ILoggingServicePtr logging, - IRequestStatsPtr requestStats, - IVolumeStatsPtr volumeStats, - IServerStatsPtr serverStats, - TExecutorPtr executor, - IEndpointStoragePtr endpointStorage, - IEndpointManagerPtr endpointManager, - NProto::TClientConfig clientConfig); + IEndpointManagerPtr endpointManager); } // namespace NCloud::NBlockStore::NServer diff --git a/cloud/blockstore/libs/endpoints/service_endpoint_ut.cpp b/cloud/blockstore/libs/endpoints/service_endpoint_ut.cpp index f2f808b1767..7f26e3b8b89 100644 --- a/cloud/blockstore/libs/endpoints/service_endpoint_ut.cpp +++ b/cloud/blockstore/libs/endpoints/service_endpoint_ut.cpp @@ -26,6 +26,8 @@ #include +#include +#include #include #include @@ -37,22 +39,38 @@ namespace { //////////////////////////////////////////////////////////////////////////////// -class TTestEndpointListener final +struct TTestEndpointListener final : public IEndpointListener { -public: - NThreading::TFuture StartEndpoint( + using TStartEndpointHandler = std::function( + const NProto::TStartEndpointRequest& request)>; + using TStopEndpointHandler = std::function( + const TString& socketPath)>; + + TStartEndpointHandler StartEndpointHandler = [] ( + const NProto::TStartEndpointRequest& request) + { + TFsPath(request.GetUnixSocketPath()).Touch(); + return MakeFuture(NProto::TError{}); + }; + + TStopEndpointHandler StopEndpointHandler = [] (const TString& socketPath) + { + TFsPath(socketPath).DeleteIfExists(); + return MakeFuture(NProto::TError{}); + }; + + TFuture StartEndpoint( const NProto::TStartEndpointRequest& request, const NProto::TVolume& volume, NClient::ISessionPtr session) override { - Y_UNUSED(request); Y_UNUSED(volume); Y_UNUSED(session); - return MakeFuture(NProto::TError()); + return StartEndpointHandler(request); } - NThreading::TFuture AlterEndpoint( + TFuture AlterEndpoint( const NProto::TStartEndpointRequest& request, const NProto::TVolume& volume, NClient::ISessionPtr session) override @@ -63,11 +81,10 @@ class TTestEndpointListener final return MakeFuture(NProto::TError()); } - NThreading::TFuture StopEndpoint( + TFuture StopEndpoint( const TString& socketPath) override { - Y_UNUSED(socketPath); - return MakeFuture(NProto::TError()); + return StopEndpointHandler(socketPath); } NProto::TError RefreshEndpoint( @@ -79,7 +96,7 @@ class TTestEndpointListener final return NProto::TError(); } - NThreading::TFuture SwitchEndpoint( + TFuture SwitchEndpoint( const NProto::TStartEndpointRequest& request, const NProto::TVolume& volume, NClient::ISessionPtr session) override @@ -97,27 +114,41 @@ struct TTestSessionManager final : public ISessionManager { using TCreateSessionHandler - = std::function()>; + = std::function()>; using TRemoveSessionHandler - = std::function()>; + = std::function()>; using TAlterSessionHandler - = std::function()>; + = std::function()>; using TGetSessionHandler - = std::function()>; + = std::function()>; using TGetProfileHandler = std::function()>; - TCreateSessionHandler CreateSessionHandler; - TRemoveSessionHandler RemoveSessionHandler; - TAlterSessionHandler AlterSessionHandler; - TGetSessionHandler GetSessionHandler; - TGetProfileHandler GetProfileHandler; + TCreateSessionHandler CreateSessionHandler = [] () { + return MakeFuture(TSessionInfo()); + }; + + TRemoveSessionHandler RemoveSessionHandler = [] () { + return MakeFuture(NProto::TError()); + }; + + TAlterSessionHandler AlterSessionHandler = [] () { + return MakeFuture(NProto::TError()); + }; + + TGetSessionHandler GetSessionHandler = [] () { + return MakeFuture(TSessionInfo()); + }; - NThreading::TFuture CreateSession( + TGetProfileHandler GetProfileHandler = [] () { + return NProto::TClientPerformanceProfile(); + }; + + TFuture CreateSession( TCallContextPtr callContext, const NProto::TStartEndpointRequest& request) override { @@ -126,7 +157,7 @@ struct TTestSessionManager final return CreateSessionHandler(); } - NThreading::TFuture RemoveSession( + TFuture RemoveSession( TCallContextPtr callContext, const TString& socketPath, const NProto::THeaders& headers) override @@ -137,7 +168,7 @@ struct TTestSessionManager final return RemoveSessionHandler(); } - NThreading::TFuture AlterSession( + TFuture AlterSession( TCallContextPtr callContext, const TString& socketPath, NProto::EVolumeAccessMode accessMode, @@ -154,7 +185,7 @@ struct TTestSessionManager final return AlterSessionHandler(); } - NThreading::TFuture GetSession( + TFuture GetSession( TCallContextPtr callContext, const TString& socketPath, const NProto::THeaders& headers) override @@ -175,38 +206,8 @@ struct TTestSessionManager final //////////////////////////////////////////////////////////////////////////////// -struct TTestEndpointManager final - : public IEndpointManager -{ -#define ENDPOINT_IMPLEMENT_METHOD(name, ...) \ - using T##name##Handler = std::function< \ - NThreading::TFuture( \ - std::shared_ptr request) \ - >; \ - \ - T##name##Handler name##Handler; \ - TFuture name( \ - TCallContextPtr ctx, \ - std::shared_ptr request) override \ - { \ - Y_UNUSED(ctx); \ - return name##Handler(std::move(request)); \ - } \ -// ENDPOINT_IMPLEMENT_METHOD - - ENDPOINT_IMPLEMENT_METHOD(StartEndpoint) - ENDPOINT_IMPLEMENT_METHOD(StopEndpoint) - ENDPOINT_IMPLEMENT_METHOD(ListEndpoints) - ENDPOINT_IMPLEMENT_METHOD(DescribeEndpoint) - ENDPOINT_IMPLEMENT_METHOD(RefreshEndpoint) - -#undef ENDPOINT_IMPLEMENT_METHOD -}; - -//////////////////////////////////////////////////////////////////////////////// - NProto::TKickEndpointResponse KickEndpoint( - IBlockStore& service, + IEndpointManager& service, ui32 keyringId, ui32 requestId = 42) { @@ -221,7 +222,6 @@ NProto::TKickEndpointResponse KickEndpoint( return future.GetValue(TDuration::Seconds(5)); } - } // namespace //////////////////////////////////////////////////////////////////////////////// @@ -243,24 +243,25 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) UNIT_ASSERT_C(!HasError(error), error); }; - TString unixSocket = "testSocket"; + TTempDir dir; + TString unixSocket = (dir.Path() / "testSocket").GetPath(); TString diskId = "testDiskId"; auto ipcType = NProto::IPC_GRPC; - TVector> endpoints; - auto endpointManager = std::make_shared(); - endpointManager->StartEndpointHandler = [&] ( - std::shared_ptr request) + TMap endpoints; + auto listener = std::make_shared(); + listener->StartEndpointHandler = [&] ( + const NProto::TStartEndpointRequest& request) { - endpoints.push_back(std::move(request)); - return MakeFuture(NProto::TStartEndpointResponse()); + endpoints.emplace(request.GetUnixSocketPath(), request); + TFsPath(request.GetUnixSocketPath()).Touch(); + return MakeFuture(NProto::TError{}); }; auto executor = TExecutor::Create("TestService"); executor->Start(); - auto endpointService = CreateMultipleEndpointService( - std::make_shared(), + auto endpointManager = CreateEndpointManager( CreateWallClockTimer(), CreateSchedulerStub(), CreateLoggingService("console"), @@ -268,9 +269,13 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) CreateVolumeStatsStub(), CreateServerStatsStub(), executor, + CreateEndpointEventProxy(), + std::make_shared(), endpointStorage, - endpointManager, - {}); + {{ ipcType, listener }}, + nullptr, // nbdDeviceFactory + {} // options + ); NProto::TStartEndpointRequest request; request.SetUnixSocketPath(unixSocket); @@ -289,13 +294,13 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) { ui32 requestId = 325; auto response = KickEndpoint( - *endpointService, + *endpointManager, keyringId, requestId); UNIT_ASSERT(!HasError(response)); UNIT_ASSERT_VALUES_EQUAL(1, endpoints.size()); - const auto& endpoint = *endpoints[0]; + const auto& endpoint = endpoints.begin()->second; google::protobuf::util::MessageDifferencer comparator; request.MutableHeaders()->SetRequestId(requestId); @@ -304,7 +309,7 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) { auto wrongKeyringId = keyringId + 42; - auto response = KickEndpoint(*endpointService, wrongKeyringId); + auto response = KickEndpoint(*endpointManager, wrongKeyringId); UNIT_ASSERT(HasError(response) && response.GetError().GetCode() == E_INVALID_STATE); } @@ -349,29 +354,30 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) executor->Start(); auto endpointManager = CreateEndpointManager( - logging, + CreateWallClockTimer(), + CreateSchedulerStub(), + CreateLoggingService("console"), + CreateRequestStatsStub(), + CreateVolumeStatsStub(), CreateServerStatsStub(), executor, CreateEndpointEventProxy(), sessionManager, + endpointStorage, {{ NProto::IPC_GRPC, std::make_shared() }}, - ""); + nullptr, // nbdDeviceFactory + {} // options + ); auto endpointService = CreateMultipleEndpointService( std::make_shared(), CreateWallClockTimer(), scheduler, - logging, - CreateRequestStatsStub(), - CreateVolumeStatsStub(), - CreateServerStatsStub(), - executor, - endpointStorage, - endpointManager, - {}); + endpointManager); + TTempDir dir; + const TString socketPath = (dir.Path() / "testSocket").GetPath(); const TString diskId = "testDiskId"; - const TString socketPath = "testSocket"; { auto request = std::make_shared(); @@ -478,20 +484,20 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) Y_UNIT_TEST(ShouldThrowCriticalEventIfFailedToRestoreEndpoint) { - const TString wrongSocketPath = "wrong.socket"; + const TString wrongSocketPath = "wrongSocket"; size_t startedEndpointCount = 0; - auto endpointManager = std::make_shared(); - endpointManager->StartEndpointHandler = [&] ( - std::shared_ptr request) + auto listener = std::make_shared(); + listener->StartEndpointHandler = [&] ( + const NProto::TStartEndpointRequest& request) { - if (request->GetUnixSocketPath() == wrongSocketPath) { - return MakeFuture( - TErrorResponse(E_FAIL)); + if (request.GetUnixSocketPath().StartsWith(wrongSocketPath)) { + return MakeFuture(MakeError(E_FAIL)); } ++startedEndpointCount; - return MakeFuture(NProto::TStartEndpointResponse()); + TFsPath(request.GetUnixSocketPath()).Touch(); + return MakeFuture(NProto::TError{}); }; NMonitoring::TDynamicCountersPtr counters = new NMonitoring::TDynamicCounters(); @@ -527,7 +533,7 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) for (size_t i = 0; i < wrongSocketCount; ++i) { NProto::TStartEndpointRequest request; - request.SetUnixSocketPath(wrongSocketPath); + request.SetUnixSocketPath(wrongSocketPath + ToString(i)); auto strOrError = SerializeEndpoint(request); UNIT_ASSERT_C(!HasError(strOrError), strOrError.GetError()); @@ -538,9 +544,12 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) UNIT_ASSERT_C(!HasError(error), error); } + TTempDir dir; + for (size_t i = 0; i < correctCount; ++i) { NProto::TStartEndpointRequest request; - request.SetUnixSocketPath("endpoint.sock"); + const TString socket = "testSocket" + ToString(i + 1); + request.SetUnixSocketPath((dir.Path() / socket).GetPath()); auto strOrError = SerializeEndpoint(request); UNIT_ASSERT_C(!HasError(strOrError), strOrError.GetError()); @@ -554,8 +563,7 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) auto executor = TExecutor::Create("TestService"); executor->Start(); - auto endpointService = CreateMultipleEndpointService( - std::make_shared(), + auto endpointManager = CreateEndpointManager( CreateWallClockTimer(), CreateSchedulerStub(), CreateLoggingService("console"), @@ -563,11 +571,15 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) CreateVolumeStatsStub(), CreateServerStatsStub(), executor, + CreateEndpointEventProxy(), + std::make_shared(), endpointStorage, - endpointManager, - {}); + {{ NProto::IPC_GRPC, listener }}, + nullptr, // nbdDeviceFactory + {} // options + ); - endpointService->RestoreEndpoints().Wait(); + endpointManager->RestoreEndpoints().Wait(); UNIT_ASSERT_VALUES_EQUAL( wrongDataCount + wrongSocketCount, @@ -592,8 +604,7 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) auto executor = TExecutor::Create("TestService"); executor->Start(); - auto endpointService = CreateMultipleEndpointService( - std::make_shared(), + auto endpointManager = CreateEndpointManager( CreateWallClockTimer(), CreateSchedulerStub(), CreateLoggingService("console"), @@ -601,11 +612,15 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) CreateVolumeStatsStub(), CreateServerStatsStub(), executor, + CreateEndpointEventProxy(), + std::make_shared(), endpointStorage, - std::make_shared(), - {}); + {}, // listeners + nullptr, // nbdDeviceFactory + {} // options + ); - endpointService->RestoreEndpoints().Wait(); + endpointManager->RestoreEndpoints().Wait(); UNIT_ASSERT_VALUES_EQUAL(1, static_cast(*configCounter)); @@ -626,8 +641,7 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) auto executor = TExecutor::Create("TestService"); executor->Start(); - auto endpointService = CreateMultipleEndpointService( - std::make_shared(), + auto endpointManager = CreateEndpointManager( CreateWallClockTimer(), CreateSchedulerStub(), CreateLoggingService("console"), @@ -635,11 +649,15 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) CreateVolumeStatsStub(), CreateServerStatsStub(), executor, + CreateEndpointEventProxy(), + std::make_shared(), endpointStorage, - std::make_shared(), - {}); + {}, // listeners + nullptr, // nbdDeviceFactory + {} // options + ); - endpointService->RestoreEndpoints().Wait(); + endpointManager->RestoreEndpoints().Wait(); UNIT_ASSERT_VALUES_EQUAL(0, static_cast(*configCounter)); @@ -648,23 +666,16 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) Y_UNIT_TEST(ShouldHandleRestoreFlagInListEndpointsResponse) { - auto trigger = NewPromise(); + auto trigger = NewPromise(); - auto endpointManager = std::make_shared(); - endpointManager->StartEndpointHandler = [&] ( - std::shared_ptr request) + auto listener = std::make_shared(); + listener->StartEndpointHandler = [&] ( + const NProto::TStartEndpointRequest& request) { - Y_UNUSED(request); + TFsPath(request.GetUnixSocketPath()).Touch(); return trigger; }; - endpointManager->ListEndpointsHandler = [&] ( - std::shared_ptr request) - { - Y_UNUSED(request); - return MakeFuture(NProto::TListEndpointsResponse()); - }; - const TString dirPath = "./" + CreateGuidAsString(); auto endpointStorage = CreateFileEndpointStorage(dirPath); auto mutableStorage = CreateFileMutableEndpointStorage(dirPath); @@ -677,11 +688,13 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) UNIT_ASSERT_C(!HasError(error), error); }; + TTempDir dir; size_t endpointCount = 5; for (size_t i = 0; i < endpointCount; ++i) { NProto::TStartEndpointRequest request; - request.SetUnixSocketPath("endpoint.sock"); + const TString socket = "testSocket" + ToString(i + 1); + request.SetUnixSocketPath((dir.Path() / socket).GetPath()); auto strOrError = SerializeEndpoint(request); UNIT_ASSERT_C(!HasError(strOrError), strOrError.GetError()); @@ -695,8 +708,7 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) auto executor = TExecutor::Create("TestService"); executor->Start(); - auto endpointService = CreateMultipleEndpointService( - std::make_shared(), + auto endpointManager = CreateEndpointManager( CreateWallClockTimer(), CreateSchedulerStub(), CreateLoggingService("console"), @@ -704,26 +716,30 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) CreateVolumeStatsStub(), CreateServerStatsStub(), executor, + CreateEndpointEventProxy(), + std::make_shared(), endpointStorage, - endpointManager, - {}); + {{ NProto::IPC_GRPC, listener }}, + nullptr, // nbdDeviceFactory + {} // options + ); - auto restoreFuture = endpointService->RestoreEndpoints(); + auto restoreFuture = endpointManager->RestoreEndpoints(); UNIT_ASSERT(!restoreFuture.HasValue()); { - auto future = endpointService->ListEndpoints( + auto future = endpointManager->ListEndpoints( MakeIntrusive(), std::make_shared()); auto response = future.GetValue(TDuration::Seconds(1)); UNIT_ASSERT(!response.GetEndpointsWereRestored()); } - trigger.SetValue(NProto::TStartEndpointResponse()); + trigger.SetValue(NProto::TError()); restoreFuture.Wait(); { - auto future = endpointService->ListEndpoints( + auto future = endpointManager->ListEndpoints( MakeIntrusive(), std::make_shared()); auto response = future.GetValue(TDuration::Seconds(1)); @@ -735,20 +751,6 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) Y_UNIT_TEST(ShouldListKeyrings) { - auto endpointManager = std::make_shared(); - endpointManager->StartEndpointHandler = [&] ( - std::shared_ptr request) - { - Y_UNUSED(request); - return MakeFuture(NProto::TStartEndpointResponse()); - }; - endpointManager->StopEndpointHandler = [&] ( - std::shared_ptr request) - { - Y_UNUSED(request); - return MakeFuture(NProto::TStopEndpointResponse()); - }; - const TString dirPath = "./" + CreateGuidAsString(); auto endpointStorage = CreateFileEndpointStorage(dirPath); auto mutableStorage = CreateFileMutableEndpointStorage(dirPath); @@ -761,12 +763,14 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) UNIT_ASSERT_C(!HasError(error), error); }; + TTempDir dir; ui32 endpointCount = 42; THashMap requests; for (size_t i = 0; i < endpointCount; ++i) { NProto::TStartEndpointRequest request; - request.SetUnixSocketPath("testSocket" + ToString(i + 1)); + const TString socket = "testSocket" + ToString(i + 1); + request.SetUnixSocketPath((dir.Path() / socket).GetPath()); request.SetDiskId("testDiskId" + ToString(i + 1)); request.SetIpcType(NProto::IPC_GRPC); @@ -784,8 +788,7 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) auto executor = TExecutor::Create("TestService"); executor->Start(); - auto endpointService = CreateMultipleEndpointService( - std::make_shared(), + auto endpointManager = CreateEndpointManager( CreateWallClockTimer(), CreateSchedulerStub(), CreateLoggingService("console"), @@ -793,11 +796,15 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) CreateVolumeStatsStub(), CreateServerStatsStub(), executor, + CreateEndpointEventProxy(), + std::make_shared(), endpointStorage, - endpointManager, - {}); + {{ NProto::IPC_GRPC, std::make_shared() }}, + nullptr, // nbdDeviceFactory + {} // options + ); - auto future = endpointService->ListKeyrings( + auto future = endpointManager->ListKeyrings( MakeIntrusive(), std::make_shared()); @@ -817,13 +824,14 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) for (size_t i = 0; i < 5; ++i) { auto request = std::make_shared(); - request->SetUnixSocketPath("testPersistentSocket" + ToString(i + 1)); + const TString socket = "testPersistentSocket" + ToString(i + 1); + request->SetUnixSocketPath((dir.Path() / socket).GetPath()); request->SetDiskId("testPersistentDiskId" + ToString(i + 1)); request->SetIpcType(NProto::IPC_GRPC); request->SetPersistent(true); ++endpointCount; - auto future = endpointService->StartEndpoint( + auto future = endpointManager->StartEndpoint( MakeIntrusive(), request); auto response = future.GetValue(TDuration::Seconds(5)); @@ -831,7 +839,7 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) } { - auto future = endpointService->ListKeyrings( + auto future = endpointManager->ListKeyrings( MakeIntrusive(), std::make_shared()); @@ -842,12 +850,13 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) for (size_t i = 0; i < 5; ++i) { auto request = std::make_shared(); - request->SetUnixSocketPath("testTemporarySocket" + ToString(i + 1)); + const TString socket = "testTemporarySocket" + ToString(i + 1); + request->SetUnixSocketPath((dir.Path() / socket).GetPath()); request->SetDiskId("testTemporaryDiskId" + ToString(i + 1)); request->SetIpcType(NProto::IPC_GRPC); request->SetPersistent(false); - auto future = endpointService->StartEndpoint( + auto future = endpointManager->StartEndpoint( MakeIntrusive(), request); auto response = future.GetValue(TDuration::Seconds(5)); @@ -855,7 +864,7 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) } { - auto future = endpointService->ListKeyrings( + auto future = endpointManager->ListKeyrings( MakeIntrusive(), std::make_shared()); @@ -866,10 +875,11 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) for (size_t i = 0; i < 3; ++i) { auto request = std::make_shared(); - request->SetUnixSocketPath("testPersistentSocket" + ToString(i + 1)); + const TString socket = "testPersistentSocket" + ToString(i + 1); + request->SetUnixSocketPath((dir.Path() / socket).GetPath()); --endpointCount; - auto future = endpointService->StopEndpoint( + auto future = endpointManager->StopEndpoint( MakeIntrusive(), request); auto response = future.GetValue(TDuration::Seconds(5)); @@ -877,7 +887,7 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) } { - auto future = endpointService->ListKeyrings( + auto future = endpointManager->ListKeyrings( MakeIntrusive(), std::make_shared()); @@ -888,9 +898,10 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) for (size_t i = 0; i < 3; ++i) { auto request = std::make_shared(); - request->SetUnixSocketPath("testTemporarySocket" + ToString(i + 1)); + const TString socket = "testTemporarySocket" + ToString(i + 1); + request->SetUnixSocketPath((dir.Path() / socket).GetPath()); - auto future = endpointService->StopEndpoint( + auto future = endpointManager->StopEndpoint( MakeIntrusive(), request); auto response = future.GetValue(TDuration::Seconds(5)); @@ -898,7 +909,7 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) } { - auto future = endpointService->ListKeyrings( + auto future = endpointManager->ListKeyrings( MakeIntrusive(), std::make_shared()); @@ -912,14 +923,19 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) Y_UNIT_TEST(ShouldHandleParallelStartStopEndpoints) { - auto startPromise = NewPromise(); + auto startPromise = NewPromise(); auto stopPromise = NewPromise(); - auto sessionManager = std::make_shared(); - sessionManager->CreateSessionHandler = [&] () { + auto listener = std::make_shared(); + listener->StartEndpointHandler = [&] ( + const NProto::TStartEndpointRequest& request) + { + Y_UNUSED(request); return startPromise.GetFuture(); }; - sessionManager->RemoveSessionHandler = [&] () { + listener->StopEndpointHandler = [&] (const TString& socketPath) + { + Y_UNUSED(socketPath); return stopPromise.GetFuture(); }; @@ -932,28 +948,23 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) executor->Start(); auto endpointManager = CreateEndpointManager( - logging, - CreateServerStatsStub(), - executor, - CreateEndpointEventProxy(), - sessionManager, - {{ NProto::IPC_GRPC, std::make_shared() }}, - ""); - - auto endpointService = CreateMultipleEndpointService( - std::make_shared(), CreateWallClockTimer(), CreateSchedulerStub(), - logging, + CreateLoggingService("console"), CreateRequestStatsStub(), CreateVolumeStatsStub(), CreateServerStatsStub(), executor, + CreateEndpointEventProxy(), + std::make_shared(), endpointStorage, - endpointManager, - {}); + {{ NProto::IPC_GRPC, listener }}, + nullptr, // nbdDeviceFactory + {} // options + ); - auto unixSocket = "testSocket"; + TTempDir dir; + TString unixSocket = (dir.Path() / "testSocket").GetPath(); auto startRequest = std::make_shared(); startRequest->SetUnixSocketPath(unixSocket); @@ -971,16 +982,16 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) auto ctx = MakeIntrusive(); { - auto future1 = endpointService->StartEndpoint(ctx, startRequest); - auto future2 = endpointService->StartEndpoint(ctx, startRequest); + auto future1 = endpointManager->StartEndpoint(ctx, startRequest); + auto future2 = endpointManager->StartEndpoint(ctx, startRequest); UNIT_ASSERT(!future1.HasValue()); UNIT_ASSERT(!future2.HasValue()); - auto future3 = endpointService->StartEndpoint(ctx, otherStartRequest); + auto future3 = endpointManager->StartEndpoint(ctx, otherStartRequest); auto response3 = future3.GetValue(TDuration::Seconds(5)); UNIT_ASSERT_C(response3.GetError().GetCode() == E_REJECTED, response3.GetError()); - auto future = endpointService->StopEndpoint(ctx, stopRequest); + auto future = endpointManager->StopEndpoint(ctx, stopRequest); auto response = future.GetValue(TDuration::Seconds(5)); UNIT_ASSERT(response.GetError().GetCode() == E_REJECTED); @@ -993,12 +1004,12 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) } { - auto future1 = endpointService->StopEndpoint(ctx, stopRequest); - auto future2 = endpointService->StopEndpoint(ctx, stopRequest); + auto future1 = endpointManager->StopEndpoint(ctx, stopRequest); + auto future2 = endpointManager->StopEndpoint(ctx, stopRequest); UNIT_ASSERT(!future1.HasValue()); UNIT_ASSERT(!future2.HasValue()); - auto future3 = endpointService->StartEndpoint(ctx, otherStartRequest); + auto future3 = endpointManager->StartEndpoint(ctx, otherStartRequest); auto response3 = future3.GetValue(TDuration::Seconds(5)); UNIT_ASSERT(response3.GetError().GetCode() == E_REJECTED); @@ -1031,17 +1042,18 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) TAtomic trigger = 0; TManualEvent event; - auto sessionManager = std::make_shared(); - sessionManager->CreateSessionHandler = [&] () { + auto listener = std::make_shared(); + listener->StartEndpointHandler = [&] ( + const NProto::TStartEndpointRequest& request) + { + Y_UNUSED(request); + NProto::TError error; if (!AtomicGet(trigger)) { error.SetCode(E_TIMEOUT); } event.Signal(); - return MakeFuture(ISessionManager::TSessionOrError(error)); - }; - sessionManager->RemoveSessionHandler = [&] () { - return MakeFuture(NProto::TError()); + return MakeFuture(error); }; auto scheduler = CreateScheduler(); @@ -1050,39 +1062,32 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) scheduler->Stop(); }; - auto logging = CreateLoggingService("console"); - auto executor = TExecutor::Create("TestService"); executor->Start(); auto endpointManager = CreateEndpointManager( - logging, - CreateServerStatsStub(), - executor, - CreateEndpointEventProxy(), - sessionManager, - {{ NProto::IPC_GRPC, std::make_shared() }}, - ""); - - auto endpointService = CreateMultipleEndpointService( - std::make_shared(), CreateWallClockTimer(), scheduler, - logging, + CreateLoggingService("console"), CreateRequestStatsStub(), CreateVolumeStatsStub(), CreateServerStatsStub(), executor, + CreateEndpointEventProxy(), + std::make_shared(), endpointStorage, - endpointManager, - {}); + {{ NProto::IPC_GRPC, listener }}, + nullptr, // nbdDeviceFactory + {} // options + ); - endpointService->Start(); + endpointManager->Start(); Y_DEFER { - endpointService->Stop(); + endpointManager->Stop(); }; - auto unixSocket = "testSocket"; + TTempDir dir; + TString unixSocket = (dir.Path() / "testSocket").GetPath(); auto startRequest = std::make_shared(); startRequest->SetUnixSocketPath(unixSocket); @@ -1103,9 +1108,9 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) strOrError.GetResult()); UNIT_ASSERT_C(!HasError(error), error); - auto restoreFuture = endpointService->RestoreEndpoints(); + auto restoreFuture = endpointManager->RestoreEndpoints(); - auto stopFuture = endpointService->StopEndpoint(ctx, stopRequest); + auto stopFuture = endpointManager->StopEndpoint(ctx, stopRequest); auto stopResponse = stopFuture.GetValue(TDuration::Seconds(5)); UNIT_ASSERT(stopResponse.GetError().GetCode() == E_REJECTED); @@ -1116,7 +1121,7 @@ Y_UNIT_TEST_SUITE(TServiceEndpointTest) UNIT_ASSERT(restoreFuture.Wait(TDuration::Seconds(5))); UNIT_ASSERT(restoreFuture.HasValue()); - auto future = endpointService->StopEndpoint(ctx, stopRequest); + auto future = endpointManager->StopEndpoint(ctx, stopRequest); auto response = future.GetValue(TDuration::Seconds(5)); UNIT_ASSERT(response.GetError().GetCode() == S_OK); diff --git a/cloud/blockstore/libs/endpoints/session_manager.cpp b/cloud/blockstore/libs/endpoints/session_manager.cpp index c17e4ff1eea..794aaafb6fc 100644 --- a/cloud/blockstore/libs/endpoints/session_manager.cpp +++ b/cloud/blockstore/libs/endpoints/session_manager.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -27,10 +26,9 @@ namespace NCloud::NBlockStore::NServer { +using namespace NClient; using namespace NThreading; -using namespace NCloud::NBlockStore::NClient; - namespace { //////////////////////////////////////////////////////////////////////////////// @@ -41,7 +39,6 @@ class TEndpoint TExecutor& Executor; const ISessionPtr Session; const IBlockStorePtr DataClient; - const IVolumeStatsPtr VolumeStats; const IThrottlerProviderPtr ThrottlerProvider; const TString ClientId; const TString DiskId; @@ -51,14 +48,12 @@ class TEndpoint TExecutor& executor, ISessionPtr session, IBlockStorePtr dataClient, - IVolumeStatsPtr volumeStats, IThrottlerProviderPtr throttlerProvider, TString clientId, TString diskId) : Executor(executor) , Session(std::move(session)) , DataClient(std::move(dataClient)) - , VolumeStats(std::move(volumeStats)) , ThrottlerProvider(std::move(throttlerProvider)) , ClientId(std::move(clientId)) , DiskId(std::move(diskId)) @@ -711,7 +706,6 @@ TResultOrError TSessionManager::CreateEndpoint( *Executor, std::move(session), std::move(client), - VolumeStats, ThrottlerProvider, clientId, volume.GetDiskId()); diff --git a/cloud/blockstore/libs/endpoints/ya.make b/cloud/blockstore/libs/endpoints/ya.make index ecd11f7c208..c701cd7f72d 100644 --- a/cloud/blockstore/libs/endpoints/ya.make +++ b/cloud/blockstore/libs/endpoints/ya.make @@ -15,6 +15,7 @@ PEERDIR( cloud/blockstore/libs/common cloud/blockstore/libs/diagnostics cloud/blockstore/libs/encryption + cloud/blockstore/libs/nbd cloud/blockstore/libs/service cloud/blockstore/libs/validation cloud/storage/core/libs/coroutine diff --git a/cloud/blockstore/libs/kikimr/components.h b/cloud/blockstore/libs/kikimr/components.h index 8e4a46499e2..fb5f65206fb 100644 --- a/cloud/blockstore/libs/kikimr/components.h +++ b/cloud/blockstore/libs/kikimr/components.h @@ -2,6 +2,8 @@ #include "public.h" +#include + #include #include @@ -53,6 +55,7 @@ namespace NCloud::NBlockStore { xxx(LOCAL_STORAGE) \ xxx(EXTERNAL_ENDPOINT) \ BLOCKSTORE_ACTORS(xxx) \ + xxx(USER_STATS) \ // BLOCKSTORE_COMPONENTS //////////////////////////////////////////////////////////////////////////////// @@ -61,7 +64,7 @@ struct TBlockStoreComponents { enum { - START = 1024, // TODO + START = TComponentsStart::BlockStoreComponentsStart, #define BLOCKSTORE_DECLARE_COMPONENT(component) \ component, \ @@ -129,20 +132,4 @@ struct TBlockStorePrivateEvents "END expected to be < EventSpaceEnd(NKikimr::TKikimrEvents::BLOCKSTORE)"); }; -//////////////////////////////////////////////////////////////////////////////// - -struct TBlockStoreActivities -{ - enum - { -#define BLOCKSTORE_DECLARE_COMPONENT(component) \ - component = NKikimrServices::TActivity::BLOCKSTORE_##component, \ -// BLOCKSTORE_DECLARE_COMPONENT - - BLOCKSTORE_ACTORS(BLOCKSTORE_DECLARE_COMPONENT) - -#undef BLOCKSTORE_DECLARE_COMPONENT - }; -}; - } // namespace NCloud::NBlockStore diff --git a/cloud/blockstore/libs/nbd/device.cpp b/cloud/blockstore/libs/nbd/device.cpp index 6c22b72ad99..2c215fbac84 100644 --- a/cloud/blockstore/libs/nbd/device.cpp +++ b/cloud/blockstore/libs/nbd/device.cpp @@ -47,6 +47,8 @@ class TDeviceConnection final TSocket Socket; TFileHandle Device; + TAtomic ShouldStop = 0; + public: TDeviceConnection( ILoggingServicePtr logging, @@ -57,12 +59,17 @@ class TDeviceConnection final , ConnectAddress(connectAddress) , DeviceName(std::move(deviceName)) , Timeout(timeout) - {} - - void Start() override { Log = Logging->CreateLog("BLOCKSTORE_NBD"); + } + + ~TDeviceConnection() + { + Stop(); + } + void Start() override + { ConnectSocket(); ConnectDevice(); @@ -71,6 +78,10 @@ class TDeviceConnection final void Stop() override { + if (AtomicSwap(&ShouldStop, 1) == 1) { + return; + } + DisconnectDevice(); ISimpleThread::Join(); @@ -195,6 +206,46 @@ void TDeviceConnection::DisconnectDevice() // device will be closed when thread exit } +//////////////////////////////////////////////////////////////////////////////// + +class TDeviceConnectionStub final + : public IDeviceConnection +{ +public: + void Start() + {} + + void Stop() + {} +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TDeviceConnectionFactory final + : public IDeviceConnectionFactory +{ +private: + const ILoggingServicePtr Logging; + const TDuration Timeout; + +public: + TDeviceConnectionFactory(ILoggingServicePtr logging, TDuration timeout) + : Logging(std::move(logging)) + , Timeout(std::move(timeout)) + {} + + IDeviceConnectionPtr Create( + TNetworkAddress connectAddress, + TString deviceName) override + { + return CreateDeviceConnection( + Logging, + std::move(connectAddress), + std::move(deviceName), + Timeout); + } +}; + } // namespace //////////////////////////////////////////////////////////////////////////////// @@ -212,4 +263,18 @@ IDeviceConnectionPtr CreateDeviceConnection( timeout); } +IDeviceConnectionPtr CreateDeviceConnectionStub() +{ + return std::make_shared(); +} + +IDeviceConnectionFactoryPtr CreateDeviceConnectionFactory( + ILoggingServicePtr logging, + TDuration timeout) +{ + return std::make_shared( + std::move(logging), + std::move(timeout)); +} + } // namespace NCloud::NBlockStore::NBD diff --git a/cloud/blockstore/libs/nbd/device.h b/cloud/blockstore/libs/nbd/device.h index d88b5a4085b..f54b6cac2a3 100644 --- a/cloud/blockstore/libs/nbd/device.h +++ b/cloud/blockstore/libs/nbd/device.h @@ -20,10 +20,27 @@ struct IDeviceConnection //////////////////////////////////////////////////////////////////////////////// +struct IDeviceConnectionFactory +{ + virtual ~IDeviceConnectionFactory() = default; + + virtual IDeviceConnectionPtr Create( + TNetworkAddress connectAddress, + TString deviceName) = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + IDeviceConnectionPtr CreateDeviceConnection( ILoggingServicePtr logging, TNetworkAddress connectAddress, TString deviceName, TDuration timeout); +IDeviceConnectionPtr CreateDeviceConnectionStub(); + +IDeviceConnectionFactoryPtr CreateDeviceConnectionFactory( + ILoggingServicePtr logging, + TDuration timeout); + } // namespace NCloud::NBlockStore::NBD diff --git a/cloud/blockstore/libs/nbd/public.h b/cloud/blockstore/libs/nbd/public.h index d4ccdf4968e..1b4a4be5d54 100644 --- a/cloud/blockstore/libs/nbd/public.h +++ b/cloud/blockstore/libs/nbd/public.h @@ -29,6 +29,9 @@ using IClientHandlerPtr = std::shared_ptr; struct IDeviceConnection; using IDeviceConnectionPtr = std::shared_ptr; +struct IDeviceConnectionFactory; +using IDeviceConnectionFactoryPtr = std::shared_ptr; + struct ILimiter; using ILimiterPtr = std::shared_ptr; diff --git a/cloud/blockstore/libs/nbd/ut/ya.make b/cloud/blockstore/libs/nbd/ut/ya.make index 0f538c0154c..1f612ec59e8 100644 --- a/cloud/blockstore/libs/nbd/ut/ya.make +++ b/cloud/blockstore/libs/nbd/ut/ya.make @@ -6,6 +6,7 @@ SRCS( client_handler_ut.cpp server_handler_ut.cpp server_ut.cpp + utils_ut.cpp ) PEERDIR( diff --git a/cloud/blockstore/libs/nbd/utils.cpp b/cloud/blockstore/libs/nbd/utils.cpp index 379975e34d5..f7b7bf4158c 100644 --- a/cloud/blockstore/libs/nbd/utils.cpp +++ b/cloud/blockstore/libs/nbd/utils.cpp @@ -1,9 +1,33 @@ #include "utils.h" +#include #include +#include +#include +#include +#include + +#include namespace NCloud::NBlockStore::NBD { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TString CleanBackingFilePath(TString path) +{ + // If the block device was deleted, the path will contain a "(deleted)" suffix + static const TString suffix = "(deleted)"; + path = Strip(path); + if (path.EndsWith(suffix)) { + path = path.substr(0, path.length() - suffix.length()); + } + return Strip(path); +} + +} // namespace + //////////////////////////////////////////////////////////////////////////////// static bool IsTcpAddress(int family) @@ -41,4 +65,85 @@ TString PrintHostAndPort(const TNetworkAddress& addr) return out.Str(); } +TSet FindMountedFiles( + const TString& device, + const TString& mountInfoFile) +{ + static constexpr int deviceNameColumn = 3; + static constexpr int mountedFileColumn = 4; + static constexpr int correctColumnCount = 11; + + if (!device.StartsWith("/dev/")) { + return {}; + } + + auto deviceName = device.substr(4); + TSet mountedFiles; + mountedFiles.insert(device); + + if (!NFs::Exists(mountInfoFile)) { + return mountedFiles; + } + + TIFStream is(mountInfoFile); + TString line; + while (is.ReadLine(line)) { + TVector items; + auto columnCount = Split(line, " ", items); + if (columnCount != correctColumnCount) { + continue; + } + + if (items[deviceNameColumn] == deviceName) { + mountedFiles.insert(items[mountedFileColumn]); + } + } + + return mountedFiles; +} + +TVector FindLoopbackDevices( + const TSet& mountedFiles, + const TString& sysBlockDir) +{ + TVector loopbackDevices; + + auto sysBlockItems = std::filesystem::directory_iterator{ + sysBlockDir.c_str()}; + + for (const auto& entry: sysBlockItems) { + if (!entry.is_directory()) { + continue; + } + + TString loopPath = entry.path().string(); + if (!loopPath.StartsWith(sysBlockDir + "loop")) { + continue; + } + + auto backingFile = loopPath + "/loop/backing_file"; + if (!NFs::Exists(backingFile)) { + continue; + } + + TFileInput in(backingFile); + auto data = in.ReadAll(); + auto backingFilePath = CleanBackingFilePath(data); + if (!mountedFiles.contains(backingFilePath)) { + continue; + } + + auto loopDevice = "/dev/" + TFsPath(loopPath).GetName(); + loopbackDevices.push_back(loopDevice); + } + + return loopbackDevices; +} + +int RemoveLoopbackDevice(const TString& loopDevice) +{ + TString cmd = "losetup -d " + loopDevice; + return std::system(cmd.c_str()); +} + } // namespace NCloud::NBlockStore::NBD diff --git a/cloud/blockstore/libs/nbd/utils.h b/cloud/blockstore/libs/nbd/utils.h index 7cd2b9ecc00..2716708e0f5 100644 --- a/cloud/blockstore/libs/nbd/utils.h +++ b/cloud/blockstore/libs/nbd/utils.h @@ -33,4 +33,14 @@ bool IsUnixAddress(const TNetworkAddress& addr); TString PrintHostAndPort(const TNetworkAddress& addr); +TSet FindMountedFiles( + const TString& device, + const TString& mountInfoFile = "/proc/self/mountinfo"); + +TVector FindLoopbackDevices( + const TSet& mountedFiles, + const TString& sysBlockDir = "/sys/block/"); + +int RemoveLoopbackDevice(const TString& loopDevice); + } // namespace NCloud::NBlockStore::NBD diff --git a/cloud/blockstore/libs/nbd/utils_ut.cpp b/cloud/blockstore/libs/nbd/utils_ut.cpp new file mode 100644 index 00000000000..a5efdea9d65 --- /dev/null +++ b/cloud/blockstore/libs/nbd/utils_ut.cpp @@ -0,0 +1,121 @@ +#include "utils.h" + +#include +#include + +#include +#include + +namespace NCloud::NBlockStore::NBD { + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(TUtilsTest) +{ + Y_UNIT_TEST(ShouldFindMountedFiles) + { + auto mountInfoStr = R"( + 356 355 0:49 / /proc rw,nosuid,nodev,noexec,relatime shared:146 - proc proc rw + 357 355 0:50 / /dev rw,nosuid shared:147 - tmpfs tmpfs rw,size=65536k,mode=755,inode64 + 358 357 0:51 / /dev/pts rw,nosuid,noexec,relatime shared:155 - devpts devpts rw,gid=5,mode=620,ptmxmode=666 + 359 355 0:52 / /sys ro,nosuid,nodev,noexec,relatime shared:176 - sysfs sysfs ro + 360 359 0:29 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:203 - cgroup2 cgroup rw,nsdelegate,memory_recursiveprot + 361 357 0:48 / /dev/mqueue rw,nosuid,nodev,noexec,relatime shared:158 - mqueue mqueue rw + 362 357 0:53 / /dev/shm rw,nosuid,nodev,noexec,relatime shared:172 - tmpfs shm rw,size=65536k,inode64 + 363 355 252:2 /var/lib/docker/volumes/minikube/_data /var rw,relatime shared:205 master:1 - ext4 /dev/vda2 rw + 364 355 0:54 / /run rw,nosuid,nodev,noexec,relatime shared:206 - tmpfs tmpfs rw,inode64 + 366 355 0:56 / /tmp rw,nosuid,nodev,noexec,relatime shared:207 - tmpfs tmpfs rw,inode64 + 368 355 252:2 /usr/lib/modules /usr/lib/modules ro,relatime shared:234 - ext4 /dev/vda2 rw + 173 357 0:51 /0 /dev/console rw,nosuid,noexec,relatime shared:175 - devpts devpts rw,gid=5,mode=620,ptmxmode=666 + 200 364 0:57 / /run/lock rw,nosuid,nodev,noexec,relatime shared:282 - tmpfs tmpfs rw,size=5120k,inode64 + 211 357 0:59 / /dev/hugepages rw,relatime shared:284 - hugetlbfs hugetlbfs rw,pagesize=2M + 224 359 0:7 / /sys/kernel/debug rw,nosuid,nodev,noexec,relatime shared:285 - debugfs debugfs rw + 227 359 0:12 / /sys/kernel/tracing rw,nosuid,nodev,noexec,relatime shared:288 - tracefs tracefs rw + 228 359 0:34 / /sys/fs/fuse/connections rw,nosuid,nodev,noexec,relatime shared:289 - fusectl fusectl rw + 511 364 0:60 / /run/credentials/systemd-sysusers.service ro,nosuid,nodev,noexec,relatime shared:290 - ramfs none rw,mode=700 + 229 209 0:36 / /proc/sys/fs/binfmt_misc rw,nosuid,nodev,noexec,relatime shared:291 - binfmt_misc binfmt_misc rw + 236 355 252:2 /var/lib/docker/volumes/minikube/_data/data /data rw,relatime shared:205 master:1 - ext4 /dev/vda2 rw + 244 366 252:2 /var/lib/docker/volumes/minikube/_data/hostpath_pv /tmp/hostpath_pv rw,relatime shared:205 master:1 - ext4 /dev/vda2 rw + 252 366 252:2 /var/lib/docker/volumes/minikube/_data/hostpath-provisioner /tmp/hostpath-provisioner rw,relatime shared:205 master:1 - ext4 /dev/vda2 rw + 272 364 0:4 net:[4026533085] /run/docker/netns/default rw shared:293 - nsfs nsfs rw + 854 364 0:4 net:[4026533185] /run/docker/netns/d8ef02b88399 rw shared:316 - nsfs nsfs rw + 959 363 0:182 / /var/lib/docker/containers/blabla1/mounts/shm rw,nosuid,nodev,noexec,relatime shared:321 - tmpfs shm rw,size=65536k,inode64 + 932 363 0:180 / /var/lib/docker/containers/blabla2/mounts/shm rw,nosuid,nodev,noexec,relatime shared:322 - tmpfs shm rw,size=65536k,inode64 + 826 363 0:139 / /var/lib/kubelet/pods/5f8a1a7e-d9f0-48be-97d3-8d655855d5fc/volumes/kubernetes.io~projected/kube-api-access-mgmsb rw,relatime shared:308 - tmpfs tmpfs rw,size=393216k,inode64 + 831 363 0:143 / /var/lib/kubelet/pods/05108687-5f0d-42c7-890d-bd2dd69d6466/volumes/kubernetes.io~projected/kube-api-access-lz87t rw,relatime shared:309 - tmpfs tmpfs rw,size=262144k,inode64 + 847 363 0:144 / /var/lib/docker/containers/blabla3/mounts/shm rw,nosuid,nodev,noexec,relatime shared:312 - tmpfs shm rw,size=65536k,inode64 + 869 363 0:163 / /var/lib/docker/containers/blabla4/mounts/shm rw,nosuid,nodev,noexec,relatime shared:323 - tmpfs shm rw,size=65536k,inode64 + 894 363 0:177 / /var/lib/kubelet/pods/blabla5/volumes/kubernetes.io~projected/kube-api-access-wf5ls rw,relatime shared:317 - tmpfs tmpfs rw,size=165029212k,inode64 + 1008 363 0:243 /nbd1 /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/publish/pvc-blabla6/blabla5 rw,nosuid shared:335 - tmpfs tmpfs rw,size=65536k,mode=755,inode64 + 1009 363 0:243 /nbd1 /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/pvc-blabla6/dev/blabla5 rw shared:335 - tmpfs tmpfs rw,size=65536k,mode=755,inode64 + 1015 363 0:253 / /var/lib/docker/containers/blabla7/mounts/shm rw,nosuid,nodev,noexec,relatime shared:337 - tmpfs shm rw,size=65536k,inode64 + 1428 364 0:4 net:[4026533292] /run/docker/netns/5dc95378b825 rw shared:338 - nsfs nsfs rw + )"; + + TTempDir dir; + auto mountInfoPath = dir.Path() / "mountinfo"; + TOFStream(mountInfoPath.GetPath()).Write(mountInfoStr); + + auto mountedFiles = FindMountedFiles( + "/dev/nbd1", + mountInfoPath.GetPath()); + + TSet expectedFiles = { + "/dev/nbd1", + "/var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/publish/pvc-blabla6/blabla5", + "/var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/pvc-blabla6/dev/blabla5", + }; + UNIT_ASSERT_VALUES_EQUAL(expectedFiles, mountedFiles); + } + + Y_UNIT_TEST(ShouldFindLoopbackDevices) + { + TTempDir sysBlockDir; + + TSet mountedFiles = { + "/dev/nbd1", + "/var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/publish/pvc-blabla6/blabla5", + "/var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/pvc-blabla6/dev/blabla5", + }; + + TFsPath backingFile; + backingFile = sysBlockDir.Path() / "loop0" / "loop" / "backing_file"; + backingFile.Parent().MkDirs(); + TOFStream(backingFile.GetPath()).Write(R"( + /var/lib/snapd/snaps/lxd_27428.snap + )"); + + backingFile = sysBlockDir.Path() / "loop1" / "loop" / "backing_file"; + backingFile.Parent().MkDirs(); + TOFStream(backingFile.GetPath()).Write(R"( + /var/lib/snapd/snaps/core20_2105.snap + )"); + + backingFile = sysBlockDir.Path() / "loop2" / "loop" / "backing_file"; + backingFile.Parent().MkDirs(); + TOFStream(backingFile.GetPath()).Write(R"( + / + )"); + + backingFile = sysBlockDir.Path() / "loop3" / "loop" / "backing_file"; + backingFile.Parent().MkDirs(); + TOFStream(backingFile.GetPath()).Write(R"( + /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/pvc-blabla6/dev/blabla5 + )"); + + backingFile = sysBlockDir.Path() / "loop4" / "loop" / "backing_file"; + backingFile.Parent().MkDirs(); + TOFStream(backingFile.GetPath()).Write(R"( + /var/lib/snapd/snaps/core20_2182.snap + )"); + + auto loopbackDevices = FindLoopbackDevices( + mountedFiles, + sysBlockDir.Path()); + + TVector expectedDevices = {"/dev/loop3"}; + UNIT_ASSERT_VALUES_EQUAL(expectedDevices, loopbackDevices); + } +} + +} // namespace NCloud::NBlockStore::NBD diff --git a/cloud/blockstore/libs/rdma/iface/client.cpp b/cloud/blockstore/libs/rdma/iface/client.cpp index c055d12d866..bd6fd1cd89a 100644 --- a/cloud/blockstore/libs/rdma/iface/client.cpp +++ b/cloud/blockstore/libs/rdma/iface/client.cpp @@ -1,5 +1,7 @@ #include "client.h" +#include + namespace NCloud::NBlockStore::NRdma { namespace { @@ -44,4 +46,51 @@ TClientConfig::TClientConfig(const NProto::TRdmaClient& config) #undef SET +void TClientConfig::DumpHtml(IOutputStream& out) const +{ +#define ENTRY(name, val, ...) \ + TABLER() { \ + TABLED() { out << #name; } \ + TABLED() { out << val; } \ + } + + HTML(out) { + TABLE_CLASS("table table-condensed") { + TABLEBODY() { + ENTRY(QueueSize, QueueSize); + ENTRY(MaxBufferSize, MaxBufferSize); + ENTRY(WaitMode, WaitMode); + ENTRY(PollerThreads, PollerThreads); + ENTRY(MaxReconnectDelay, MaxReconnectDelay.ToString()); + ENTRY(MaxResponseDelay, MaxResponseDelay.ToString()); + ENTRY(AdaptiveWaitSleepDelay, AdaptiveWaitSleepDelay.ToString()); + ENTRY(AdaptiveWaitSleepDuration, AdaptiveWaitSleepDuration.ToString()); + } + } + } +#undef ENTRY +} + } // namespace NCloud::NBlockStore::NRdma + +//////////////////////////////////////////////////////////////////////////////// + +template <> +inline void Out( + IOutputStream& o, + const NCloud::NBlockStore::NRdma::EWaitMode mode) +{ + switch (mode) { + case NCloud::NBlockStore::NRdma::EWaitMode::Poll: + o << "POLL"; + break; + + case NCloud::NBlockStore::NRdma::EWaitMode::BusyWait: + o << "BUSY_WAIT"; + break; + + case NCloud::NBlockStore::NRdma::EWaitMode::AdaptiveWait: + o << "ADAPTIVE_WAIT"; + break; + } +} diff --git a/cloud/blockstore/libs/rdma/iface/client.h b/cloud/blockstore/libs/rdma/iface/client.h index 8c461147321..14f07c65291 100644 --- a/cloud/blockstore/libs/rdma/iface/client.h +++ b/cloud/blockstore/libs/rdma/iface/client.h @@ -33,6 +33,8 @@ struct TClientConfig TClientConfig() = default; TClientConfig(const NProto::TRdmaClient& config); + + void DumpHtml(IOutputStream& out) const; }; //////////////////////////////////////////////////////////////////////////////// @@ -109,6 +111,8 @@ struct IClient virtual NThreading::TFuture StartEndpoint( TString host, ui32 port) = 0; + + virtual void DumpHtml(IOutputStream& out) const = 0; }; } // namespace NCloud::NBlockStore::NRdma diff --git a/cloud/blockstore/libs/rdma/iface/config.cpp b/cloud/blockstore/libs/rdma/iface/config.cpp new file mode 100644 index 00000000000..8ff801a0f06 --- /dev/null +++ b/cloud/blockstore/libs/rdma/iface/config.cpp @@ -0,0 +1,11 @@ +#include "config.h" + +namespace NCloud::NBlockStore::NRdma { + +//////////////////////////////////////////////////////////////////////////////// + +TRdmaConfig::TRdmaConfig(NProto::TRdmaConfig config) + : Config(config) +{} + +} // namespace NCloud::NBlockStore::NRdma diff --git a/cloud/blockstore/libs/rdma/iface/config.h b/cloud/blockstore/libs/rdma/iface/config.h new file mode 100644 index 00000000000..57feaa76874 --- /dev/null +++ b/cloud/blockstore/libs/rdma/iface/config.h @@ -0,0 +1,40 @@ +#pragma once + +#include "public.h" + +#include + +namespace NCloud::NBlockStore::NRdma { + +//////////////////////////////////////////////////////////////////////////////// + +class TRdmaConfig +{ +private: + const NProto::TRdmaConfig Config; + +public: + explicit TRdmaConfig(NProto::TRdmaConfig config = {}); + + auto GetClientEnabled() const + { + return Config.GetClientEnabled(); + } + + auto GetClient() const + { + return Config.GetClient(); + } + + auto GetServerEnabled() const + { + return Config.GetServerEnabled(); + } + + auto GetServer() const + { + return Config.GetServer(); + } +}; + +} // namespace NCloud::NBlockStore::NRdma diff --git a/cloud/blockstore/libs/rdma/iface/public.h b/cloud/blockstore/libs/rdma/iface/public.h index 13fc0468587..a0ee2d79b86 100644 --- a/cloud/blockstore/libs/rdma/iface/public.h +++ b/cloud/blockstore/libs/rdma/iface/public.h @@ -53,4 +53,7 @@ using TProtoMessagePtr = std::unique_ptr; class TProtoMessageSerializer; +class TRdmaConfig; +using TRdmaConfigPtr = std::shared_ptr; + } // namespace NCloud::NBlockStore::NRdma diff --git a/cloud/blockstore/libs/rdma/iface/server.cpp b/cloud/blockstore/libs/rdma/iface/server.cpp index 5676ac864b9..8ee45e54563 100644 --- a/cloud/blockstore/libs/rdma/iface/server.cpp +++ b/cloud/blockstore/libs/rdma/iface/server.cpp @@ -1,5 +1,7 @@ #include "server.h" +#include + namespace NCloud::NBlockStore::NRdma { namespace { @@ -47,4 +49,52 @@ TServerConfig::TServerConfig(const NProto::TRdmaServer& config) #undef SET +void TServerConfig::DumpHtml(IOutputStream& out) const +{ +#define ENTRY(name, val, ...) \ + TABLER() { \ + TABLED() { out << #name; } \ + TABLED() { out << val; } \ + } + + HTML(out) { + TABLE_CLASS("table table-condensed") { + TABLEBODY() { + ENTRY(Backlog, Backlog); + ENTRY(QueueSize, QueueSize); + ENTRY(MaxBufferSize, MaxBufferSize); + ENTRY(WaitMode, WaitMode); + ENTRY(PollerThreads, PollerThreads); + ENTRY(MaxInflightBytes, MaxInflightBytes); + ENTRY(AdaptiveWaitSleepDelay, AdaptiveWaitSleepDelay.ToString()); + ENTRY(AdaptiveWaitSleepDuration, AdaptiveWaitSleepDuration.ToString()); + } + } + } + +#undef ENTRY +} + } // namespace NCloud::NBlockStore::NRdma + +//////////////////////////////////////////////////////////////////////////////// + +template <> +inline void Out( + IOutputStream& o, + const NCloud::NBlockStore::NRdma::EWaitMode mode) +{ + switch (mode) { + case NCloud::NBlockStore::NRdma::EWaitMode::Poll: + o << "POLL"; + break; + + case NCloud::NBlockStore::NRdma::EWaitMode::BusyWait: + o << "BUSY_WAIT"; + break; + + case NCloud::NBlockStore::NRdma::EWaitMode::AdaptiveWait: + o << "ADAPTIVE_WAIT"; + break; + } +} diff --git a/cloud/blockstore/libs/rdma/iface/server.h b/cloud/blockstore/libs/rdma/iface/server.h index 8ad0a44b580..a83b6e365a1 100644 --- a/cloud/blockstore/libs/rdma/iface/server.h +++ b/cloud/blockstore/libs/rdma/iface/server.h @@ -32,6 +32,8 @@ struct TServerConfig TServerConfig() = default; TServerConfig(const NProto::TRdmaServer& config); + + void DumpHtml(IOutputStream& out) const; }; //////////////////////////////////////////////////////////////////////////////// @@ -68,6 +70,8 @@ struct IServer TString host, ui32 port, IServerHandlerPtr handler) = 0; + + virtual void DumpHtml(IOutputStream& out) const = 0; }; } // namespace NCloud::NBlockStore::NRdma diff --git a/cloud/blockstore/libs/rdma/iface/ya.make b/cloud/blockstore/libs/rdma/iface/ya.make index 1e4dfb64c4c..4840cb46ba1 100644 --- a/cloud/blockstore/libs/rdma/iface/ya.make +++ b/cloud/blockstore/libs/rdma/iface/ya.make @@ -2,6 +2,7 @@ LIBRARY() SRCS( client.cpp + config.cpp probes.cpp protobuf.cpp protocol.cpp diff --git a/cloud/blockstore/libs/rdma/impl/client.cpp b/cloud/blockstore/libs/rdma/impl/client.cpp index a80283e39fb..c4484749714 100644 --- a/cloud/blockstore/libs/rdma/impl/client.cpp +++ b/cloud/blockstore/libs/rdma/impl/client.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include @@ -1376,6 +1377,11 @@ class TCompletionPoller final } } + auto GetEndpoints() + { + return Endpoints.Get(); + } + private: bool ShouldStop() const { @@ -1574,6 +1580,7 @@ class TClient final TFuture StartEndpoint( TString host, ui32 port) noexcept override; + void DumpHtml(IOutputStream& out) const override; private: // called from external thread @@ -1989,6 +1996,77 @@ TCompletionPoller& TClient::PickPoller() noexcept return *CompletionPollers[index]; } +void TClient::DumpHtml(IOutputStream& out) const +{ + HTML(out) { + TAG(TH4) { out << "Config"; } + Config->DumpHtml(out); + + TAG(TH4) { out << "Counters"; } + TABLE_CLASS("table table-bordered") { + TABLEHEAD() { + TABLER() { + TABLEH() { out << "QueuedRequests"; } + TABLEH() { out << "ActiveRequests"; } + TABLEH() { out << "AbortedRequests"; } + TABLEH() { out << "CompletedRequests"; } + TABLEH() { out << "UnknownRequests"; } + TABLEH() { out << "ActiveSend"; } + TABLEH() { out << "ActiveRecv"; } + TABLEH() { out << "SendErrors"; } + TABLEH() { out << "RecvErrors"; } + TABLEH() { out << "UnexpectedCompletions"; } + } + TABLER() { + TABLED() { out << Counters->QueuedRequests->Val(); } + TABLED() { out << Counters->ActiveRequests->Val(); } + TABLED() { out << Counters->AbortedRequests->Val(); } + TABLED() { out << Counters->CompletedRequests->Val(); } + TABLED() { out << Counters->UnknownRequests->Val(); } + TABLED() { out << Counters->ActiveSend->Val(); } + TABLED() { out << Counters->ActiveRecv->Val(); } + TABLED() { out << Counters->SendErrors->Val(); } + TABLED() { out << Counters->RecvErrors->Val(); } + TABLED() { out << Counters->UnexpectedCompletions->Val(); } + } + } + } + + TAG(TH4) { out << "Endpoints"; } + TABLE_SORTABLE_CLASS("table table-bordered") { + TABLEHEAD() { + TABLER() { + TABLEH() { out << "Poller"; } + TABLEH() { out << "Host"; } + TABLEH() { out << "Port"; } + TABLEH() { out << "Magic"; } + } + } + + for (size_t i = 0; i < CompletionPollers.size(); ++i) { + auto& poller = CompletionPollers[i]; + auto endpoints = poller->GetEndpoints(); + for (auto& ep : *endpoints) { + TABLER() { + TABLED() { out << i; } + TABLED() { out << ep->Host; } + TABLED() { out << ep->Port; } + TABLED() + { + Printf( + out, + "%08X:%08X:%d", + ep->SendMagic, + ep->RecvMagic, + ep->Generation); + } + } + } + } + } + } +} + } // namespace //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/blockstore/libs/rdma/impl/server.cpp b/cloud/blockstore/libs/rdma/impl/server.cpp index 5469c4e6356..5ea56507c8c 100644 --- a/cloud/blockstore/libs/rdma/impl/server.cpp +++ b/cloud/blockstore/libs/rdma/impl/server.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -319,6 +320,7 @@ class TServerSession final // called from external thread void EnqueueRequest(TRequestPtr req) noexcept; + TString GetAddress() const; // called from CQ thread void HandleCompletionEvent(ibv_wc* wc) override; @@ -454,7 +456,7 @@ TServerSession::~TServerSession() STORAGE_INFO("close session [send_magic=%X recv_magic=%X] to %s", SendMagic, RecvMagic, - NVerbs::PrintAddress(rdma_get_peer_addr(Connection.get())).c_str()); + GetAddress().c_str()); Verbs->DestroyQP(Connection.get()); @@ -500,6 +502,11 @@ void TServerSession::EnqueueRequest(TRequestPtr req) noexcept } } +TString TServerSession::GetAddress() const +{ + return NVerbs::PrintAddress(rdma_get_peer_addr(Connection.get())); +} + bool TServerSession::HandleInputRequests() { if (Config->WaitMode == EWaitMode::Poll) { @@ -1276,6 +1283,10 @@ class TCompletionPoller final } } + auto GetSessions() { + return Sessions.Get(); + } + private: bool ShouldStop() const { @@ -1432,6 +1443,7 @@ class TServer final TString host, ui32 port, IServerHandlerPtr handler) override; + void DumpHtml(IOutputStream& out) const override; private: // called from external thread @@ -1564,6 +1576,107 @@ void TServer::Listen(TServerEndpoint* endpoint) Verbs->Listen(endpoint->Connection.get(), Config->Backlog); } +// implements IServer +void TServer::DumpHtml(IOutputStream& out) const +{ + HTML(out) { + TAG(TH4) { out << "Config"; } + Config->DumpHtml(out); + + TAG(TH4) { out << "Counters"; } + TABLE_CLASS("table table-bordered") { + TABLEHEAD() { + TABLER() { + TABLEH() { out << "QueuedRequests"; } + TABLEH() { out << "ActiveRequests"; } + TABLEH() { out << "CompletedRequests"; } + TABLEH() { out << "ThrottledRequests"; } + TABLEH() { out << "ActiveSend"; } + TABLEH() { out << "ActiveRecv"; } + TABLEH() { out << "ActiveRead"; } + TABLEH() { out << "ActiveWrite"; } + TABLEH() { out << "SendErrors"; } + TABLEH() { out << "RecvErrors"; } + TABLEH() { out << "ReadErrors"; } + TABLEH() { out << "WriteErrors"; } + TABLEH() { out << "UnexpectedCompletions"; } + } + TABLER() { + TABLED() { out << Counters->QueuedRequests->Val(); } + TABLED() { out << Counters->ActiveRequests->Val(); } + TABLED() { out << Counters->CompletedRequests->Val(); } + TABLED() { out << Counters->ThrottledRequests->Val(); } + TABLED() { out << Counters->ActiveSend->Val(); } + TABLED() { out << Counters->ActiveRecv->Val(); } + TABLED() { out << Counters->ActiveRead->Val(); } + TABLED() { out << Counters->ActiveWrite->Val(); } + TABLED() { out << Counters->SendErrors->Val(); } + TABLED() { out << Counters->RecvErrors->Val(); } + TABLED() { out << Counters->ReadErrors->Val(); } + TABLED() { out << Counters->WriteErrors->Val(); } + TABLED() { out << Counters->UnexpectedCompletions->Val(); } + } + } + } + + TAG(TH4) { out << "Endpoints"; } + TABLE_CLASS("table table-bordered") { + TABLEHEAD() { + TABLER() { + TABLEH() { out << "Host"; } + TABLEH() { out << "Port"; } + } + } + + with_lock (EndpointsLock) { + for (auto& ep : Endpoints) { + TABLER() { + TABLED() { out << ep->Host; } + TABLED() { out << ep->Port; } + } + } + } + } + + for (size_t i = 0; i < CompletionPollers.size(); ++i) { + auto& poller = CompletionPollers[i]; + auto sessions = poller->GetSessions(); + + if (sessions->empty()) { + continue; + } + + TAG(TH4) { + out << "Poller-" << i << " ServerSessions" + << " " << sessions->size() << ""; + } + + TABLE_SORTABLE_CLASS("table table-bordered") { + TABLEHEAD() { + TABLER() { + TABLEH() { out << "Address"; } + TABLEH() { out << "Magic"; } + } + } + + for (auto& session: *sessions) { + TABLER() { + TABLED() { out << session->GetAddress(); } + TABLED() + { + Printf( + out, + "%08X:%08X", + session->SendMagic, + session->RecvMagic); + } + } + } + } + } + } +} + //////////////////////////////////////////////////////////////////////////////// // implements IConnectionEventHandler diff --git a/cloud/blockstore/libs/rdma_test/client_test.h b/cloud/blockstore/libs/rdma_test/client_test.h index c397fac7f34..8ede1697186 100644 --- a/cloud/blockstore/libs/rdma_test/client_test.h +++ b/cloud/blockstore/libs/rdma_test/client_test.h @@ -41,6 +41,11 @@ struct TRdmaClientTest: NRdma::IClient { } + void DumpHtml(IOutputStream& out) const override + { + Y_UNUSED(out); + } + void InjectErrors( NProto::TError allocationError, NProto::TError rdmaResponseError, diff --git a/cloud/blockstore/libs/rdma_test/server_test_async.h b/cloud/blockstore/libs/rdma_test/server_test_async.h index bb9eeb9e160..31640bc2eb5 100644 --- a/cloud/blockstore/libs/rdma_test/server_test_async.h +++ b/cloud/blockstore/libs/rdma_test/server_test_async.h @@ -30,6 +30,11 @@ class TRdmaAsyncTestServer: public NRdma::IServer void Stop() override {} + void DumpHtml(IOutputStream& out) const override + { + Y_UNUSED(out); + } + NThreading::TFuture Run( TString host, ui32 port, diff --git a/cloud/blockstore/libs/server/config.cpp b/cloud/blockstore/libs/server/config.cpp index 6f7d73e6fda..ba285c21cb6 100644 --- a/cloud/blockstore/libs/server/config.cpp +++ b/cloud/blockstore/libs/server/config.cpp @@ -88,6 +88,7 @@ constexpr TDuration Seconds(int s) NCloud::NProto::ENDPOINT_STORAGE_KEYRING )\ xxx(EndpointStorageDir, TString, {} )\ xxx(VhostServerPath, TString, {} )\ + xxx(NbdDevicePrefix, TString, "/dev/nbd" )\ // BLOCKSTORE_SERVER_CONFIG #define BLOCKSTORE_SERVER_DECLARE_CONFIG(name, type, value) \ @@ -293,4 +294,15 @@ void TServerAppConfig::DumpHtml(IOutputStream& out) const #undef BLOCKSTORE_CONFIG_DUMP } +bool TServerAppConfig::DeprecatedGetRdmaClientEnabled() const +{ + return GetRdmaClientEnabled(); +} + +const NProto::TRdmaClient& +TServerAppConfig::DeprecatedGetRdmaClientConfig() const +{ + return ServerConfig->GetRdmaClientConfig(); +} + } // namespace NCloud::NBlockStore::NServer diff --git a/cloud/blockstore/libs/server/config.h b/cloud/blockstore/libs/server/config.h index 49575d54189..49f5781a053 100644 --- a/cloud/blockstore/libs/server/config.h +++ b/cloud/blockstore/libs/server/config.h @@ -120,13 +120,18 @@ class TServerAppConfig ui32 GetMaxReadIops() const; ui32 GetMaxWriteIops() const; TDuration GetMaxBurstTime() const; - bool GetRdmaClientEnabled() const; + bool DeprecatedGetRdmaClientEnabled() const; + const NProto::TRdmaClient& DeprecatedGetRdmaClientConfig() const; NCloud::NProto::EEndpointStorageType GetEndpointStorageType() const; TString GetEndpointStorageDir() const; TString GetVhostServerPath() const; + TString GetNbdDevicePrefix() const; void Dump(IOutputStream& out) const override; void DumpHtml(IOutputStream& out) const override; + +private: + bool GetRdmaClientEnabled() const; }; } // namespace NCloud::NBlockStore::NServer diff --git a/cloud/blockstore/libs/server/server.cpp b/cloud/blockstore/libs/server/server.cpp index bf4ff42ec30..bea4c9db756 100644 --- a/cloud/blockstore/libs/server/server.cpp +++ b/cloud/blockstore/libs/server/server.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -244,6 +245,9 @@ class TSessionStorage final auto it = FindClient(socket); Y_ABORT_UNLESS(it == ClientInfos.end()); + // create duplicate socket. we poll socket to know when + // client disconnects and dupSocket is passed to GRPC to read + // messages. dupSocket = SafeCreateDuplicate(socket); auto client = TClientInfo { @@ -302,7 +306,10 @@ class TSessionStorage final return TSocketHolder(duplicateFd); } - // need for avoid race, see NBS-3325 + // Grpc can release socket before we call RemoveClient. So it is possible + // to observe the situation when CreateDuplicate returns fd which was not + // yet removed from ClientInfo's. So completed RemoveClient will close fd + // related to another connection. TSocketHolder SafeCreateDuplicate(const TSocketHolder& socket) { TList holders; @@ -649,7 +656,7 @@ class TRequestHandler final if (!source) { ui32 fd = 0; auto peer = Context->peer(); - bool result = TryParseSourceFd(peer, fd); + bool result = TryParseSourceFd(peer, &fd); if (!result) { ythrow TServiceError(E_FAIL) @@ -698,17 +705,6 @@ class TRequestHandler final } } - bool TryParseSourceFd(const TStringBuf& peer, ui32& fd) - { - static const TString PeerFdPrefix = "fd:"; - if (!peer.StartsWith(PeerFdPrefix)) { - return false; - } - - auto peerFd = peer.SubString(PeerFdPrefix.length(), peer.length()); - return TryFromString(peerFd, fd); - } - bool EndpointIsStopped() const { if (SourceFd == InvalidFd) { diff --git a/cloud/blockstore/libs/server/tsan.supp b/cloud/blockstore/libs/server/tsan.supp new file mode 100644 index 00000000000..7d1039d2522 --- /dev/null +++ b/cloud/blockstore/libs/server/tsan.supp @@ -0,0 +1,2 @@ +# There is race between c_call() & set_call() +race:server.cpp diff --git a/cloud/blockstore/libs/service/request.cpp b/cloud/blockstore/libs/service/request.cpp index 54e72d4818f..edd96f65936 100644 --- a/cloud/blockstore/libs/service/request.cpp +++ b/cloud/blockstore/libs/service/request.cpp @@ -35,6 +35,8 @@ TStringBuf GetSysRequestName(ESysRequestType requestType) case ESysRequestType::Migration: return "Migration"; case ESysRequestType::WriteDeviceBlocks: return "WriteDeviceBlocks"; case ESysRequestType::ZeroDeviceBlocks: return "ZeroDeviceBlocks"; + case ESysRequestType::Resync: return "Resync"; + case ESysRequestType::ConfirmBlobs: return "ConfirmBlobs"; default: return "unknown"; } } diff --git a/cloud/blockstore/libs/service/request.h b/cloud/blockstore/libs/service/request.h index 1ba43cf4897..adf7520f2ea 100644 --- a/cloud/blockstore/libs/service/request.h +++ b/cloud/blockstore/libs/service/request.h @@ -161,6 +161,7 @@ enum class ESysRequestType WriteDeviceBlocks = 10006, ZeroDeviceBlocks = 10007, Resync = 10008, + ConfirmBlobs = 10009, MAX }; diff --git a/cloud/blockstore/libs/service_local/storage_rdma_ut.cpp b/cloud/blockstore/libs/service_local/storage_rdma_ut.cpp index ae30513596a..4c4920ed93e 100644 --- a/cloud/blockstore/libs/service_local/storage_rdma_ut.cpp +++ b/cloud/blockstore/libs/service_local/storage_rdma_ut.cpp @@ -63,6 +63,11 @@ class TRdmaClientHelper: public NRdma::IClient void Stop() override {} + + void DumpHtml(IOutputStream& out) const override + { + Y_UNUSED(out); + } }; } // namespace diff --git a/cloud/blockstore/libs/service_throttling/ya.make b/cloud/blockstore/libs/service_throttling/ya.make index 4ed3c23f368..846d6794415 100644 --- a/cloud/blockstore/libs/service_throttling/ya.make +++ b/cloud/blockstore/libs/service_throttling/ya.make @@ -22,7 +22,7 @@ PEERDIR( END() -# Until https://st.yandex-team.ru/DEVTOOLSSUPPORT-25698 is not solved. +# Until DEVTOOLSSUPPORT-25698 is not solved. IF (SANITIZER_TYPE == "address" OR SANITIZER_TYPE == "memory") RECURSE_FOR_TESTS( ut diff --git a/cloud/blockstore/libs/storage/core/config.cpp b/cloud/blockstore/libs/storage/core/config.cpp index 007133522d6..0b6eb7e1ec7 100644 --- a/cloud/blockstore/libs/storage/core/config.cpp +++ b/cloud/blockstore/libs/storage/core/config.cpp @@ -109,6 +109,8 @@ TDuration MSeconds(ui32 value) xxx(TenantHiveTabletId, ui64, 0 )\ \ xxx(MaxChangedBlocksRangeBlocksCount, ui64, 1 << 20 )\ + xxx(CachedDiskAgentConfigPath, TString, "" )\ + xxx(CachedDiskAgentSessionsPath, TString, "" )\ // BLOCKSTORE_STORAGE_CONFIG_RO #define BLOCKSTORE_STORAGE_CONFIG_RW(xxx) \ @@ -466,6 +468,15 @@ TDuration MSeconds(ui32 value) xxx(CachedAcquireRequestLifetime, TDuration, Seconds(40) )\ \ xxx(UnconfirmedBlobCountHardLimit, ui32, 1000 )\ + \ + xxx(MaxShadowDiskFillBandwidth, ui32, 500 )\ + xxx(MinAcquireShadowDiskRetryDelayWhenBlocked, TDuration, MSeconds(250) )\ + xxx(MaxAcquireShadowDiskRetryDelayWhenBlocked, TDuration, Seconds(1) )\ + xxx(MinAcquireShadowDiskRetryDelayWhenNonBlocked, TDuration, Seconds(1) )\ + xxx(MaxAcquireShadowDiskRetryDelayWhenNonBlocked, TDuration, Seconds(10) )\ + xxx(MaxAcquireShadowDiskTotalTimeoutWhenBlocked, TDuration, Seconds(5) )\ + xxx(MaxAcquireShadowDiskTotalTimeoutWhenNonBlocked, TDuration, Seconds(600) )\ + // BLOCKSTORE_STORAGE_CONFIG_RW #define BLOCKSTORE_STORAGE_CONFIG(xxx) \ @@ -494,6 +505,7 @@ BLOCKSTORE_STORAGE_CONFIG(BLOCKSTORE_STORAGE_DECLARE_CONFIG) xxx(ChangeThrottlingPolicy) \ xxx(ReplaceDevice) \ xxx(UseNonReplicatedHDDInsteadOfReplicated) \ + xxx(AddingUnconfirmedBlobs) \ // BLOCKSTORE_BINARY_FEATURES diff --git a/cloud/blockstore/libs/storage/core/config.h b/cloud/blockstore/libs/storage/core/config.h index cf578a39868..1c3ca0af9b6 100644 --- a/cloud/blockstore/libs/storage/core/config.h +++ b/cloud/blockstore/libs/storage/core/config.h @@ -348,6 +348,10 @@ class TStorageConfig const TString& cloudId, const TString& folderId, const TString& diskId) const; + bool IsAddingUnconfirmedBlobsFeatureEnabled( + const TString& cloudId, + const TString& folderId, + const TString& diskId) const; TDuration GetMaxTimedOutDeviceStateDurationFeatureValue( const TString& cloudId, @@ -546,6 +550,17 @@ class TStorageConfig TDuration GetCachedAcquireRequestLifetime() const; ui32 GetUnconfirmedBlobCountHardLimit() const; + + TString GetCachedDiskAgentConfigPath() const; + TString GetCachedDiskAgentSessionsPath() const; + + ui32 GetMaxShadowDiskFillBandwidth() const; + TDuration GetMinAcquireShadowDiskRetryDelayWhenBlocked() const; + TDuration GetMaxAcquireShadowDiskRetryDelayWhenBlocked() const; + TDuration GetMinAcquireShadowDiskRetryDelayWhenNonBlocked() const; + TDuration GetMaxAcquireShadowDiskRetryDelayWhenNonBlocked() const; + TDuration GetMaxAcquireShadowDiskTotalTimeoutWhenBlocked() const; + TDuration GetMaxAcquireShadowDiskTotalTimeoutWhenNonBlocked() const; }; ui64 GetAllocationUnit( diff --git a/cloud/blockstore/libs/storage/core/disk_counters.h b/cloud/blockstore/libs/storage/core/disk_counters.h index 522e0394db2..732dace809a 100644 --- a/cloud/blockstore/libs/storage/core/disk_counters.h +++ b/cloud/blockstore/libs/storage/core/disk_counters.h @@ -45,6 +45,8 @@ enum class EPublishingPolicy xxx(CompactionGarbageScore, Max, Permanent, __VA_ARGS__)\ xxx(ChannelHistorySize, Max, Permanent, __VA_ARGS__)\ xxx(CompactionRangeCountPerRun, Max, Permanent, __VA_ARGS__)\ + xxx(UnconfirmedBlobCount, Generic, Permanent, __VA_ARGS__)\ + xxx(ConfirmedBlobCount, Generic, Permanent, __VA_ARGS__)\ // BLOCKSTORE_REPL_PART_SIMPLE_COUNTERS //////////////////////////////////////////////////////////////////////////////// @@ -83,6 +85,7 @@ enum class EPublishingPolicy xxx(TrimFreshLog, __VA_ARGS__)\ xxx(AddConfirmedBlobs, __VA_ARGS__)\ xxx(AddUnconfirmedBlobs, __VA_ARGS__)\ + xxx(ConfirmBlobs, __VA_ARGS__)\ // BLOCKSTORE_REPL_PART_REQUEST_COUNTERS #define BLOCKSTORE_PART_REQUEST_COUNTERS_WITH_SIZE(xxx, ...) \ diff --git a/cloud/blockstore/libs/storage/core/request_info.h b/cloud/blockstore/libs/storage/core/request_info.h index e4545a9537c..00dd5dbecdd 100644 --- a/cloud/blockstore/libs/storage/core/request_info.h +++ b/cloud/blockstore/libs/storage/core/request_info.h @@ -2,6 +2,7 @@ #include "public.h" +#include #include #include #include @@ -48,7 +49,10 @@ struct TRequestInfo void CancelRequest(const NActors::TActorContext& ctx) { - Y_ABORT_UNLESS(CancelRoutine); + if (!CancelRoutine) { + ReportCancelRoutineIsNotSet(); + return; + }; CancelRoutine(ctx, *this); } @@ -144,7 +148,7 @@ TRequestInfoPtr CreateRequestInfo( TRequestInfo& requestInfo) { auto response = std::make_unique( - MakeError(E_REJECTED, "Tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); NCloud::Reply(ctx, requestInfo, std::move(response)); }; diff --git a/cloud/blockstore/libs/storage/disk_agent/actors/session_cache_actor.cpp b/cloud/blockstore/libs/storage/disk_agent/actors/session_cache_actor.cpp new file mode 100644 index 00000000000..1fae763dfeb --- /dev/null +++ b/cloud/blockstore/libs/storage/disk_agent/actors/session_cache_actor.cpp @@ -0,0 +1,152 @@ +#include "session_cache_actor.h" + +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include + +using namespace NActors; + +namespace NCloud::NBlockStore::NStorage::NDiskAgent { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +NProto::TError SaveSessionCache( + const TString& path, + const TVector& sessions, + TInstant deadline) +{ + try { + NProto::TDiskAgentDeviceSessionCache proto; + proto.MutableSessions()->Reserve(static_cast(sessions.size())); + + // saving only active sessions + for (const auto& session: sessions) { + if (session.GetLastActivityTs() > deadline.MicroSeconds()) { + *proto.MutableSessions()->Add() = session; + } + } + + const TString tmpPath{path + ".tmp"}; + + SerializeToTextFormat(proto, tmpPath); + + if (!NFs::Rename(tmpPath, path)) { + char buf[64] = {}; + const auto ec = errno; + + return MakeError( + MAKE_SYSTEM_ERROR(ec), + strerror_r(ec, buf, sizeof(buf))); + } + } catch (...) { + return MakeError(E_FAIL, CurrentExceptionMessage()); + } + + return {}; +} + +//////////////////////////////////////////////////////////////////////////////// + +class TSessionCacheActor: public TActorBootstrapped +{ +private: + const TString CachePath; + const TDuration ReleaseInactiveSessionsTimeout; + +public: + TSessionCacheActor( + TString cachePath, + TDuration releaseInactiveSessionsTimeout) + : CachePath{std::move(cachePath)} + , ReleaseInactiveSessionsTimeout{releaseInactiveSessionsTimeout} + { + ActivityType = TBlockStoreComponents::DISK_AGENT_WORKER; + } + + void Bootstrap(const TActorContext& ctx) + { + Become(&TThis::StateWork); + + LOG_INFO( + ctx, + TBlockStoreComponents::DISK_AGENT_WORKER, + "Session Cache Actor started"); + } + +private: + STFUNC(StateWork) + { + switch (ev->GetTypeRewrite()) { + HFunc(NActors::TEvents::TEvPoisonPill, HandlePoisonPill); + + HFunc( + TEvDiskAgentPrivate::TEvUpdateSessionCacheRequest, + HandleUpdateSessionCache); + + default: + HandleUnexpectedEvent( + ev, + TBlockStoreComponents::DISK_AGENT_WORKER); + break; + } + } + + void HandlePoisonPill( + const TEvents::TEvPoisonPill::TPtr& ev, + const TActorContext& ctx) + { + Y_UNUSED(ev); + + Die(ctx); + } + + void HandleUpdateSessionCache( + const TEvDiskAgentPrivate::TEvUpdateSessionCacheRequest::TPtr& ev, + const TActorContext& ctx) + { + LOG_INFO( + ctx, + TBlockStoreComponents::DISK_AGENT_WORKER, + "Update the session cache"); + + auto* msg = ev->Get(); + + const auto deadline = ctx.Now() - ReleaseInactiveSessionsTimeout; + Y_DEBUG_ABORT_UNLESS(deadline); + + SaveSessionCache(CachePath, msg->Sessions, deadline); + + NCloud::Reply( + ctx, + *ev, + std::make_unique< + TEvDiskAgentPrivate::TEvUpdateSessionCacheResponse>()); + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +std::unique_ptr CreateSessionCacheActor( + TString cachePath, + TDuration releaseInactiveSessionsTimeout) +{ + return std::make_unique( + std::move(cachePath), + releaseInactiveSessionsTimeout); +} + +} // namespace NCloud::NBlockStore::NStorage::NDiskAgent diff --git a/cloud/blockstore/libs/storage/disk_agent/actors/session_cache_actor.h b/cloud/blockstore/libs/storage/disk_agent/actors/session_cache_actor.h new file mode 100644 index 00000000000..63ad4e807ef --- /dev/null +++ b/cloud/blockstore/libs/storage/disk_agent/actors/session_cache_actor.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +#include + +#include + +namespace NCloud::NBlockStore::NStorage::NDiskAgent { + +//////////////////////////////////////////////////////////////////////////////// + +std::unique_ptr CreateSessionCacheActor( + TString cachePath, + TDuration releaseInactiveSessionsTimeout); + +} // namespace NCloud::NBlockStore::NStorage::NDiskAgent diff --git a/cloud/blockstore/libs/storage/disk_agent/actors/session_cache_actor_ut.cpp b/cloud/blockstore/libs/storage/disk_agent/actors/session_cache_actor_ut.cpp new file mode 100644 index 00000000000..b7eec939173 --- /dev/null +++ b/cloud/blockstore/libs/storage/disk_agent/actors/session_cache_actor_ut.cpp @@ -0,0 +1,185 @@ +#include "session_cache_actor.h" + +#include +#include + +#include + +#include +#include + +#include + +#include + +namespace NCloud::NBlockStore::NStorage::NDiskAgent { + +using namespace NActors; +using namespace std::chrono_literals; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +struct TActorSystem: NActors::TTestActorRuntimeBase +{ + void Start() + { + SetDispatchTimeout(5s); + InitNodes(); + AppendToLogSettings( + TBlockStoreComponents::START, + TBlockStoreComponents::END, + GetComponentName); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TFixture: public NUnitTest::TBaseFixture +{ + const TTempDir TempDir; + const TString CachedSessionsPath = + TempDir.Path() / "nbs-disk-agent-sessions.txt"; + const TDuration ReleaseInactiveSessionsTimeout = 10s; + + TActorSystem ActorSystem; + TActorId SessionCacheActor; + TActorId EdgeActor; + + void SetUp(NUnitTest::TTestContext& /*context*/) override + { + ActorSystem.Start(); + + EdgeActor = ActorSystem.AllocateEdgeActor(); + + SessionCacheActor = + ActorSystem.Register(CreateSessionCacheActor( + CachedSessionsPath, + ReleaseInactiveSessionsTimeout) + .release()); + + ActorSystem.DispatchEvents( + {.FinalEvents = {{TEvents::TSystem::Bootstrap}}}, + 10ms); + } + + void UpdateSessionCache(TVector sessions) + { + ActorSystem.Send( + SessionCacheActor, + EdgeActor, + std::make_unique( + std::move(sessions)) + .release()); + + auto response = ActorSystem.GrabEdgeEvent< + TEvDiskAgentPrivate::TEvUpdateSessionCacheResponse>(); + + UNIT_ASSERT(response); + UNIT_ASSERT_VALUES_EQUAL_C( + S_OK, + response->GetStatus(), + response->GetError()); + } + + auto LoadSessionCache() + { + NProto::TDiskAgentDeviceSessionCache proto; + + ParseProtoTextFromFileRobust(CachedSessionsPath, proto); + + return TVector( + std::make_move_iterator(proto.MutableSessions()->begin()), + std::make_move_iterator(proto.MutableSessions()->end())); + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(TSessionCacheActorTest) +{ + Y_UNIT_TEST_F(ShouldUpdateSessionCache, TFixture) + { + UNIT_ASSERT(!NFs::Exists(CachedSessionsPath)); + + ActorSystem.AdvanceCurrentTime(1h); + + UpdateSessionCache({}); + + UNIT_ASSERT(NFs::Exists(CachedSessionsPath)); + UNIT_ASSERT(LoadSessionCache().empty()); + + const auto updateTs1 = ActorSystem.GetCurrentTime(); + + { + NProto::TDiskAgentDeviceSession writer; + writer.SetClientId("client-1"); + writer.SetDiskId("vol0"); + writer.SetLastActivityTs( + (updateTs1 - ReleaseInactiveSessionsTimeout).MicroSeconds()); + + NProto::TDiskAgentDeviceSession reader; + reader.SetClientId("client-2"); + reader.SetReadOnly(true); + reader.SetDiskId("vol0"); + reader.SetLastActivityTs(updateTs1.MicroSeconds()); + + UpdateSessionCache({writer, reader}); + } + + { + auto sessions = LoadSessionCache(); + + // writer session was dropped because it was stale + UNIT_ASSERT_VALUES_EQUAL(1, sessions.size()); + UNIT_ASSERT_VALUES_EQUAL("client-2", sessions[0].GetClientId()); + UNIT_ASSERT_VALUES_EQUAL( + updateTs1.MicroSeconds(), + sessions[0].GetLastActivityTs()); + } + + ActorSystem.AdvanceCurrentTime(3s); + + const auto updateTs2 = ActorSystem.GetCurrentTime(); + + { + NProto::TDiskAgentDeviceSession writer; + writer.SetClientId("client-1"); + writer.SetDiskId("vol0"); + writer.SetLastActivityTs(updateTs2.MicroSeconds()); + + NProto::TDiskAgentDeviceSession reader; + reader.SetClientId("client-2"); + reader.SetReadOnly(true); + reader.SetDiskId("vol0"); + reader.SetLastActivityTs(updateTs2.MicroSeconds()); + + UpdateSessionCache({writer, reader}); + } + + { + auto sessions = LoadSessionCache(); + + UNIT_ASSERT_VALUES_EQUAL(2, sessions.size()); + SortBy(sessions, [](auto& s) { return s.GetClientId(); }); + + UNIT_ASSERT_VALUES_EQUAL("client-1", sessions[0].GetClientId()); + UNIT_ASSERT_VALUES_EQUAL( + updateTs2.MicroSeconds(), + sessions[0].GetLastActivityTs()); + + UNIT_ASSERT_VALUES_EQUAL("client-2", sessions[1].GetClientId()); + UNIT_ASSERT_VALUES_EQUAL( + updateTs2.MicroSeconds(), + sessions[1].GetLastActivityTs()); + } + + UpdateSessionCache({}); + UNIT_ASSERT(LoadSessionCache().empty()); + } +} + +} // namespace NCloud::NBlockStore::NStorage::NDiskAgent diff --git a/cloud/blockstore/libs/storage/disk_agent/actors/ut/ya.make b/cloud/blockstore/libs/storage/disk_agent/actors/ut/ya.make new file mode 100644 index 00000000000..726914ef1df --- /dev/null +++ b/cloud/blockstore/libs/storage/disk_agent/actors/ut/ya.make @@ -0,0 +1,15 @@ +UNITTEST_FOR(cloud/blockstore/libs/storage/disk_agent/actors) + +INCLUDE(${ARCADIA_ROOT}/cloud/storage/core/tests/recipes/small.inc) + +SRCS( + session_cache_actor_ut.cpp +) + +PEERDIR( + cloud/blockstore/libs/kikimr + + contrib/ydb/library/actors/testlib +) + +END() diff --git a/cloud/blockstore/libs/storage/disk_agent/actors/ya.make b/cloud/blockstore/libs/storage/disk_agent/actors/ya.make new file mode 100644 index 00000000000..eb248b53321 --- /dev/null +++ b/cloud/blockstore/libs/storage/disk_agent/actors/ya.make @@ -0,0 +1,22 @@ +LIBRARY() + +SRCS( + session_cache_actor.cpp +) + +PEERDIR( + cloud/blockstore/libs/storage/disk_agent/model + cloud/blockstore/libs/storage/protos + + cloud/storage/core/libs/actors + cloud/storage/core/protos + + contrib/ydb/library/actors/core + contrib/ydb/core/protos +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent.cpp index 30c0b9e89a6..eeac4aedc47 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent.cpp @@ -11,6 +11,7 @@ using namespace NActors; IActorPtr CreateDiskAgent( TStorageConfigPtr config, TDiskAgentConfigPtr agentConfig, + NRdma::TRdmaConfigPtr rdmaConfig, NSpdk::ISpdkEnvPtr spdk, ICachingAllocatorPtr allocator, IStorageProviderPtr storageProvider, @@ -23,6 +24,7 @@ IActorPtr CreateDiskAgent( return std::make_unique( std::move(config), std::move(agentConfig), + std::move(rdmaConfig), std::move(spdk), std::move(allocator), std::move(storageProvider), diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent.h b/cloud/blockstore/libs/storage/disk_agent/disk_agent.h index a7ea4c6aaac..efcd7360561 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent.h +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent.h @@ -17,6 +17,7 @@ namespace NCloud::NBlockStore::NStorage { NActors::IActorPtr CreateDiskAgent( TStorageConfigPtr config, TDiskAgentConfigPtr agentConfig, + NRdma::TRdmaConfigPtr rdmaConfig, NSpdk::ISpdkEnvPtr spdk, ICachingAllocatorPtr allocator, IStorageProviderPtr storageProvider, diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor.cpp index df89ee8c335..d9c1c1f7551 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor.cpp @@ -1,5 +1,8 @@ #include "disk_agent_actor.h" +#include "actors/session_cache_actor.h" + +#include #include #include #include @@ -18,6 +21,7 @@ using namespace NKikimr; TDiskAgentActor::TDiskAgentActor( TStorageConfigPtr config, TDiskAgentConfigPtr agentConfig, + NRdma::TRdmaConfigPtr rdmaConfig, NSpdk::ISpdkEnvPtr spdk, ICachingAllocatorPtr allocator, IStorageProviderPtr storageProvider, @@ -28,6 +32,7 @@ TDiskAgentActor::TDiskAgentActor( NNvme::INvmeManagerPtr nvmeManager) : Config(std::move(config)) , AgentConfig(std::move(agentConfig)) + , RdmaConfig(std::move(rdmaConfig)) , Spdk(std::move(spdk)) , Allocator(std::move(allocator)) , StorageProvider(std::move(storageProvider)) @@ -108,8 +113,58 @@ void TDiskAgentActor::UpdateActorStats() } } +void TDiskAgentActor::UpdateSessionCache(const TActorContext& ctx) +{ + if (!SessionCacheActor) { + return; + } + + NCloud::Send( + ctx, + SessionCacheActor, + 0, // cookie + State->GetSessions()); +} + +void TDiskAgentActor::RunSessionCacheActor(const TActorContext& ctx) +{ + auto path = GetCachedSessionsPath(); + if (path.empty()) { + return; + } + + auto actor = NDiskAgent::CreateSessionCacheActor( + std::move(path), + AgentConfig->GetReleaseInactiveSessionsTimeout()); + + // Starting SessionCacheActor on the IO pool to avoid file operations in the + // User pool + SessionCacheActor = ctx.Register( + actor.release(), + TMailboxType::HTSwap, + NKikimr::AppData()->IOPoolId); +} + +TString TDiskAgentActor::GetCachedSessionsPath() const +{ + const TString storagePath = Config->GetCachedDiskAgentSessionsPath(); + const TString agentPath = AgentConfig->GetCachedSessionsPath(); + return agentPath.empty() ? storagePath : agentPath; +} + //////////////////////////////////////////////////////////////////////////////// +void TDiskAgentActor::HandleReportDelayedDiskAgentConfigMismatch( + const TEvDiskAgentPrivate::TEvReportDelayedDiskAgentConfigMismatch::TPtr& + ev, + const TActorContext& ctx) +{ + Y_UNUSED(ctx); + const auto* msg = ev->Get(); + ReportDiskAgentConfigMismatch( + TStringBuilder() << "[duplicate] " << msg->ErrorText); +} + void TDiskAgentActor::HandlePoisonPill( const TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx) @@ -123,6 +178,11 @@ void TDiskAgentActor::HandlePoisonPill( StatsActor = {}; } + if (SessionCacheActor) { + NCloud::Send(ctx, SessionCacheActor); + SessionCacheActor = {}; + } + State->StopTarget(); for (auto& [uuid, pendingRequests]: SecureErasePendingRequests) { @@ -175,6 +235,10 @@ STFUNC(TDiskAgentActor::StateInit) HFunc(TEvDiskAgentPrivate::TEvInitAgentCompleted, HandleInitAgentCompleted); + HFunc( + TEvDiskAgentPrivate::TEvReportDelayedDiskAgentConfigMismatch, + HandleReportDelayedDiskAgentConfigMismatch); + BLOCKSTORE_HANDLE_REQUEST(WaitReady, TEvDiskAgent) default: @@ -203,6 +267,12 @@ STFUNC(TDiskAgentActor::StateWork) HFunc(TEvDiskAgentPrivate::TEvWriteOrZeroCompleted, HandleWriteOrZeroCompleted); + HFunc( + TEvDiskAgentPrivate::TEvReportDelayedDiskAgentConfigMismatch, + HandleReportDelayedDiskAgentConfigMismatch); + + IgnoreFunc(TEvDiskAgentPrivate::TEvUpdateSessionCacheResponse); + default: if (!HandleRequests(ev)) { HandleUnexpectedEvent(ev, TBlockStoreComponents::DISK_AGENT); @@ -211,4 +281,21 @@ STFUNC(TDiskAgentActor::StateWork) } } +STFUNC(TDiskAgentActor::StateIdle) +{ + UpdateActorStatsSampled(); + switch (ev->GetTypeRewrite()) { + HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); + HFunc(TEvents::TEvWakeup, HandleWakeup); + + HFunc(NMon::TEvHttpInfo, HandleHttpInfo); + + BLOCKSTORE_HANDLE_REQUEST(WaitReady, TEvDiskAgent) + + default: + HandleUnexpectedEvent(ev, TBlockStoreComponents::DISK_AGENT); + break; + } +} + } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor.h b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor.h index 443e5e7db64..cebee9b3b04 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor.h +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -44,6 +45,7 @@ class TDiskAgentActor final private: const TStorageConfigPtr Config; const TDiskAgentConfigPtr AgentConfig; + const NRdma::TRdmaConfigPtr RdmaConfig; const NSpdk::ISpdkEnvPtr Spdk; const ICachingAllocatorPtr Allocator; const IStorageProviderPtr StorageProvider; @@ -73,10 +75,13 @@ class TDiskAgentActor final THashMap RecentBlocksTrackers; TList PostponedRequests; + NActors::TActorId SessionCacheActor; + public: TDiskAgentActor( TStorageConfigPtr config, TDiskAgentConfigPtr agentConfig, + NRdma::TRdmaConfigPtr rdmaConfig, NSpdk::ISpdkEnvPtr spdk, ICachingAllocatorPtr allocator, IStorageProviderPtr storageProvider, @@ -132,9 +137,15 @@ class TDiskAgentActor final TRecentBlocksTracker& GetRecentBlocksTracker(const TString& deviceUUID); + TString GetCachedSessionsPath() const; + + void UpdateSessionCache(const NActors::TActorContext& ctx); + void RunSessionCacheActor(const NActors::TActorContext& ctx); + private: STFUNC(StateInit); STFUNC(StateWork); + STFUNC(StateIdle); void HandlePoisonPill( const NActors::TEvents::TEvPoisonPill::TPtr& ev, @@ -172,6 +183,11 @@ class TDiskAgentActor final const TEvDiskAgentPrivate::TEvWriteOrZeroCompleted::TPtr& ev, const NActors::TActorContext& ctx); + void HandleReportDelayedDiskAgentConfigMismatch( + const TEvDiskAgentPrivate::TEvReportDelayedDiskAgentConfigMismatch:: + TPtr& ev, + const NActors::TActorContext& ctx); + bool HandleRequests(STFUNC_SIG); BLOCKSTORE_DISK_AGENT_REQUESTS(BLOCKSTORE_IMPLEMENT_REQUEST, TEvDiskAgent) diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_acquire.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_acquire.cpp index e40855a2e7b..893702eef5d 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_acquire.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_acquire.cpp @@ -2,6 +2,8 @@ #include +#include + #include namespace NCloud::NBlockStore::NStorage { @@ -56,7 +58,7 @@ void TDiskAgentActor::HandleAcquireDevices( << ", uuids=" << JoinSeq(",", uuids) ); - State->AcquireDevices( + const bool updated = State->AcquireDevices( uuids, clientId, ctx.Now(), @@ -65,6 +67,10 @@ void TDiskAgentActor::HandleAcquireDevices( record.GetDiskId(), record.GetVolumeGeneration()); + if (updated) { + UpdateSessionCache(ctx); + } + if (!Spdk || !record.HasRateLimits()) { reply(NProto::TError()); return; diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_init.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_init.cpp index 623e5229661..312cb89e92b 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_init.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_init.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -18,7 +19,9 @@ using namespace NActors; void TDiskAgentActor::InitAgent(const TActorContext& ctx) { State = std::make_unique( + Config, AgentConfig, + RdmaConfig, Spdk, Allocator, StorageProvider, @@ -50,7 +53,8 @@ void TDiskAgentActor::InitAgent(const TActorContext& ctx) auto response = std::make_unique( std::move(r.Configs), - std::move(r.Errors)); + std::move(r.Errors), + std::move(r.ConfigMismatchErrors)); actorSystem->Send( new IEventHandle( @@ -85,6 +89,17 @@ void TDiskAgentActor::HandleInitAgentCompleted( LOG_WARN_S(ctx, TBlockStoreComponents::DISK_AGENT, error); } + // Crit events that reported on startup have issue with them being invisible + // on second restart. Here, we schedule the event to allow monitoring + // initially to read counters without event and then with the event. + for (const auto& configMismatchError: msg->ConfigMismatchErrors) { + const TDuration startupCritEventDelay = UpdateCountersInterval * 2; + ctx.Schedule( + startupCritEventDelay, + new TEvDiskAgentPrivate::TEvReportDelayedDiskAgentConfigMismatch( + configMismatchError)); + } + if (const auto& error = msg->GetError(); HasError(error)) { LOG_ERROR_S(ctx, TBlockStoreComponents::DISK_AGENT, "DiskAgent initialization failed. Error: " << FormatError(error).data()); @@ -103,10 +118,23 @@ void TDiskAgentActor::HandleInitAgentCompleted( // resend pending requests SendPendingRequests(ctx, PendingRequests); + if (msg->Configs.empty()) { + LOG_INFO( + ctx, + TBlockStoreComponents::DISK_AGENT, + "No devices: become idle"); + + Become(&TThis::StateIdle); + + return; + } + Become(&TThis::StateWork); SendRegisterRequest(ctx); ScheduleUpdateStats(ctx); + + RunSessionCacheActor(ctx); } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_io.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_io.cpp index 5284a12497e..3e846e6723e 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_io.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_io.cpp @@ -22,6 +22,10 @@ constexpr bool IsWriteDeviceMethod = std::is_same_v || std::is_same_v; +template +constexpr bool IsReadDeviceMethod = + std::is_same_v; + //////////////////////////////////////////////////////////////////////////////// template @@ -117,6 +121,10 @@ void TDiskAgentActor::PerformIO( if constexpr (IsWriteDeviceMethod) { volumeRequestId = GetVolumeRequestId(*msg); range = BuildRequestBlockRange(*msg); + } else { + range = TBlockRange64::WithLength( + msg->Record.GetStartIndex(), + msg->Record.GetBlocksCount()); } auto requestInfo = CreateRequestInfo( @@ -183,7 +191,18 @@ void TDiskAgentActor::PerformIO( clientId.c_str()); if (SecureErasePendingRequests.contains(deviceUUID)) { - ReportDiskAgentIoDuringSecureErase(); + const bool isHealthCheckRead = + IsReadDeviceMethod && clientId == CheckHealthClientId; + if (!isHealthCheckRead) { + ReportDiskAgentIoDuringSecureErase( + TStringBuilder() + << " Device=" << deviceUUID + << ", ClientId=" << clientId + << ", StartIndex=" << range.Start + << ", BlocksCount=" << range.Size() + << ", IsWrite=" << IsWriteDeviceMethod + << ", IsRdma=0"); + } replyError(E_REJECTED, "Secure erase in progress"); return; } diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_monitoring.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_monitoring.cpp index db2aaac108f..e5ef2293aa3 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_monitoring.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_monitoring.cpp @@ -1,5 +1,6 @@ #include "disk_agent_actor.h" +#include #include #include @@ -28,10 +29,14 @@ void TDiskAgentActor::HandleHttpInfo( TStringStream out; HTML(out) { - if (RegistrationInProgress) { - DIV() { out << "Registration in progress"; } + if (CurrentStateFunc() == &TThis::StateIdle) { + DIV() { out << "Unregistered (Idle)"; } } else { - DIV() { out << "Registered"; } + if (RegistrationInProgress) { + DIV() { out << "Registration in progress"; } + } else { + DIV() { out << "Registered"; } + } } TAG(TH3) { out << "Devices"; } @@ -39,6 +44,11 @@ void TDiskAgentActor::HandleHttpInfo( TAG(TH3) { out << "Config"; } AgentConfig->DumpHtml(out); + + if (RdmaServer) { + TAG(TH3) { out << "RdmaServer"; } + RdmaServer->DumpHtml(out); + } } NCloud::Reply( @@ -80,8 +90,9 @@ void TDiskAgentActor::RenderDevices(IOutputStream& out) const DumpDeviceState( out, config.GetState(), - false, - State->IsDeviceDisabled(uuid)); + State->IsDeviceDisabled(uuid) + ? EDeviceStateFlags::DISABLED + : EDeviceStateFlags::NONE); } TABLED() { if (config.GetStateTs()) { diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_register.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_register.cpp index 5e8a5c115b4..f41df2adb94 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_register.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_register.cpp @@ -182,11 +182,6 @@ void TDiskAgentActor::SendRegisterRequest(const TActorContext& ctx) return; } - if (State->GetDevicesCount() == 0) { - LOG_WARN(ctx, TBlockStoreComponents::DISK_AGENT, "No devices to register"); - return; - } - RegistrationInProgress = true; NCloud::Send( ctx, diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_release.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_release.cpp index ca638aafcb5..957af08a56b 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_release.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_release.cpp @@ -1,5 +1,7 @@ #include "disk_agent_actor.h" +#include + namespace NCloud::NBlockStore::NStorage { using namespace NActors; @@ -25,6 +27,9 @@ void TDiskAgentActor::HandleReleaseDevices( record.GetHeaders().GetClientId(), record.GetDiskId(), record.GetVolumeGeneration()); + + UpdateSessionCache(ctx); + } catch (const TServiceError& e) { *response->Record.MutableError() = MakeError(e.GetCode(), e.what()); } diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_ut.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_ut.cpp index e12fd747dbf..ccebe24a772 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_ut.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_ut.cpp @@ -7,14 +7,21 @@ #include #include #include +#include + +#include #include #include #include #include +#include + #include +#include + namespace NCloud::NBlockStore::NStorage { using namespace NActors; @@ -24,6 +31,8 @@ using namespace NThreading; using namespace NDiskAgentTest; +using namespace std::chrono_literals; + namespace { //////////////////////////////////////////////////////////////////////////////// @@ -49,6 +58,174 @@ TFsPath TryGetRamDrivePath() : p; } +//////////////////////////////////////////////////////////////////////////////// + +struct THttpGetRequest: NMonitoring::IHttpRequest +{ + TCgiParameters CgiParameters; + THttpHeaders HttpHeaders; + + const char* GetURI() const override + { + return ""; + } + + const char* GetPath() const override + { + return ""; + } + + const TCgiParameters& GetParams() const override + { + return CgiParameters; + } + + const TCgiParameters& GetPostParams() const override + { + return CgiParameters; + } + + TStringBuf GetPostContent() const override + { + return {}; + } + + HTTP_METHOD GetMethod() const override + { + return HTTP_METHOD_GET; + } + + const THttpHeaders& GetHeaders() const override + { + return HttpHeaders; + } + + TString GetRemoteAddr() const override + { + return {}; + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TFixture + : public NUnitTest::TBaseFixture +{ + const TTempDir TempDir; + const TString CachedSessionsPath = + TempDir.Path() / "nbs-disk-agent-sessions.txt"; + const TDuration ReleaseInactiveSessionsTimeout = 10s; + + TTestBasicRuntime Runtime; + NMonitoring::TDynamicCountersPtr Counters = MakeIntrusive(); + + TVector Files; + + const ui64 DefaultFileSize = DefaultDeviceBlockSize * DefaultBlocksCount; + + void PrepareFile(const TString& name, size_t size) + { + TFile fileData(TempDir.Path() / name, EOpenModeFlag::CreateNew); + fileData.Resize(size); + + Files.push_back(fileData.GetName()); + } + + auto CreateDiskAgentConfig() + { + auto config = DiskAgentConfig(); + + auto& discovery = *config.MutableStorageDiscoveryConfig(); + auto& path = *discovery.AddPathConfigs(); + path.SetPathRegExp(TempDir.Path() / "NVMENBS([0-9]{2})"); + auto& pool = *path.AddPoolConfigs(); + pool.SetMaxSize(DefaultFileSize); + + config.SetCachedSessionsPath(CachedSessionsPath); + config.SetBackend(NProto::DISK_AGENT_BACKEND_AIO); + config.SetAcquireRequired(true); + config.SetReleaseInactiveSessionsTimeout( + ReleaseInactiveSessionsTimeout.MilliSeconds()); + + return config; + } + + NProto::TError Write( + TDiskAgentClient& diskAgent, + const TString& clientId, + const TString& deviceId) + { + auto request = std::make_unique(); + request->Record.MutableHeaders()->SetClientId(clientId); + request->Record.SetDeviceUUID(deviceId); + request->Record.SetStartIndex(1); + request->Record.SetBlockSize(DefaultBlockSize); + + auto sgList = ResizeIOVector(*request->Record.MutableBlocks(), 10, 4_KB); + + for (auto& buffer: sgList) { + memset(const_cast(buffer.Data()), 'Y', buffer.Size()); + } + + diskAgent.SendRequest(MakeDiskAgentServiceId(), std::move(request)); + Runtime.DispatchEvents({}); + auto response = diskAgent.RecvWriteDeviceBlocksResponse(); + + return response->Record.GetError(); + } + + NProto::TError Read( + TDiskAgentClient& diskAgent, + const TString& clientId, + const TString& deviceId) + { + const auto response = ReadDeviceBlocks( + Runtime, diskAgent, deviceId, 1, 10, clientId); + + return response->Record.GetError(); + } + + auto LoadSessionCache() + { + NProto::TDiskAgentDeviceSessionCache cache; + ParseProtoTextFromFileRobust(CachedSessionsPath, cache); + + TVector sessions( + std::make_move_iterator(cache.MutableSessions()->begin()), + std::make_move_iterator(cache.MutableSessions()->end()) + ); + + SortBy(sessions, [] (auto& session) { + return session.GetClientId(); + }); + + for (auto& session: sessions) { + Sort(*session.MutableDeviceIds()); + } + + return sessions; + } + + void SetUp(NUnitTest::TTestContext& /*context*/) override + { + InitCriticalEventsCounter(Counters); + + PrepareFile("NVMENBS01", DefaultFileSize); + PrepareFile("NVMENBS02", DefaultFileSize); + PrepareFile("NVMENBS03", DefaultFileSize); + PrepareFile("NVMENBS04", DefaultFileSize); + } + + void TearDown(NUnitTest::TTestContext& /*context*/) override + { + for (const auto& path: Files) { + path.DeleteIfExists(); + } + } +}; + +//////////////////////////////////////////////////////////////////////////////// + } // namespace //////////////////////////////////////////////////////////////////////////////// @@ -200,9 +377,10 @@ Y_UNIT_TEST_SUITE(TDiskAgentTest) uuids, "client-1", NProto::VOLUME_ACCESS_READ_WRITE, - 0, - limits - ); + 0, // mountSeqNumber + "", // diskId + 0, // volumeGeneration + limits); diskAgent.ReleaseDevices(uuids, "client-1"); } @@ -2994,6 +3172,36 @@ Y_UNIT_TEST_SUITE(TDiskAgentTest) true); UNIT_ASSERT_VALUES_EQUAL(counter->Val(), 3); + // Read during secure erase from HealthCheck client doesn't generate + // critical event + UNIT_ASSERT_VALUES_EQUAL( + E_REJECTED, + ReadDeviceBlocks( + runtime, + diskAgent, + "uuid", + 0, + 1, + TString(CheckHealthClientId)) + ->Record.GetError() + .GetCode()); + UNIT_ASSERT_VALUES_EQUAL(counter->Val(), 3); + + // Write zero during secure erase from HealthCheck client generates + // critical event + UNIT_ASSERT_VALUES_EQUAL( + E_REJECTED, + ZeroDeviceBlocks( + runtime, + diskAgent, + "uuid", + 0, + 1, + TString(CheckHealthClientId)) + ->Record.GetError() + .GetCode()); + UNIT_ASSERT_VALUES_EQUAL(counter->Val(), 4); + spdk->SecureEraseResult.SetValue(NProto::TError()); runtime.DispatchEvents(TDispatchOptions(), TDuration::MilliSeconds(10)); @@ -3575,6 +3783,448 @@ Y_UNIT_TEST_SUITE(TDiskAgentTest) UNIT_ASSERT_VALUES_EQUAL(0, env.DiskRegistryState->Devices.size()); UNIT_ASSERT_VALUES_EQUAL(1, mismatch->Val()); } + + Y_UNIT_TEST_F(ShouldCacheSessions, TFixture) + { + auto cacheRestoreError = Counters->GetCounter( + "AppCriticalEvents/DiskAgentSessionCacheRestoreError", + true); + + UNIT_ASSERT_EQUAL(0, *cacheRestoreError); + + NProto::TAgentConfig agentConfig; + + Runtime.SetEventFilter( + [&](auto&, TAutoPtr& event) + { + if (event->GetTypeRewrite() == + TEvDiskRegistry::EvRegisterAgentRequest) + { + auto& msg = + *event->Get(); + + agentConfig = msg.Record.GetAgentConfig(); + } + + return false; + }); + + auto env = + TTestEnvBuilder(Runtime).With(CreateDiskAgentConfig()).Build(); + + Runtime.AdvanceCurrentTime(1h); + + TDiskAgentClient diskAgent(Runtime); + diskAgent.WaitReady(); + + UNIT_ASSERT_EQUAL(0, *cacheRestoreError); + + UNIT_ASSERT_VALUES_EQUAL(4, agentConfig.DevicesSize()); + + TVector devices; + for (const auto& d: agentConfig.GetDevices()) { + devices.push_back(d.GetDeviceUUID()); + } + Sort(devices); + + // acquire a bunch of devices + + const auto acquireTs = Runtime.GetCurrentTime(); + + diskAgent.AcquireDevices( + TVector{devices[0], devices[1]}, + "writer-1", + NProto::VOLUME_ACCESS_READ_WRITE, + 42, // MountSeqNumber + "vol0", + 1000); // VolumeGeneration + + diskAgent.AcquireDevices( + TVector{devices[0], devices[1]}, + "reader-1", + NProto::VOLUME_ACCESS_READ_ONLY, + -1, // MountSeqNumber + "vol0", + 1001); // VolumeGeneration + + diskAgent.AcquireDevices( + TVector{devices[2], devices[3]}, + "reader-2", + NProto::VOLUME_ACCESS_READ_ONLY, + -1, // MountSeqNumber + "vol1", + 2000); // VolumeGeneration + + // validate the cache file + + { + auto sessions = LoadSessionCache(); + + UNIT_ASSERT_VALUES_EQUAL(3, sessions.size()); + + { + auto& session = sessions[0]; + + UNIT_ASSERT_VALUES_EQUAL("reader-1", session.GetClientId()); + UNIT_ASSERT_VALUES_EQUAL(2, session.DeviceIdsSize()); + ASSERT_VECTORS_EQUAL( + TVector({devices[0], devices[1]}), + session.GetDeviceIds()); + + // MountSeqNumber is not applicable to read sessions + UNIT_ASSERT_VALUES_EQUAL(0, session.GetMountSeqNumber()); + UNIT_ASSERT_VALUES_EQUAL("vol0", session.GetDiskId()); + UNIT_ASSERT_VALUES_EQUAL(1001, session.GetVolumeGeneration()); + UNIT_ASSERT_VALUES_EQUAL( + acquireTs.MicroSeconds(), + session.GetLastActivityTs()); + UNIT_ASSERT(session.GetReadOnly()); + } + + { + auto& session = sessions[1]; + + UNIT_ASSERT_VALUES_EQUAL("reader-2", session.GetClientId()); + UNIT_ASSERT_VALUES_EQUAL(2, session.DeviceIdsSize()); + ASSERT_VECTORS_EQUAL( + TVector({devices[2], devices[3]}), + session.GetDeviceIds()); + + // MountSeqNumber is not applicable to read sessions + UNIT_ASSERT_VALUES_EQUAL(0, session.GetMountSeqNumber()); + UNIT_ASSERT_VALUES_EQUAL("vol1", session.GetDiskId()); + UNIT_ASSERT_VALUES_EQUAL(2000, session.GetVolumeGeneration()); + UNIT_ASSERT_VALUES_EQUAL( + acquireTs.MicroSeconds(), + session.GetLastActivityTs()); + UNIT_ASSERT(session.GetReadOnly()); + } + + { + auto& session = sessions[2]; + + UNIT_ASSERT_VALUES_EQUAL("writer-1", session.GetClientId()); + UNIT_ASSERT_VALUES_EQUAL(2, session.DeviceIdsSize()); + ASSERT_VECTORS_EQUAL( + TVector({devices[0], devices[1]}), + session.GetDeviceIds()); + + UNIT_ASSERT_VALUES_EQUAL(42, session.GetMountSeqNumber()); + UNIT_ASSERT_VALUES_EQUAL("vol0", session.GetDiskId()); + + // VolumeGeneration was updated by reader-1 + UNIT_ASSERT_VALUES_EQUAL(1001, session.GetVolumeGeneration()); + UNIT_ASSERT_VALUES_EQUAL( + acquireTs.MicroSeconds(), + session.GetLastActivityTs()); + UNIT_ASSERT(!session.GetReadOnly()); + } + } + + Runtime.AdvanceCurrentTime(5s); + + diskAgent.ReleaseDevices( + TVector{devices[0], devices[1]}, + "writer-1", + "vol0", + 1001); + + diskAgent.ReleaseDevices( + TVector{devices[2], devices[3]}, + "reader-2", + "vol1", + 2000); + + // check the session cache again + { + auto sessions = LoadSessionCache(); + + // now we have only one session + UNIT_ASSERT_VALUES_EQUAL(1, sessions.size()); + + auto& session = sessions[0]; + + UNIT_ASSERT_VALUES_EQUAL("reader-1", session.GetClientId()); + UNIT_ASSERT_VALUES_EQUAL(2, session.DeviceIdsSize()); + ASSERT_VECTORS_EQUAL( + TVector({devices[0], devices[1]}), + session.GetDeviceIds()); + + // MountSeqNumber is not applicable to read sessions + UNIT_ASSERT_VALUES_EQUAL(0, session.GetMountSeqNumber()); + UNIT_ASSERT_VALUES_EQUAL("vol0", session.GetDiskId()); + UNIT_ASSERT_VALUES_EQUAL(1001, session.GetVolumeGeneration()); + UNIT_ASSERT_VALUES_EQUAL( + acquireTs.MicroSeconds(), + session.GetLastActivityTs()); + UNIT_ASSERT(session.GetReadOnly()); + } + } + + Y_UNIT_TEST_F(ShouldRestoreCacheSessions, TFixture) + { + const TString devices[] { + "5ea2fcdce0a180a63db2b5f6a5b34221", + "657dabaf3d224c9177b00c437716dfb1", + "79955ae90189fe8a89ab832a8b0cb57d", + "e85cd1d217c3239507fc0cd180a075fd" + }; + + const auto initialTs = Now(); + + { + NProto::TDiskAgentDeviceSessionCache cache; + auto& writeSession = *cache.AddSessions(); + writeSession.SetClientId("writer-1"); + *writeSession.MutableDeviceIds()->Add() = devices[0]; + *writeSession.MutableDeviceIds()->Add() = devices[1]; + writeSession.SetReadOnly(false); + writeSession.SetMountSeqNumber(42); + writeSession.SetDiskId("vol0"); + writeSession.SetVolumeGeneration(1000); + writeSession.SetLastActivityTs(initialTs.MicroSeconds()); + + auto& reader1 = *cache.AddSessions(); + reader1.SetClientId("reader-1"); + *reader1.MutableDeviceIds()->Add() = devices[0]; + *reader1.MutableDeviceIds()->Add() = devices[1]; + reader1.SetReadOnly(true); + reader1.SetDiskId("vol0"); + reader1.SetVolumeGeneration(1000); + reader1.SetLastActivityTs(initialTs.MicroSeconds()); + + auto& reader2 = *cache.AddSessions(); + reader2.SetClientId("reader-2"); + *reader2.MutableDeviceIds()->Add() = devices[2]; + *reader2.MutableDeviceIds()->Add() = devices[3]; + reader2.SetReadOnly(true); + reader2.SetDiskId("vol1"); + reader2.SetVolumeGeneration(2000); + reader2.SetLastActivityTs(initialTs.MicroSeconds()); + + SerializeToTextFormat(cache, CachedSessionsPath); + } + + auto cacheRestoreError = Counters->GetCounter( + "AppCriticalEvents/DiskAgentSessionCacheRestoreError", + true); + + UNIT_ASSERT_EQUAL(0, *cacheRestoreError); + + auto env = TTestEnvBuilder(Runtime) + .With(CreateDiskAgentConfig()) + .Build(); + + Runtime.UpdateCurrentTime(initialTs + 3s); + + TDiskAgentClient diskAgent(Runtime); + diskAgent.WaitReady(); + + UNIT_ASSERT_EQUAL(0, *cacheRestoreError); + + // should reject a request with a wrong client id + { + auto error = Write(diskAgent, "unknown", devices[1]); + UNIT_ASSERT_EQUAL_C(E_BS_INVALID_SESSION, error.GetCode(), error); + } + + // should reject a request with a wrong client id + { + auto error = Write(diskAgent, "reader-2", devices[0]); + UNIT_ASSERT_EQUAL_C(E_BS_INVALID_SESSION, error.GetCode(), error); + } + + { + auto error = Write(diskAgent, "reader-2", devices[1]); + UNIT_ASSERT_EQUAL_C(E_BS_INVALID_SESSION, error.GetCode(), error); + } + + // should reject a write request for the read only session + { + auto error = Write(diskAgent, "reader-1", devices[1]); + UNIT_ASSERT_EQUAL_C(E_BS_INVALID_SESSION, error.GetCode(), error); + } + + // should be ok + + { + auto error = Write(diskAgent, "writer-1", devices[0]); + UNIT_ASSERT_EQUAL_C(S_OK, error.GetCode(), error); + } + + { + auto error = Write(diskAgent, "writer-1", devices[1]); + UNIT_ASSERT_EQUAL_C(S_OK, error.GetCode(), error); + } + + { + auto error = Read(diskAgent, "reader-1", devices[0]); + UNIT_ASSERT_EQUAL_C(S_OK, error.GetCode(), error); + } + + { + auto error = Read(diskAgent, "reader-1", devices[1]); + UNIT_ASSERT_EQUAL_C(S_OK, error.GetCode(), error); + } + + { + auto error = Read(diskAgent, "reader-2", devices[2]); + UNIT_ASSERT_EQUAL_C(S_OK, error.GetCode(), error); + } + + { + auto error = Read(diskAgent, "reader-2", devices[3]); + UNIT_ASSERT_EQUAL_C(S_OK, error.GetCode(), error); + } + + // make all sessions stale + Runtime.AdvanceCurrentTime(ReleaseInactiveSessionsTimeout); + + const auto acquireTs = Runtime.GetCurrentTime(); + + diskAgent.AcquireDevices( + TVector{devices[0], devices[1]}, + "writer-1", + NProto::VOLUME_ACCESS_READ_WRITE, + 42, // MountSeqNumber + "vol0", + 1000); // VolumeGeneration + + { + auto sessions = LoadSessionCache(); + + // now we have only one session + UNIT_ASSERT_VALUES_EQUAL(1, sessions.size()); + + auto& session = sessions[0]; + + UNIT_ASSERT_VALUES_EQUAL("writer-1", session.GetClientId()); + UNIT_ASSERT_VALUES_EQUAL(2, session.DeviceIdsSize()); + ASSERT_VECTORS_EQUAL( + TVector({devices[0], devices[1]}), + session.GetDeviceIds()); + + UNIT_ASSERT_VALUES_EQUAL(42, session.GetMountSeqNumber()); + UNIT_ASSERT_VALUES_EQUAL("vol0", session.GetDiskId()); + UNIT_ASSERT_VALUES_EQUAL(1000, session.GetVolumeGeneration()); + UNIT_ASSERT_VALUES_EQUAL( + acquireTs.MicroSeconds(), + session.GetLastActivityTs()); + } + } + + Y_UNIT_TEST_F(ShouldHandleBrokenCacheSessions, TFixture) + { + { + TFile file {CachedSessionsPath, EOpenModeFlag::CreateAlways}; + TFileOutput(file).Write("{ broken protobuf#"); + } + + auto cacheRestoreError = Counters->GetCounter( + "AppCriticalEvents/DiskAgentSessionCacheRestoreError", + true); + + UNIT_ASSERT_EQUAL(0, *cacheRestoreError); + + auto env = TTestEnvBuilder(Runtime) + .With(CreateDiskAgentConfig()) + .Build(); + + TDiskAgentClient diskAgent(Runtime); + diskAgent.WaitReady(); + + UNIT_ASSERT_EQUAL(1, *cacheRestoreError); + } + + Y_UNIT_TEST_F(ShouldGetCacheSessionsPathFromStorageConfig, TFixture) + { + { + TFile file {CachedSessionsPath, EOpenModeFlag::CreateAlways}; + TFileOutput(file).Write("{ broken protobuf#"); + } + + auto cacheRestoreError = Counters->GetCounter( + "AppCriticalEvents/DiskAgentSessionCacheRestoreError", + true); + + UNIT_ASSERT_EQUAL(0, *cacheRestoreError); + + auto diskAgentConfig = CreateDiskAgentConfig(); + auto storageConfig = NProto::TStorageServiceConfig(); + storageConfig.SetCachedDiskAgentSessionsPath( + diskAgentConfig.GetCachedSessionsPath()); + diskAgentConfig.ClearCachedSessionsPath(); + + auto env = TTestEnvBuilder(Runtime) + .With(diskAgentConfig) + .With(storageConfig) + .Build(); + + TDiskAgentClient diskAgent(Runtime); + diskAgent.WaitReady(); + + UNIT_ASSERT_EQUAL(1, *cacheRestoreError); + } + + Y_UNIT_TEST_F(ShouldNotGetCacheSessionsPathFromStorageConfig, TFixture) + { + const TString diskAgentCachedSessionsPath = CachedSessionsPath; + const TString storageCachedSessionsPath = + TempDir.Path() / "must-not-use-sessions.txt"; + + { + TFile file {storageCachedSessionsPath, EOpenModeFlag::CreateAlways}; + TFileOutput(file).Write("{ broken protobuf#"); + } + + auto cacheRestoreError = Counters->GetCounter( + "AppCriticalEvents/DiskAgentSessionCacheRestoreError", + true); + + UNIT_ASSERT_EQUAL(0, *cacheRestoreError); + + auto storageConfig = NProto::TStorageServiceConfig(); + storageConfig.SetCachedDiskAgentSessionsPath(storageCachedSessionsPath); + auto env = TTestEnvBuilder(Runtime) + .With(CreateDiskAgentConfig()) + .With(storageConfig) + .Build(); + + TDiskAgentClient diskAgent(Runtime); + diskAgent.WaitReady(); + + UNIT_ASSERT_EQUAL(0, *cacheRestoreError); + } + + Y_UNIT_TEST_F(ShouldSwitchToIdleModeWithoutDevices, TFixture) + { + auto config = CreateDiskAgentConfig(); + + config.MutableStorageDiscoveryConfig() + ->MutablePathConfigs(0) + ->SetPathRegExp(TempDir.Path() / "non-existent([0-9]{2})"); + + auto env = TTestEnvBuilder(Runtime).With(std::move(config)).Build(); + + TDiskAgentClient diskAgent(Runtime); + diskAgent.WaitReady(); + + THttpGetRequest httpRequest; + NMonitoring::TMonService2HttpRequest monService2HttpRequest{ + nullptr, + &httpRequest, + nullptr, + nullptr, + "", + nullptr}; + + diskAgent.SendRequest( + MakeDiskAgentServiceId(), + std::make_unique(monService2HttpRequest)); + + auto response = diskAgent.RecvResponse(); + UNIT_ASSERT_STRING_CONTAINS(response->Answer, "Unregistered (Idle)"); + } } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_private.h b/cloud/blockstore/libs/storage/disk_agent/disk_agent_private.h index 99096226461..4d8058d5a4a 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_private.h +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_private.h @@ -4,6 +4,7 @@ #include "storage_with_stats.h" +#include #include #include #include @@ -35,14 +36,17 @@ struct TEvDiskAgentPrivate { TVector Configs; TVector Errors; + TVector ConfigMismatchErrors; TInitAgentCompleted() = default; TInitAgentCompleted( TVector configs, - TVector errors) + TVector errors, + TVector configMismatchErrors) : Configs(std::move(configs)) , Errors(std::move(errors)) + , ConfigMismatchErrors(std::move(configMismatchErrors)) {} }; @@ -113,6 +117,37 @@ struct TEvDiskAgentPrivate {} }; + // + // TReportDelayedDiskAgentConfigMismatch + // + + struct TReportDelayedDiskAgentConfigMismatch + { + TString ErrorText; + + explicit TReportDelayedDiskAgentConfigMismatch(TString errorText) + : ErrorText(std::move(errorText)) + {} + }; + + // + // UpdateSessionCache + // + + struct TUpdateSessionCacheRequest + { + TVector Sessions; + + TUpdateSessionCacheRequest() = default; + explicit TUpdateSessionCacheRequest( + TVector sessions) + : Sessions(std::move(sessions)) + {} + }; + + struct TUpdateSessionCacheResponse + {}; + // // Events declaration // @@ -126,6 +161,9 @@ struct TEvDiskAgentPrivate EvInitAgentCompleted, EvSecureEraseCompleted, EvWriteOrZeroCompleted, + EvReportDelayedDiskAgentConfigMismatch, + + BLOCKSTORE_DECLARE_EVENT_IDS(UpdateSessionCache) EvEnd }; @@ -146,6 +184,12 @@ struct TEvDiskAgentPrivate using TEvWriteOrZeroCompleted = TResponseEvent< TWriteOrZeroCompleted, EvWriteOrZeroCompleted>; + + using TEvReportDelayedDiskAgentConfigMismatch = TResponseEvent< + TReportDelayedDiskAgentConfigMismatch, + EvReportDelayedDiskAgentConfigMismatch>; + + BLOCKSTORE_DECLARE_EVENTS(UpdateSessionCache) }; } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.cpp index 6fdfaef472c..dfa46c5852c 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.cpp @@ -10,10 +10,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include @@ -25,7 +27,6 @@ #include #include -#include #include #include @@ -240,7 +241,9 @@ TVector ComputeDigest( //////////////////////////////////////////////////////////////////////////////// TDiskAgentState::TDiskAgentState( + TStorageConfigPtr storageConfig, TDiskAgentConfigPtr agentConfig, + NRdma::TRdmaConfigPtr rdmaConfig, NSpdk::ISpdkEnvPtr spdk, ICachingAllocatorPtr allocator, IStorageProviderPtr storageProvider, @@ -249,7 +252,9 @@ TDiskAgentState::TDiskAgentState( ILoggingServicePtr logging, NRdma::IServerPtr rdmaServer, NNvme::INvmeManagerPtr nvmeManager) - : AgentConfig(std::move(agentConfig)) + : StorageConfig(std::move(storageConfig)) + , AgentConfig(std::move(agentConfig)) + , RdmaConfig(std::move(rdmaConfig)) , Spdk(std::move(spdk)) , Allocator(std::move(allocator)) , StorageProvider(std::move(storageProvider)) @@ -374,6 +379,7 @@ TFuture TDiskAgentState::InitAioStorage() { return InitializeStorage( Logging->CreateLog("BLOCKSTORE_DISK_AGENT"), + StorageConfig, AgentConfig, StorageProvider, NvmeManager) @@ -413,6 +419,7 @@ TFuture TDiskAgentState::InitAioStorage() return TInitializeResult { .Configs = std::move(r.Configs), .Errors = std::move(r.Errors), + .ConfigMismatchErrors = std::move(r.ConfigMismatchErrors), .Guard = std::move(r.Guard) }; }); @@ -423,7 +430,7 @@ void TDiskAgentState::InitRdmaTarget(TRdmaTargetConfig rdmaTargetConfig) if (RdmaServer) { THashMap devices; - auto endpoint = AgentConfig->GetRdmaTarget().GetEndpoint(); + auto endpoint = AgentConfig->GetRdmaEndpoint(); if (endpoint.GetHost().empty()) { endpoint.SetHost(FQDNHostName()); @@ -756,7 +763,7 @@ TVector TDiskAgentState::GetReaderSessions( return DeviceClient->GetReaderSessions(uuid); } -void TDiskAgentState::AcquireDevices( +bool TDiskAgentState::AcquireDevices( const TVector& uuids, const TString& clientId, TInstant now, @@ -776,9 +783,7 @@ void TDiskAgentState::AcquireDevices( CheckError(error); - if (updated) { - UpdateSessionCache(*DeviceClient); - } + return updated; } void TDiskAgentState::ReleaseDevices( @@ -792,8 +797,6 @@ void TDiskAgentState::ReleaseDevices( clientId, diskId, volumeGeneration)); - - UpdateSessionCache(*DeviceClient); } void TDiskAgentState::DisableDevice(const TString& uuid) @@ -829,40 +832,11 @@ void TDiskAgentState::StopTarget() } } -void TDiskAgentState::UpdateSessionCache(TDeviceClient& client) const -{ - const auto path = AgentConfig->GetCachedSessionsPath(); - - if (path.empty()) { - return; - } - - try { - auto sessions = client.GetSessions(); - - NProto::TDiskAgentDeviceSessionCache proto; - proto.MutableSessions()->Assign( - std::make_move_iterator(sessions.begin()), - std::make_move_iterator(sessions.end()) - ); - - const TString tmpPath {path + ".tmp"}; - - SerializeToTextFormat(proto, tmpPath); - - if (!NFs::Rename(tmpPath, path)) { - const auto ec = errno; - ythrow TServiceError {MAKE_SYSTEM_ERROR(ec)} << strerror(ec); - } - } catch (...) { - STORAGE_ERROR("Can't update session cache: " << CurrentExceptionMessage()); - ReportDiskAgentSessionCacheUpdateError(); - } -} - void TDiskAgentState::RestoreSessions(TDeviceClient& client) const { - const auto path = AgentConfig->GetCachedSessionsPath(); + const TString storagePath = StorageConfig->GetCachedDiskAgentSessionsPath(); + const TString agentPath = AgentConfig->GetCachedSessionsPath(); + const TString& path = agentPath.empty() ? storagePath : agentPath; if (path.empty()) { STORAGE_INFO("Session cache is not configured."); @@ -895,9 +869,8 @@ void TDiskAgentState::RestoreSessions(TDeviceClient& client) const uuids, session.GetClientId(), TInstant::MicroSeconds(session.GetLastActivityTs()), - session.GetReadOnly() - ? NProto::VOLUME_ACCESS_READ_ONLY - : NProto::VOLUME_ACCESS_READ_WRITE, + session.GetReadOnly() ? NProto::VOLUME_ACCESS_READ_ONLY + : NProto::VOLUME_ACCESS_READ_WRITE, session.GetMountSeqNumber(), session.GetDiskId(), session.GetVolumeGeneration()); @@ -928,4 +901,9 @@ void TDiskAgentState::RestoreSessions(TDeviceClient& client) const } } +TVector TDiskAgentState::GetSessions() const +{ + return DeviceClient->GetSessions(); +} + } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.h b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.h index a46c8c91456..79cd7a3e496 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.h +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state.h @@ -5,6 +5,8 @@ #include "rdma_target.h" #include "storage_with_stats.h" +#include + #include #include #include @@ -40,7 +42,9 @@ class TDiskAgentState }; private: + const TStorageConfigPtr StorageConfig; const TDiskAgentConfigPtr AgentConfig; + const NRdma::TRdmaConfigPtr RdmaConfig; const NSpdk::ISpdkEnvPtr Spdk; const ICachingAllocatorPtr Allocator; const IStorageProviderPtr StorageProvider; @@ -61,7 +65,9 @@ class TDiskAgentState public: TDiskAgentState( + TStorageConfigPtr storageConfig, TDiskAgentConfigPtr agentConfig, + NRdma::TRdmaConfigPtr rdmaConfig, NSpdk::ISpdkEnvPtr spdk, ICachingAllocatorPtr allocator, IStorageProviderPtr storageProvider, @@ -75,6 +81,7 @@ class TDiskAgentState { TVector Configs; TVector Errors; + TVector ConfigMismatchErrors; TDeviceGuard Guard; }; @@ -113,7 +120,9 @@ class TDiskAgentState TVector GetReaderSessions( const TString& uuid) const; - void AcquireDevices( + // @return `true` if any session has been updated (excluding `LastActivityTs` + // field) or a new one has been added. + bool AcquireDevices( const TVector& uuids, const TString& clientId, TInstant now, @@ -128,6 +137,8 @@ class TDiskAgentState const TString& diskId, ui32 volumeGeneration); + TVector GetSessions() const; + void DisableDevice(const TString& uuid); void EnableDevice(const TString& uuid); bool IsDeviceDisabled(const TString& uuid) const; @@ -160,7 +171,6 @@ class TDiskAgentState void InitRdmaTarget(TRdmaTargetConfig rdmaTargetConfig); - void UpdateSessionCache(TDeviceClient& client) const; void RestoreSessions(TDeviceClient& client) const; }; diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state_ut.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state_ut.cpp index 7ea8ec0ff4d..4c6cfcdb368 100644 --- a/cloud/blockstore/libs/storage/disk_agent/disk_agent_state_ut.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/disk_agent_state_ut.cpp @@ -139,13 +139,23 @@ auto CreateNullConfig(TNullConfigParams params) return std::make_shared(std::move(config), "rack"); } +TStorageConfigPtr CreateStorageConfig() +{ + NProto::TStorageServiceConfig config; + return std::make_shared( + std::move(config), + std::make_shared()); +} + auto CreateDiskAgentStateSpdk(TDiskAgentConfigPtr config) { return std::make_unique( + CreateStorageConfig(), std::move(config), + nullptr, // rdmaConfig NSpdk::CreateEnvStub(), CreateTestAllocator(), - nullptr, // storageProvider + nullptr, // storageProvider CreateProfileLogStub(), CreateBlockDigestGeneratorStub(), CreateLoggingService("console"), @@ -330,7 +340,9 @@ struct TFiles auto CreateDiskAgentStateNull(TDiskAgentConfigPtr config) { return std::make_unique( + CreateStorageConfig(), std::move(config), + nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), NServer::CreateNullStorageProvider(), @@ -462,28 +474,6 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) false); // checkSerialNumbers } - Y_UNIT_TEST_F(ShouldInitializeWithNull, TFiles) - { - auto counters = MakeIntrusive(); - InitCriticalEventsCounter(counters); - - auto mismatch = counters->GetCounter( - "AppCriticalEvents/DiskAgentDeviceGeneratedConfigMismatch", - true); - - UNIT_ASSERT_EQUAL(0, *mismatch); - - auto state = CreateDiskAgentStateNull( - CreateNullConfig({ .Files = Nvme3s }) - ); - - ShouldInitialize( - *state, - true); // checkSerialNumbers - - UNIT_ASSERT_EQUAL(0, *mismatch); - } - Y_UNIT_TEST_F(ShouldReportDeviceGeneratedConfigMismatch, TFiles) { auto counters = MakeIntrusive(); @@ -499,13 +489,9 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) auto& path = *discoveryConfig.AddPathConfigs(); path.SetPathRegExp(TempDir.Path() / "nvme3n([0-9])"); auto& pool = *path.AddPoolConfigs(); - // the IDs of the generated devices will differ from those in the config pool.SetHashSuffix("random"); - pool.SetMinSize(DefaultFileSize); - pool.SetMaxSize(DefaultFileSize); - auto state = CreateDiskAgentStateNull( CreateNullConfig({ .Files = Nvme3s, @@ -524,7 +510,9 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) auto config = CreateNullConfig({ .Files = Nvme3s }); TDiskAgentState state( + CreateStorageConfig(), config, + nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), std::make_shared(THashSet{ @@ -734,7 +722,9 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) auto config = std::make_shared(cfg, "rack"); TDiskAgentState state( + CreateStorageConfig(), config, + nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), std::make_shared(THashSet{ @@ -803,7 +793,9 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) } TDiskAgentState state( + CreateStorageConfig(), std::make_shared(std::move(config), "rack"), + nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), NServer::CreateNullStorageProvider(), @@ -1221,7 +1213,9 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) TTestStorageStatePtr storageState = MakeIntrusive(); TDiskAgentState state( + CreateStorageConfig(), config, + nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), std::make_shared(storageState), @@ -1369,9 +1363,7 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) NProto::TStorageDiscoveryConfig discovery; auto& path = *discovery.AddPathConfigs(); path.SetPathRegExp(TempDir.Path() / "nvme3n([0-9])"); - auto& pool = *path.AddPoolConfigs(); - pool.SetMinSize(DefaultFileSize); - pool.SetMaxSize(DefaultFileSize); + path.AddPoolConfigs(); return discovery; }(), @@ -1379,7 +1371,9 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) }); auto state = std::make_unique( + CreateStorageConfig(), config, + nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), std::make_shared(storageState), @@ -1449,8 +1443,6 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) // limit the number of devices to one path.SetPathRegExp(TempDir.Path() / "nvme3n(1)"); auto& pool = *path.AddPoolConfigs(); - pool.SetMinSize(DefaultFileSize); - pool.SetMaxSize(DefaultFileSize); pool.SetPoolName("foo"); return discovery; @@ -1459,7 +1451,9 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) }); auto state = std::make_unique( + CreateStorageConfig(), config, + nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), std::make_shared(storageState), @@ -1526,7 +1520,6 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) path.SetMaxDeviceCount(8); auto& pool = *path.AddPoolConfigs(); - pool.SetMinSize(DefaultFileSize); pool.SetMaxSize(10 * DefaultFileSize); auto& layout = *pool.MutableLayout(); @@ -1535,10 +1528,12 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) auto newState = [&] (auto discoveryConfig) { return std::make_unique( + CreateStorageConfig(), CreateNullConfig({ .DiscoveryConfig = discoveryConfig, .CachedConfigPath = CachedConfigPath }), + nullptr, // rdmaConfig nullptr, // spdk CreateTestAllocator(), std::make_shared(storageState), @@ -1622,8 +1617,7 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) auto& path = *discovery.AddPathConfigs(); path.SetPathRegExp(TempDir.Path() / "NVMENBS([0-9]+)"); - auto& pool = *path.AddPoolConfigs(); - pool.SetMaxSize(DefaultFileSize); + path.AddPoolConfigs(); auto state = CreateDiskAgentStateNull( CreateNullConfig({ @@ -1644,6 +1638,16 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) UNIT_ASSERT_EQUAL(0, *restoreError); + auto dumpSessionsToFile = [&] { + auto sessions = state->GetSessions(); + NProto::TDiskAgentDeviceSessionCache cache; + cache.MutableSessions()->Assign( + std::make_move_iterator(sessions.begin()), + std::make_move_iterator(sessions.end())); + + SerializeToTextFormat(cache, CachedSessionsPath); + }; + TVector devices; for (auto& d: state->GetDevices()) { devices.push_back(d.GetDeviceUUID()); @@ -1681,72 +1685,7 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) "vol1", 2000); // VolumeGeneration - // validate the cache file - - { - NProto::TDiskAgentDeviceSessionCache cache; - ParseProtoTextFromFileRobust(CachedSessionsPath, cache); - - UNIT_ASSERT_VALUES_EQUAL(3, cache.SessionsSize()); - - SortBy(*cache.MutableSessions(), [] (auto& session) { - return session.GetClientId(); - }); - - { - auto& session = *cache.MutableSessions(0); - Sort(*session.MutableDeviceIds()); - - UNIT_ASSERT_VALUES_EQUAL("reader-1", session.GetClientId()); - UNIT_ASSERT_VALUES_EQUAL(2, session.DeviceIdsSize()); - ASSERT_VECTORS_EQUAL( - TVector({devices[0], devices[1]}), - session.GetDeviceIds()); - - // MountSeqNumber is not applicable to read sessions - UNIT_ASSERT_VALUES_EQUAL(0, session.GetMountSeqNumber()); - UNIT_ASSERT_VALUES_EQUAL("vol0", session.GetDiskId()); - UNIT_ASSERT_VALUES_EQUAL(1001, session.GetVolumeGeneration()); - UNIT_ASSERT_VALUES_EQUAL(0, session.GetLastActivityTs()); - UNIT_ASSERT(session.GetReadOnly()); - } - - { - auto& session = *cache.MutableSessions(1); - - UNIT_ASSERT_VALUES_EQUAL("reader-2", session.GetClientId()); - UNIT_ASSERT_VALUES_EQUAL(2, session.DeviceIdsSize()); - ASSERT_VECTORS_EQUAL( - TVector({devices[2], devices[3]}), - session.GetDeviceIds()); - - // MountSeqNumber is not applicable to read sessions - UNIT_ASSERT_VALUES_EQUAL(0, session.GetMountSeqNumber()); - UNIT_ASSERT_VALUES_EQUAL("vol1", session.GetDiskId()); - UNIT_ASSERT_VALUES_EQUAL(2000, session.GetVolumeGeneration()); - UNIT_ASSERT_VALUES_EQUAL(0, session.GetLastActivityTs()); - UNIT_ASSERT(session.GetReadOnly()); - } - - { - auto& session = *cache.MutableSessions(2); - Sort(*session.MutableDeviceIds()); - - UNIT_ASSERT_VALUES_EQUAL("writer-1", session.GetClientId()); - UNIT_ASSERT_VALUES_EQUAL(2, session.DeviceIdsSize()); - ASSERT_VECTORS_EQUAL( - TVector({devices[0], devices[1]}), - session.GetDeviceIds()); - - UNIT_ASSERT_VALUES_EQUAL(42, session.GetMountSeqNumber()); - UNIT_ASSERT_VALUES_EQUAL("vol0", session.GetDiskId()); - - // VolumeGeneration was updated by reader-1 - UNIT_ASSERT_VALUES_EQUAL(1001, session.GetVolumeGeneration()); - UNIT_ASSERT_VALUES_EQUAL(0, session.GetLastActivityTs()); - UNIT_ASSERT(!session.GetReadOnly()); - } - } + dumpSessionsToFile(); // restart state = createState(); @@ -1848,6 +1787,8 @@ Y_UNIT_TEST_SUITE(TDiskAgentStateTest) "vol0", 1001); // VolumeGeneration + dumpSessionsToFile(); + // remove reader-2's file TFsPath { state->GetDeviceName(devices[3]) }.DeleteIfExists(); diff --git a/cloud/blockstore/libs/storage/disk_agent/model/config.h b/cloud/blockstore/libs/storage/disk_agent/model/config.h index a091eab5803..1c1aa5bb47c 100644 --- a/cloud/blockstore/libs/storage/disk_agent/model/config.h +++ b/cloud/blockstore/libs/storage/disk_agent/model/config.h @@ -55,12 +55,17 @@ class TDiskAgentConfig return Config.GetNvmeTarget(); } - auto GetRdmaTarget() + auto GetRdmaEndpoint() + { + return Config.GetRdmaTarget().GetEndpoint(); + } + + auto DeprecatedGetRdmaTarget() { return Config.GetRdmaTarget(); } - auto HasRdmaTarget() + auto DeprecatedHasRdmaTarget() { return Config.HasRdmaTarget(); } diff --git a/cloud/blockstore/libs/storage/disk_agent/model/device_client.cpp b/cloud/blockstore/libs/storage/disk_agent/model/device_client.cpp index f2a460fa7f0..33a96993599 100644 --- a/cloud/blockstore/libs/storage/disk_agent/model/device_client.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/model/device_client.cpp @@ -39,6 +39,7 @@ TVector TDeviceClient::GetSessions() const session.SetDiskId(state->DiskId); session.SetVolumeGeneration(state->VolumeGeneration); session.SetMountSeqNumber(ws.MountSeqNumber); + session.SetLastActivityTs(ws.LastActivityTs.MicroSeconds()); } *session.AddDeviceIds() = id; } @@ -51,6 +52,7 @@ TVector TDeviceClient::GetSessions() const session.SetDiskId(state->DiskId); session.SetVolumeGeneration(state->VolumeGeneration); session.SetMountSeqNumber(rs.MountSeqNumber); + session.SetLastActivityTs(rs.LastActivityTs.MicroSeconds()); } *session.AddDeviceIds() = id; } @@ -154,12 +156,17 @@ TResultOrError TDeviceClient::AcquireDevices( somethingHasChanged = true; } + // a new session or activation of a stale session + if (s == ds.ReaderSessions.end() || + s->LastActivityTs + ReleaseInactiveSessionsTimeout <= now) + { + somethingHasChanged = true; + } + if (s == ds.ReaderSessions.end()) { ds.ReaderSessions.push_back({clientId, now}); STORAGE_INFO("Device %s was acquired by client %s for reading.", uuid.Quote().c_str(), clientId.c_str()); - - somethingHasChanged = true; } else if (now > s->LastActivityTs) { s->LastActivityTs = now; } @@ -179,7 +186,11 @@ TResultOrError TDeviceClient::AcquireDevices( somethingHasChanged = true; } - if (ds.WriterSession.MountSeqNumber != mountSeqNumber) { + if (ds.WriterSession.MountSeqNumber != mountSeqNumber || + ds.WriterSession.LastActivityTs + + ReleaseInactiveSessionsTimeout <= + now) + { somethingHasChanged = true; } diff --git a/cloud/blockstore/libs/storage/disk_agent/model/device_client_ut.cpp b/cloud/blockstore/libs/storage/disk_agent/model/device_client_ut.cpp index 07645498fcd..05be345c90a 100644 --- a/cloud/blockstore/libs/storage/disk_agent/model/device_client_ut.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/model/device_client_ut.cpp @@ -8,8 +8,12 @@ #include +#include + namespace NCloud::NBlockStore::NStorage { +using namespace std::chrono_literals; + namespace { //////////////////////////////////////////////////////////////////////////////// @@ -130,13 +134,15 @@ struct TDeviceClientParams struct TFixture : public NUnitTest::TBaseFixture { + const TDuration ReleaseInactiveSessionsTimeout = 10s; + ILoggingServicePtr Logging = CreateLoggingService("console"); TDeviceClient CreateClient(TDeviceClientParams params = {}) { return TDeviceClient( - TDuration::Seconds(10), - params.Devices, + ReleaseInactiveSessionsTimeout, + std::move(params.Devices), Logging->CreateLog("BLOCKSTORE_DISK_AGENT")); } }; @@ -601,13 +607,15 @@ Y_UNIT_TEST_SUITE(TDeviceClientTest) UNIT_ASSERT_VALUES_EQUAL(0, client.GetSessions().size()); + const auto acquireWriterTs = TInstant::Seconds(42); + AcquireDevices( client, TAcquireParamsBuilder() .SetUuids({"uuid2", "uuid1"}) .SetClientId("writer") .SetDiskId("vol1") - .SetNow(TInstant::Seconds(42)) + .SetNow(acquireWriterTs) .SetVolumeGeneration(1)); { @@ -618,12 +626,16 @@ Y_UNIT_TEST_SUITE(TDeviceClientTest) UNIT_ASSERT_VALUES_EQUAL("writer", session.GetClientId()); UNIT_ASSERT_VALUES_EQUAL(1, session.GetVolumeGeneration()); UNIT_ASSERT(!session.GetReadOnly()); - UNIT_ASSERT_VALUES_EQUAL(0, session.GetLastActivityTs()); + UNIT_ASSERT_VALUES_EQUAL( + acquireWriterTs.MicroSeconds(), + session.GetLastActivityTs()); UNIT_ASSERT_VALUES_EQUAL(2, session.DeviceIdsSize()); UNIT_ASSERT_VALUES_EQUAL("uuid1", session.GetDeviceIds(0)); UNIT_ASSERT_VALUES_EQUAL("uuid2", session.GetDeviceIds(1)); } + const auto acquireReaderTs = TInstant::Seconds(42); + AcquireDevices( client, TAcquireParamsBuilder() @@ -631,7 +643,7 @@ Y_UNIT_TEST_SUITE(TDeviceClientTest) .SetClientId("reader") .SetDiskId("vol1") .SetAccessMode(NProto::VOLUME_ACCESS_READ_ONLY) - .SetNow(TInstant::Seconds(100)) + .SetNow(acquireReaderTs) .SetVolumeGeneration(2)); { @@ -642,7 +654,9 @@ Y_UNIT_TEST_SUITE(TDeviceClientTest) UNIT_ASSERT_VALUES_EQUAL("reader", session.GetClientId()); UNIT_ASSERT_VALUES_EQUAL(2, session.GetVolumeGeneration()); UNIT_ASSERT(session.GetReadOnly()); - UNIT_ASSERT_VALUES_EQUAL(0, session.GetLastActivityTs()); + UNIT_ASSERT_VALUES_EQUAL( + acquireReaderTs.MicroSeconds(), + session.GetLastActivityTs()); UNIT_ASSERT_VALUES_EQUAL(2, session.DeviceIdsSize()); UNIT_ASSERT_VALUES_EQUAL("uuid1", session.GetDeviceIds(0)); UNIT_ASSERT_VALUES_EQUAL("uuid2", session.GetDeviceIds(1)); @@ -653,7 +667,9 @@ Y_UNIT_TEST_SUITE(TDeviceClientTest) UNIT_ASSERT_VALUES_EQUAL("writer", session.GetClientId()); UNIT_ASSERT_VALUES_EQUAL(2, session.GetVolumeGeneration()); UNIT_ASSERT(!session.GetReadOnly()); - UNIT_ASSERT_VALUES_EQUAL(0, session.GetLastActivityTs()); + UNIT_ASSERT_VALUES_EQUAL( + acquireWriterTs.MicroSeconds(), + session.GetLastActivityTs()); UNIT_ASSERT_VALUES_EQUAL(2, session.DeviceIdsSize()); UNIT_ASSERT_VALUES_EQUAL("uuid1", session.GetDeviceIds(0)); UNIT_ASSERT_VALUES_EQUAL("uuid2", session.GetDeviceIds(1)); @@ -669,11 +685,13 @@ Y_UNIT_TEST_SUITE(TDeviceClientTest) UNIT_ASSERT_VALUES_EQUAL(0, client.GetSessions().size()); + TInstant now = Now(); + { auto [updated, error] = client.AcquireDevices( {"uuid2", "uuid1"}, // uuids "writer", // ClientId - TInstant::Seconds(10), // now + now, NProto::VOLUME_ACCESS_READ_WRITE, 1, // MountSeqNumber "vol0", // DiskId @@ -684,11 +702,33 @@ Y_UNIT_TEST_SUITE(TDeviceClientTest) UNIT_ASSERT(updated); // new write session } + // make the writer session stale + now += ReleaseInactiveSessionsTimeout; + + { + auto [updated, error] = client.AcquireDevices( + {"uuid2", "uuid1"}, // uuids + "writer", // ClientId + now, + NProto::VOLUME_ACCESS_READ_WRITE, + 1, // MountSeqNumber + "vol0", // DiskId + 1 // VolumeGeneration + ); + + UNIT_ASSERT_C(!HasError(error), error); + // writer session was activated + UNIT_ASSERT(updated); + } + + // writer session still active + now += 5s; + { auto [updated, error] = client.AcquireDevices( {"uuid2", "uuid1"}, // uuids "writer", // ClientId - TInstant::Seconds(100), // now + now, NProto::VOLUME_ACCESS_READ_WRITE, 1, // MountSeqNumber "vol0", // DiskId @@ -696,14 +736,17 @@ Y_UNIT_TEST_SUITE(TDeviceClientTest) ); UNIT_ASSERT_C(!HasError(error), error); + // nothing was changed UNIT_ASSERT(!updated); } + now += 5s; + { auto [updated, error] = client.AcquireDevices( {"uuid2", "uuid1"}, // uuids "writer", // ClientId - TInstant::Seconds(200), // now + now, NProto::VOLUME_ACCESS_READ_WRITE, 1, // MountSeqNumber "vol0", // DiskId @@ -714,11 +757,13 @@ Y_UNIT_TEST_SUITE(TDeviceClientTest) UNIT_ASSERT(updated); // new volumeGeneration } + now += 5s; + { auto [updated, error] = client.AcquireDevices( {"uuid2", "uuid1"}, // uuids "reader", // ClientId - TInstant::Seconds(300), // now + now, NProto::VOLUME_ACCESS_READ_ONLY, 1, // MountSeqNumber "vol0", // DiskId @@ -729,11 +774,13 @@ Y_UNIT_TEST_SUITE(TDeviceClientTest) UNIT_ASSERT(updated); // new read session } + now += 5s; + { auto [updated, error] = client.AcquireDevices( {"uuid2", "uuid1"}, // uuids "reader2", // ClientId - TInstant::Seconds(300), // now + now, NProto::VOLUME_ACCESS_READ_ONLY, 1, // MountSeqNumber "vol0", // DiskId diff --git a/cloud/blockstore/libs/storage/disk_agent/model/device_scanner.cpp b/cloud/blockstore/libs/storage/disk_agent/model/device_scanner.cpp index 815a7874c59..ca6f1fcf99d 100644 --- a/cloud/blockstore/libs/storage/disk_agent/model/device_scanner.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/model/device_scanner.cpp @@ -113,7 +113,19 @@ NProto::TError FindDevices( const ui64 size = GetFileLength(path); auto* pool = FindIfPtr(p.GetPoolConfigs(), [&] (const auto& pool) { - return pool.GetMinSize() <= size && size <= pool.GetMaxSize(); + ui64 minSize = pool.GetMinSize(); + + if (!minSize && pool.HasLayout()) { + minSize = + pool.GetLayout().GetHeaderSize() + + pool.GetLayout().GetDeviceSize(); + } + + const ui64 maxSize = pool.GetMaxSize() + ? pool.GetMaxSize() + : size; + + return minSize <= size && size <= maxSize; }); if (!pool) { diff --git a/cloud/blockstore/libs/storage/disk_agent/model/device_scanner_ut.cpp b/cloud/blockstore/libs/storage/disk_agent/model/device_scanner_ut.cpp index 076193a383e..34a37f44235 100644 --- a/cloud/blockstore/libs/storage/disk_agent/model/device_scanner_ut.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/model/device_scanner_ut.cpp @@ -261,6 +261,91 @@ Y_UNIT_TEST_SUITE(TDeviceScannerTest) UNIT_ASSERT_VALUES_EQUAL_C(expectedPathIndex, pathIndex, f); } } + + Y_UNIT_TEST_F(ShouldInferMinSizeFromLayout, TFixture) + { + PrepareFiles({ + { "dev/disk/by-partlabel/NVMENBS01", 100_KB }, + }); + + auto& nvme = *Config.AddPathConfigs(); + nvme.SetPathRegExp(RootDir / "dev/disk/by-partlabel/NVMENBS([0-9]{2})"); + nvme.SetBlockSize(4_KB); + + auto& def = *nvme.AddPoolConfigs(); + auto& layout = *def.MutableLayout(); + // MinSize & MaxSize are not specified + layout.SetHeaderSize(8_KB); + layout.SetDeviceSize(2_KB); + layout.SetDevicePadding(1_KB); + + TVector> r; + + auto error = FindDevices(Config, [&] ( + auto& path, + auto& pool, + auto pathIndex, + auto maxDeviceCount, + auto blockSize, + auto fileSize) + { + UNIT_ASSERT_VALUES_EQUAL(0, maxDeviceCount); + + NProto::TFileDeviceArgs f; + f.SetPath(path); + f.SetPoolName(pool.GetPoolName()); + f.SetBlockSize(blockSize); + f.SetFileSize(fileSize); + + r.emplace_back(std::move(f), pathIndex); + + return MakeError(S_OK); + }); + + // NVMENBS01 was accepted because of implicit MinSize = 10Kb and + // the unlimited MaxSize. + UNIT_ASSERT_VALUES_EQUAL_C(S_OK, error.GetCode(), error.GetMessage()); + UNIT_ASSERT_VALUES_EQUAL(1, r.size()); + + auto& [f, pathIndex] = r.front(); + + UNIT_ASSERT_VALUES_EQUAL("", f.GetPoolName()); + UNIT_ASSERT_VALUES_EQUAL( + RootDir / "dev/disk/by-partlabel/NVMENBS01", + f.GetPath()); + UNIT_ASSERT_VALUES_EQUAL(4_KB, f.GetBlockSize()); + UNIT_ASSERT_VALUES_EQUAL(100_KB, f.GetFileSize()); + UNIT_ASSERT_VALUES_EQUAL(1, pathIndex); + } + + Y_UNIT_TEST_F(ShouldIgnoreDevicesThatAreTooSmall, TFixture) + { + PrepareFiles({ + { "dev/disk/by-partlabel/NVMENBS01", 9_KB }, + }); + + auto& nvme = *Config.AddPathConfigs(); + nvme.SetPathRegExp(RootDir / "dev/disk/by-partlabel/NVMENBS([0-9]{2})"); + nvme.SetBlockSize(4_KB); + + auto& def = *nvme.AddPoolConfigs(); + auto& layout = *def.MutableLayout(); + // MinSize & MaxSize are not specified + layout.SetHeaderSize(8_KB); + layout.SetDeviceSize(2_KB); + layout.SetDevicePadding(1_KB); + + int success = 0; + + auto error = FindDevices(Config, [&] (auto ...) { + ++success; + return MakeError(S_OK); + }); + + // NVMENBS01 wasn't accepted because of implicit MinSize = 10Kb + UNIT_ASSERT_VALUES_EQUAL_C(E_NOT_FOUND, error.GetCode(), error.GetMessage()); + UNIT_ASSERT_VALUES_EQUAL(0, success); + } } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/disk_agent/rdma_target.cpp b/cloud/blockstore/libs/storage/disk_agent/rdma_target.cpp index 8fe79eba38c..f873793267e 100644 --- a/cloud/blockstore/libs/storage/disk_agent/rdma_target.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/rdma_target.cpp @@ -368,7 +368,14 @@ class TRequestHandler final TString* overlapDetails) const { if (synchronizedData.SecureEraseInProgress) { - ReportDiskAgentIoDuringSecureErase(); + ReportDiskAgentIoDuringSecureErase( + TStringBuilder() + << " Device=" << requestDetails.DeviceUUID + << ", ClientId=" << requestDetails.ClientId + << ", StartIndex=" << requestDetails.Range.Start + << ", BlocksCount=" << requestDetails.Range.Size() + << ", IsWrite=1" + << ", IsRdma=1"); *overlapDetails = "Secure erase in progress"; return ECheckRange::ResponseRejected; } @@ -509,6 +516,22 @@ class TRequestHandler final request.GetHeaders().GetClientId(), NProto::VOLUME_ACCESS_READ_ONLY); + auto token = GetAccessToken(request.GetDeviceUUID()); + if (token->SecureEraseInProgress) { + const auto& clientId = request.GetHeaders().GetClientId(); + if (clientId != CheckHealthClientId) { + ReportDiskAgentIoDuringSecureErase( + TStringBuilder() + << " Device=" << request.GetDeviceUUID() + << ", ClientId=" << clientId + << ", StartIndex=" << request.GetStartIndex() + << ", BlocksCount=" << request.GetBlocksCount() + << ", IsWrite=0" + << ", IsRdma=1"); + } + return MakeError(E_REJECTED, "Secure erase in progress"); + } + auto req = std::make_shared(); req->SetStartIndex(request.GetStartIndex()); @@ -944,6 +967,7 @@ class TRdmaTarget final void Stop() override { + Server->Stop(); TaskQueue->Stop(); } diff --git a/cloud/blockstore/libs/storage/disk_agent/storage_initializer.cpp b/cloud/blockstore/libs/storage/disk_agent/storage_initializer.cpp index 69a2e27ffef..b77fdb15f35 100644 --- a/cloud/blockstore/libs/storage/disk_agent/storage_initializer.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/storage_initializer.cpp @@ -95,6 +95,7 @@ class TInitializer { private: const TLog Log; + const TStorageConfigPtr StorageConfig; const TDiskAgentConfigPtr AgentConfig; const IStorageProviderPtr StorageProvider; const NNvme::INvmeManagerPtr NvmeManager; @@ -107,11 +108,13 @@ class TInitializer TDeviceGuard Guard; TVector Errors; + TVector ConfigMismatchErrors; TMutex Lock; public: TInitializer( TLog log, + TStorageConfigPtr storageConfig, TDiskAgentConfigPtr agentConfig, IStorageProviderPtr storageProvider, NNvme::INvmeManagerPtr nvmeManager); @@ -142,16 +145,20 @@ class TInitializer TVector LoadCachedConfig() const; void SaveCurrentConfig(); + + void ReportDiskAgentConfigMismatchEvent(const TString& error); }; //////////////////////////////////////////////////////////////////////////////// TInitializer::TInitializer( TLog log, + TStorageConfigPtr storageConfig, TDiskAgentConfigPtr agentConfig, IStorageProviderPtr storageProvider, NNvme::INvmeManagerPtr nvmeManager) : Log {std::move(log)} + , StorageConfig(std::move(storageConfig)) , AgentConfig(std::move(agentConfig)) , StorageProvider(std::move(storageProvider)) , NvmeManager(std::move(nvmeManager)) @@ -233,10 +240,27 @@ bool TInitializer::ValidateStorageDiscoveryConfig() const for (const auto& path: config.GetPathConfigs()) { for (const auto& pool: path.GetPoolConfigs()) { - if (pool.HasLayout() && !pool.GetLayout().GetDeviceSize()) { - STORAGE_WARN("Bad pool configuration: " << pool); - - return false; + if (pool.HasLayout()) { + const auto& layout = pool.GetLayout(); + + if (!layout.GetDeviceSize()) { + STORAGE_WARN( + "Bad pool configuration: the device size is not set. " + "Config: " << pool); + + return false; + } + + if (pool.GetMinSize() && pool.GetMinSize() < + layout.GetHeaderSize() + layout.GetDeviceSize()) + { + STORAGE_WARN( + "Bad pool configuration: the min size is less than the " + "minimum layout. " + "Config: " << pool); + + return false; + } } } } @@ -271,7 +295,7 @@ bool TInitializer::ValidateGeneratedConfigs( void TInitializer::ScanFileDevices() { if (!ValidateStorageDiscoveryConfig()) { - ReportDiskAgentConfigMismatch("Bad storage discovery config"); + ReportDiskAgentConfigMismatchEvent("Bad storage discovery config"); return; } @@ -281,9 +305,8 @@ void TInitializer::ScanFileDevices() AgentConfig->GetStorageDiscoveryConfig(), std::ref(gen)); HasError(error)) { - ReportDiskAgentConfigMismatch(TStringBuilder() + ReportDiskAgentConfigMismatchEvent(TStringBuilder() << "Can't generate config: " << FormatError(error)); - return; } @@ -296,7 +319,7 @@ void TInitializer::ScanFileDevices() SortBy(files, GetDeviceId); if (!ValidateGeneratedConfigs(files)) { - ReportDiskAgentConfigMismatch("Bad generated config"); + ReportDiskAgentConfigMismatchEvent("Bad generated config"); return; } @@ -322,13 +345,15 @@ void TInitializer::ScanFileDevices() ss << d << "\n"; } - ReportDiskAgentConfigMismatch(ss.Str()); + ReportDiskAgentConfigMismatchEvent(ss.Str()); } } TVector TInitializer::LoadCachedConfig() const { - const auto path = AgentConfig->GetCachedConfigPath(); + const TString storagePath = StorageConfig->GetCachedDiskAgentConfigPath(); + const TString diskAgentPath = AgentConfig->GetCachedConfigPath(); + const TString& path = diskAgentPath.empty() ? storagePath : diskAgentPath; if (path.empty()) { return {}; @@ -408,7 +433,7 @@ void TInitializer::ValidateCurrentConfigs() ss << d << "\n"; } - ReportDiskAgentConfigMismatch(ss.Str()); + ReportDiskAgentConfigMismatchEvent(ss.Str()); STORAGE_WARN("Current config is broken, fallback to the cached one."); FileDevices.swap(cachedDevices); @@ -444,7 +469,7 @@ TFuture TInitializer::Initialize() Configs[i] = CreateConfig(device); Stats[i] = std::make_shared(); - auto onInitError = [=] () { + auto onInitError = [i, this] () { OnError(i, CurrentExceptionMessage()); Configs[i].SetState(NProto::DEVICE_STATE_ERROR); @@ -584,23 +609,31 @@ TInitializeStorageResult TInitializer::GetResult() } r.Errors = std::move(Errors); + r.ConfigMismatchErrors = std::move(ConfigMismatchErrors); r.Guard = std::move(Guard); return r; } +void TInitializer::ReportDiskAgentConfigMismatchEvent(const TString& error) { + ReportDiskAgentConfigMismatch(error); + ConfigMismatchErrors.push_back(error); +} + } // namespace //////////////////////////////////////////////////////////////////////////////// TFuture InitializeStorage( TLog log, + TStorageConfigPtr storageConfig, TDiskAgentConfigPtr agentConfig, IStorageProviderPtr storageProvider, NNvme::INvmeManagerPtr nvmeManager) { auto initializer = std::make_shared( std::move(log), + std::move(storageConfig), std::move(agentConfig), std::move(storageProvider), std::move(nvmeManager)); diff --git a/cloud/blockstore/libs/storage/disk_agent/storage_initializer.h b/cloud/blockstore/libs/storage/disk_agent/storage_initializer.h index 8b962d4a2f6..a01970afdda 100644 --- a/cloud/blockstore/libs/storage/disk_agent/storage_initializer.h +++ b/cloud/blockstore/libs/storage/disk_agent/storage_initializer.h @@ -28,11 +28,13 @@ struct TInitializeStorageResult TVector Devices; TVector Stats; TVector Errors; + TVector ConfigMismatchErrors; TDeviceGuard Guard; }; NThreading::TFuture InitializeStorage( TLog log, + TStorageConfigPtr storageConfig, TDiskAgentConfigPtr agentConfig, IStorageProviderPtr storageProvider, NNvme::INvmeManagerPtr nvmeManager); diff --git a/cloud/blockstore/libs/storage/disk_agent/testlib/test_env.cpp b/cloud/blockstore/libs/storage/disk_agent/testlib/test_env.cpp index 1d64d56c6c5..6c87a989dc0 100644 --- a/cloud/blockstore/libs/storage/disk_agent/testlib/test_env.cpp +++ b/cloud/blockstore/libs/storage/disk_agent/testlib/test_env.cpp @@ -304,6 +304,8 @@ TTestEnv TTestEnvBuilder::Build() // } // Runtime.SetLogPriority(NLog::InvalidComponent, NLog::PRI_DEBUG); + Runtime.SetLogPriority(TBlockStoreComponents::DISK_AGENT, NLog::PRI_INFO); + Runtime.SetRegistrationObserverFunc( [] (auto& runtime, const auto& parentId, const auto& actorId) { @@ -353,6 +355,7 @@ TTestEnv TTestEnvBuilder::Build() auto diskAgent = CreateDiskAgent( config, agentConfig, + nullptr, // rdmaConfig Spdk, allocator, StorageProvider, diff --git a/cloud/blockstore/libs/storage/disk_agent/testlib/test_env.h b/cloud/blockstore/libs/storage/disk_agent/testlib/test_env.h index 680cfa07718..91c89e48720 100644 --- a/cloud/blockstore/libs/storage/disk_agent/testlib/test_env.h +++ b/cloud/blockstore/libs/storage/disk_agent/testlib/test_env.h @@ -135,9 +135,9 @@ class TDiskAgentClient const TString& clientId, const NProto::EVolumeAccessMode accessMode, const ui64 mountSeqNumber = 0, - const NSpdk::TDeviceRateLimits& limits = {}, const TString& diskId = "", - const ui32 volumeGeneration = 0) + const ui32 volumeGeneration = 0, + const NSpdk::TDeviceRateLimits& limits = {}) { auto request = std::make_unique(); diff --git a/cloud/blockstore/libs/storage/disk_agent/ya.make b/cloud/blockstore/libs/storage/disk_agent/ya.make index 915a2f97b47..dcf7fa037a2 100644 --- a/cloud/blockstore/libs/storage/disk_agent/ya.make +++ b/cloud/blockstore/libs/storage/disk_agent/ya.make @@ -33,6 +33,7 @@ PEERDIR( cloud/blockstore/libs/spdk/iface cloud/blockstore/libs/storage/api cloud/blockstore/libs/storage/core + cloud/blockstore/libs/storage/disk_agent/actors cloud/blockstore/libs/storage/disk_agent/model cloud/blockstore/libs/storage/disk_common cloud/blockstore/libs/storage/model @@ -52,6 +53,7 @@ PEERDIR( END() RECURSE( + actors model ) diff --git a/cloud/blockstore/libs/storage/disk_common/monitoring_utils.cpp b/cloud/blockstore/libs/storage/disk_common/monitoring_utils.cpp index af91e576778..5bd62bea3e3 100644 --- a/cloud/blockstore/libs/storage/disk_common/monitoring_utils.cpp +++ b/cloud/blockstore/libs/storage/disk_common/monitoring_utils.cpp @@ -47,8 +47,7 @@ IOutputStream& DumpDiskState( IOutputStream& DumpDeviceState( IOutputStream& out, NProto::EDeviceState state, - bool isFresh, - bool isDisabled, + EDeviceStateFlags flags, TString suffix) { switch (state) { @@ -67,11 +66,17 @@ IOutputStream& DumpDeviceState( << static_cast(state) << ")" << suffix; } - if (isFresh) { - out << " [Fresh]"; + if (flags & EDeviceStateFlags::FRESH) { + out << " [fresh]"; } - if (isDisabled) { - out << " [Disabled]"; + if (flags & EDeviceStateFlags::DISABLED) { + out << " [disabled]"; + } + if (flags & EDeviceStateFlags::DIRTY) { + out << " [dirty]"; + } + if (flags & EDeviceStateFlags::SUSPENDED) { + out << " [suspended]"; } return out; } diff --git a/cloud/blockstore/libs/storage/disk_common/monitoring_utils.h b/cloud/blockstore/libs/storage/disk_common/monitoring_utils.h index 61ebdb6d665..7f1fe9430d1 100644 --- a/cloud/blockstore/libs/storage/disk_common/monitoring_utils.h +++ b/cloud/blockstore/libs/storage/disk_common/monitoring_utils.h @@ -8,6 +8,28 @@ namespace NCloud::NBlockStore::NStorage { //////////////////////////////////////////////////////////////////////////////// +enum EDeviceStateFlags : uint16_t +{ + NONE = 0, + FRESH = 1 << 0, + DISABLED = 1 << 1, + DIRTY = 1 << 2, + SUSPENDED = 1 << 3, +}; + +inline EDeviceStateFlags operator|(EDeviceStateFlags a, EDeviceStateFlags b) +{ + return static_cast( + static_cast(a) | static_cast(b)); +} + +inline EDeviceStateFlags& operator|=(EDeviceStateFlags& a, EDeviceStateFlags b) +{ + return a = a | b; +} + +//////////////////////////////////////////////////////////////////////////////// + IOutputStream& DumpAgentState( IOutputStream& out, NProto::EAgentState state); @@ -19,8 +41,7 @@ IOutputStream& DumpDiskState( IOutputStream& DumpDeviceState( IOutputStream& out, NProto::EDeviceState state, - bool isFresh = false, - bool isDisabled = false, + EDeviceStateFlags flags = EDeviceStateFlags::NONE, TString suffix = ""); } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.cpp index 58fdc3670c9..2f9b14af635 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.cpp @@ -754,6 +754,9 @@ STFUNC(TDiskRegistryActor::StateReadOnly) TEvDiskRegistryPrivate::TEvDiskRegistryAgentListExpiredParamsCleanup, TDiskRegistryActor::HandleDiskRegistryAgentListExpiredParamsCleanupReadOnly); + HFunc(TEvDiskRegistryPrivate::TEvCleanupDisksResponse, + HandleCleanupDisksResponse); + default: if (!RejectRequests(ev)) { LogUnexpectedEvent( diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.h b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.h index 741abc30c69..759f583df1a 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.h +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.h @@ -74,6 +74,11 @@ class TDiskRegistryActor final const NProto::TError& error); }; + struct TAdditionalColumn { + std::function TitleInserter; + std::function DataInserter; + }; + private: const TStorageConfigPtr Config; const TDiagnosticsConfigPtr DiagnosticsConfig; @@ -292,14 +297,15 @@ class TDiskRegistryActor final void RenderPoolRacks(IOutputStream& out, const TString& poolName) const; void RenderAgentList(TInstant now, IOutputStream& out, ui32 limit) const; void RenderConfig(IOutputStream& out, ui32 limit) const; - void RenderDirtyDeviceList(IOutputStream& out) const; - void RenderSuspendedDeviceList(IOutputStream& out) const; + void RenderDirtyDeviceList(IOutputStream& out, ui32 limit) const; + void RenderSuspendedDeviceList(IOutputStream& out, ui32 limit) const; void RenderAutomaticallyReplacedDeviceList(IOutputStream& out) const; template void RenderDevicesWithDetails( IOutputStream& out, const TDevices& devices, - const TString& title) const; + const TString& title, + const TVector& additionalColumns = {}) const; void RenderBrokenDeviceList(IOutputStream& out, ui32 limit) const; void RenderDeviceHtmlInfo(IOutputStream& out, const TString& id) const; void RenderAgentHtmlInfo(IOutputStream& out, const TString& id) const; @@ -385,6 +391,16 @@ class TDiskRegistryActor final const TCgiParameters& params, TRequestInfoPtr requestInfo); + void HandleHttpInfo_RenderDirtyDeviceList( + const NActors::TActorContext& ctx, + const TCgiParameters& params, + TRequestInfoPtr requestInfo); + + void HandleHttpInfo_RenderSuspendedDeviceList( + const NActors::TActorContext& ctx, + const TCgiParameters& params, + TRequestInfoPtr requestInfo); + void HandleHttpInfo_RenderConfig( const NActors::TActorContext& ctx, const TCgiParameters& params, diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_acquire.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_acquire.cpp index 4082d96c735..f1a91d5a617 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_acquire.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_acquire.cpp @@ -136,7 +136,7 @@ void TAcquireDiskActor::Bootstrap(const TActorContext& ctx) Become(&TThis::StateAcquire); if (Devices.empty()) { - FinishAcquireDisk(ctx, MakeError(E_REJECTED, "nothing to acquire")); + FinishAcquireDisk(ctx, {}); return; } @@ -155,10 +155,10 @@ void TAcquireDiskActor::Bootstrap(const TActorContext& ctx) SentAcquireRequests.reserve(sentRequests.size()); TInstant now = ctx.Now(); for (auto& x: sentRequests) { - SentAcquireRequests.emplace_back( + SentAcquireRequests.push_back(TAgentAcquireDevicesCachedRequest{ std::move(x.AgentId), std::move(x.Record), - now); + now}); } } diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_monitoring.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_monitoring.cpp index 5ba52c77c26..92e7b7f257f 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_monitoring.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_monitoring.cpp @@ -22,6 +22,20 @@ namespace { //////////////////////////////////////////////////////////////////////////////// +EDeviceStateFlags GetDeviceStateFlags( + const TDiskRegistryState& state, + const TString& deviceUUID) +{ + EDeviceStateFlags deviceStateFlags = EDeviceStateFlags::NONE; + if (state.IsDirtyDevice(deviceUUID)) { + deviceStateFlags |= EDeviceStateFlags::DIRTY; + } + if (state.IsSuspendedDevice(deviceUUID)) { + deviceStateFlags |= EDeviceStateFlags::SUSPENDED; + } + return deviceStateFlags; +} + void BuildVolumeReallocateButton( IOutputStream& out, ui64 tabletId, @@ -32,7 +46,7 @@ void BuildVolumeReallocateButton( void TDiskRegistryActor::RenderDevicesWithDetails( IOutputStream& out, const TDevices& devices, - const TString& title) const + const TString& title, + const TVector& additionalColumns) const { HTML(out) { if (title) { @@ -228,22 +243,35 @@ void TDiskRegistryActor::RenderDevicesWithDetails( TABLEH() { out << "Block size"; } TABLEH() { out << "Block count"; } TABLEH() { out << "Size"; } - TABLEH() { out << "PhysicalOffset"; } + TABLEH() { out << "Physical offset"; } TABLEH() { out << "Transport id"; } TABLEH() { out << "Rdma endpoint"; } TABLEH() { out << "DiskId"; } TABLEH() { out << "Pool"; } + for (const auto& additionalColumn: additionalColumns) { + TABLEH() { additionalColumn.TitleInserter(out); } + } } } - for (const auto& device: devices) { + for (size_t index = 0; index < static_cast(devices.size()); + ++index) + { + const auto& device = devices[index]; TABLER() { TABLED() { DumpDeviceLink(out, TabletID(), device.GetDeviceUUID()); } TABLED() { out << device.GetDeviceName(); } TABLED() { out << device.GetSerialNumber(); } - TABLED() { DumpDeviceState(out, device.GetState()); } + TABLED() { + DumpDeviceState( + out, + device.GetState(), + GetDeviceStateFlags( + *State, + device.GetDeviceUUID())); + } TABLED() { out << TInstant::MicroSeconds(device.GetStateTs()); } @@ -261,7 +289,9 @@ void TDiskRegistryActor::RenderDevicesWithDetails( out << FormatByteSize(bytes); } TABLED() { - out << FormatByteSize(device.GetPhysicalOffset()); + out << device.GetPhysicalOffset() << " (" + << FormatByteSize(device.GetPhysicalOffset()) + << ")"; } TABLED() { out << device.GetTransportId(); } TABLED() { @@ -276,6 +306,9 @@ void TDiskRegistryActor::RenderDevicesWithDetails( } } TABLED() { out << device.GetPoolName(); } + for (const auto& additionalColumn: additionalColumns) { + TABLED() { additionalColumn.DataInserter(index, out); } + } } } } @@ -299,13 +332,16 @@ void TDiskRegistryActor::RenderBrokenDeviceList( ui32 limit) const { auto brokenDevices = State->GetBrokenDevices(); + if (brokenDevices.empty()) { + return; + } if (brokenDevices.size() > limit) { DumpActionLink( out, TabletID(), "RenderBrokenDeviceList", - "BrokenDevices", + "Broken devices", brokenDevices.size()); return; @@ -396,7 +432,13 @@ void TDiskRegistryActor::RenderDeviceHtmlInfo( out << device.GetRack(); } - DIV() { out << "State: "; DumpDeviceState(out, device.GetState()); } + DIV() { + out << "State: "; + DumpDeviceState( + out, + device.GetState(), + GetDeviceStateFlags(*State, id)); + } DIV() { out << "State Timestamp: " << TInstant::MicroSeconds(device.GetStateTs()); @@ -593,6 +635,11 @@ void TDiskRegistryActor::RenderDiskHtmlInfo( DIV() { out << "Acquire in progress"; } } + DIV() + { + BuildVolumeReallocateButton(out, TabletID(), id); + } + const bool isNonrepl = info.Replicas.empty(); auto makeHeaderCells = [&] (const ui32 replicaNo) { @@ -641,12 +688,16 @@ void TDiskRegistryActor::RenderDiskHtmlInfo( } } TABLED() { - const bool isFresh = - FindPtr( + EDeviceStateFlags flags = + GetDeviceStateFlags(*State, device.GetDeviceUUID()); + if (FindPtr( info.MasterDiskId ? masterDiskInfo.DeviceReplacementIds : info.DeviceReplacementIds, - device.GetDeviceUUID()) != nullptr; - DumpDeviceState(out, device.GetState(), isFresh); + device.GetDeviceUUID()) != nullptr) + { + flags |= EDeviceStateFlags::FRESH; + } + DumpDeviceState(out, device.GetState(), flags); } TABLED() { out << TInstant::MicroSeconds(device.GetStateTs()); @@ -821,7 +872,6 @@ void TDiskRegistryActor::RenderDiskList(IOutputStream& out) const TABLEH() { out << "Disk"; } TABLEH() { out << "State"; } TABLEH() { out << "State Timestamp"; } - TABLEH() { out << "Action"; } } } @@ -833,9 +883,6 @@ void TDiskRegistryActor::RenderDiskList(IOutputStream& out) const TABLED() { DumpDiskLink(out, TabletID(), id); } TABLED() { DumpDiskState(out, diskInfo.State); } TABLED() { out << diskInfo.StateTs; } - TABLED() { - BuildVolumeReallocateButton(out, TabletID(), id); - } } } } @@ -1461,7 +1508,7 @@ void TDiskRegistryActor::RenderPlacementGroupList( out, TabletID(), "RenderPlacementGroupList", - "PlacementGroups", + "Placement groups", State->GetPlacementGroups().size()); return; @@ -1731,16 +1778,14 @@ void TDiskRegistryActor::RenderAgentList( DumpDeviceState( out, NProto::DEVICE_STATE_ONLINE, - false, - false, + EDeviceStateFlags::NONE, TStringBuilder() << " " << onlineDevs); if (warningDevs) { out << " / "; DumpDeviceState( out, NProto::DEVICE_STATE_WARNING, - false, - false, + EDeviceStateFlags::NONE, TStringBuilder() << " " << warningDevs); } if (errorDevs) { @@ -1748,8 +1793,7 @@ void TDiskRegistryActor::RenderAgentList( DumpDeviceState( out, NProto::DEVICE_STATE_ERROR, - false, - false, + EDeviceStateFlags::NONE, TStringBuilder() << " " << errorDevs); } } @@ -1917,57 +1961,132 @@ void TDiskRegistryActor::RenderConfig(IOutputStream& out, ui32 limit) const //////////////////////////////////////////////////////////////////////////////// -void TDiskRegistryActor::RenderDirtyDeviceList(IOutputStream& out) const +void TDiskRegistryActor::HandleHttpInfo_RenderDirtyDeviceList( + const NActors::TActorContext& ctx, + const TCgiParameters& params, + TRequestInfoPtr requestInfo) { - const auto devices = State->GetDirtyDevices(); + Y_UNUSED(params); - HTML(out) { - TAG(TH3) { out << "Dirty devices"; DumpSize(out, devices); } + TStringStream out; + RenderDirtyDeviceList(out, Max()); + SendHttpResponse(ctx, *requestInfo, std::move(out.Str())); +} - UL() { - for (const auto& device: devices) { - LI() { - DumpDeviceLink(out, TabletID(), device.GetDeviceUUID()); - if (State->IsAutomaticallyReplaced(device.GetDeviceUUID())) - { - out << " Automatically replaced "; - } - out << " (#" << device.GetNodeId() << " )"; - } - } - } +void TDiskRegistryActor::RenderDirtyDeviceList(IOutputStream& out, ui32 limit) + const +{ + auto dirtyDevices = State->GetDirtyDevices(); + if (dirtyDevices.empty()) { + return; + } + + if (dirtyDevices.size() > limit) { + DumpActionLink( + out, + TabletID(), + "RenderDirtyDeviceList", + "Dirty devices", + dirtyDevices.size()); + + return; } + + TVector additionalColumns; + additionalColumns.push_back(TAdditionalColumn{ + .TitleInserter = [](IOutputStream& out) + { out << "Automatically replaced"; }, + .DataInserter = + [&dirtyDevices, this](size_t index, IOutputStream& out) + { + if (dirtyDevices.size() <= index) { + Y_DEBUG_ABORT_UNLESS(false); + out << "null"; + return; + } + if (State->IsAutomaticallyReplaced( + dirtyDevices[index].GetDeviceUUID())) { + out << "Yes"; + } else { + out << "No"; + } + }}); + + RenderDevicesWithDetails( + out, + dirtyDevices, + "Dirty Devices", + additionalColumns); } -void TDiskRegistryActor::RenderSuspendedDeviceList(IOutputStream& out) const +//////////////////////////////////////////////////////////////////////////////// + +void TDiskRegistryActor::HandleHttpInfo_RenderSuspendedDeviceList( + const NActors::TActorContext& ctx, + const TCgiParameters& params, + TRequestInfoPtr requestInfo) { - const auto devices = State->GetSuspendedDevices(); - if (devices.empty()) { + Y_UNUSED(params); + + TStringStream out; + RenderSuspendedDeviceList(out, Max()); + SendHttpResponse(ctx, *requestInfo, std::move(out.Str())); +} + +void TDiskRegistryActor::RenderSuspendedDeviceList( + IOutputStream& out, + ui32 limit) const +{ + const auto suspendedDevices = State->GetSuspendedDevices(); + if (suspendedDevices.empty()) { return; } - HTML(out) { - TAG(TH3) { out << "Suspended devices"; } + if (suspendedDevices.size() > limit) { + DumpActionLink( + out, + TabletID(), + "RenderSuspendedDeviceList", + "Suspended devices", + suspendedDevices.size()); + return; + } - UL() { - for (const auto& device: devices) { - const auto& uuid = device.GetId(); - LI() { - DumpDeviceLink(out, TabletID(), uuid); - if (device.GetResumeAfterErase()) { - out << " [resuming]"; - } + TVector suspendedDeviceConfigs; + suspendedDeviceConfigs.reserve(suspendedDevices.size()); + for (const auto& suspendedDevice: suspendedDevices) { + suspendedDeviceConfigs.push_back( + State->GetDevice(suspendedDevice.GetId())); + } - auto config = State->GetDevice(uuid); - if (config.GetNodeId() != 0) { - out << " (#" << config.GetNodeId() << " )"; - } - } + TVector additionalColumns; + additionalColumns.push_back(TAdditionalColumn{ + .TitleInserter = [](IOutputStream& out) + { out << "Resume after erase"; }, + .DataInserter = + [&suspendedDevices](size_t index, IOutputStream& out) + { + if (suspendedDevices.size() <= index) { + Y_DEBUG_ABORT_UNLESS(false); + out << "null"; + return; } - } - } + if (suspendedDevices[index].GetResumeAfterErase()) { + out << "Yes"; + } else { + out << "No"; + } + }}); + + RenderDevicesWithDetails( + out, + suspendedDeviceConfigs, + "Suspended Devices", + additionalColumns); } +//////////////////////////////////////////////////////////////////////////////// + void TDiskRegistryActor::RenderAutomaticallyReplacedDeviceList( IOutputStream& out) const { @@ -2033,11 +2152,11 @@ void TDiskRegistryActor::RenderHtmlInfo(TInstant now, IOutputStream& out) const RenderConfig(out, 20); - RenderDirtyDeviceList(out); + RenderDirtyDeviceList(out, 20); - RenderBrokenDeviceList(out, 30); + RenderBrokenDeviceList(out, 20); - RenderSuspendedDeviceList(out); + RenderSuspendedDeviceList(out, 20); RenderAutomaticallyReplacedDeviceList(out); } else { @@ -2098,17 +2217,18 @@ void TDiskRegistryActor::HandleHttpInfo( {"disk", &TDiskRegistryActor::HandleHttpInfo_RenderDiskHtmlInfo }, {"RenderDisks", &TDiskRegistryActor::HandleHttpInfo_RenderDisks}, - { - "RenderBrokenDeviceList", - &TDiskRegistryActor::HandleHttpInfo_RenderBrokenDeviceList - }, - { - "RenderPlacementGroupList", - &TDiskRegistryActor::HandleHttpInfo_RenderPlacementGroupList - }, + {"RenderBrokenDeviceList", + &TDiskRegistryActor::HandleHttpInfo_RenderBrokenDeviceList}, + {"RenderPlacementGroupList", + &TDiskRegistryActor::HandleHttpInfo_RenderPlacementGroupList}, {"RenderRacks", &TDiskRegistryActor::HandleHttpInfo_RenderRacks}, - {"RenderAgentList", &TDiskRegistryActor::HandleHttpInfo_RenderAgentList}, + {"RenderAgentList", + &TDiskRegistryActor::HandleHttpInfo_RenderAgentList}, {"RenderConfig", &TDiskRegistryActor::HandleHttpInfo_RenderConfig}, + {"RenderDirtyDeviceList", + &TDiskRegistryActor::HandleHttpInfo_RenderDirtyDeviceList}, + {"RenderSuspendedDeviceList", + &TDiskRegistryActor::HandleHttpInfo_RenderSuspendedDeviceList}, }}; auto* msg = ev->Get(); diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state.cpp index 1b6106d15d9..bd55a0c5845 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state.cpp @@ -495,6 +495,10 @@ void TDiskRegistryState::ProcessDisks(TVector configs) disk.MigrationStartTs = disk.StateTs; } + if (disk.MigrationSource2Target.empty()) { + disk.MigrationStartTs = {}; + } + if (!config.GetFinishedMigrations().empty()) { ui64 seqNo = NotificationSystem.GetDiskSeqNo(diskId); if (!seqNo) { @@ -754,6 +758,15 @@ NProto::TError TDiskRegistryState::ValidateAgent( << "all agent devices should come from the same rack, mismatch: " << rack << " != " << device.GetRack()); } + + if (device.GetState() != NProto::DEVICE_STATE_ONLINE && + device.GetState() != NProto::DEVICE_STATE_ERROR) + { + return MakeError(E_ARGUMENT, TStringBuilder() + << "unexpected state of the device " << device.GetDeviceUUID() + << ": " << NProto::EDeviceState_Name(device.GetState()) + << " (" << static_cast(device.GetState()) << ")"); + } } return {}; diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_ut.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_ut.cpp index 0c8ee45e7a4..f95865c5371 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_ut.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state_ut.cpp @@ -11141,9 +11141,9 @@ Y_UNIT_TEST_SUITE(TDiskRegistryStateTest) Y_UNIT_TEST(ShouldPullInLegacyDiskErrorUserNotifications) { - TDiskRegistryStateBuilder builder; - builder.ErrorNotifications.assign({"disk0", "disk1", "disk2"}); - auto state = builder.Build(); + auto state = TDiskRegistryStateBuilder() + .WithErrorNotifications({"disk0", "disk1", "disk2"}) + .Build(); UNIT_ASSERT_VALUES_EQUAL(3, state.GetUserNotifications().Count); @@ -11324,6 +11324,88 @@ Y_UNIT_TEST_SUITE(TDiskRegistryStateTest) // make sure uuid-2.1 is not considered dirty or automatically replaced anymore checkDevices({"uuid-2.1", "uuid-2.2"}, {"uuid-1.1", "uuid-2.4"}, {}); } + + Y_UNIT_TEST(ShouldPreserveDeviceErrorState) + { + const TString errorMessage = "broken device"; + + TTestExecutor executor; + executor.WriteTx([&] (TDiskRegistryDatabase db) { + db.InitSchema(); + }); + + auto agentConfig = AgentConfig(1000, { + Device("dev-1", "uuid-1", "rack-1") + }); + + auto state = TDiskRegistryStateBuilder() + .WithConfig({agentConfig}) + .Build(); + + // Register new agent with one device. + executor.WriteTx([&] (TDiskRegistryDatabase db) { + const TInstant now = Now(); + + TVector affectedDisks; + TVector disksToReallocate; + UNIT_ASSERT_SUCCESS(state.RegisterAgent( + db, + agentConfig, + now, + &affectedDisks, + &disksToReallocate)); + + auto d = state.GetDevice("uuid-1"); + UNIT_ASSERT_EQUAL(NProto::DEVICE_STATE_ONLINE, d.GetState()); + UNIT_ASSERT_VALUES_EQUAL(now.MicroSeconds(), d.GetStateTs()); + UNIT_ASSERT_VALUES_EQUAL("", d.GetStateMessage()); + }); + + // Break the device. + agentConfig.MutableDevices(0)->SetState(NProto::DEVICE_STATE_ERROR); + agentConfig.MutableDevices(0)->SetStateMessage(errorMessage); + + const TInstant errorTs = Now(); + + // Register the agent with the broken device. + // Now we expect to see our device in an error state. + executor.WriteTx([&] (TDiskRegistryDatabase db) { + TVector affectedDisks; + TVector disksToReallocate; + UNIT_ASSERT_SUCCESS(state.RegisterAgent( + db, + agentConfig, + errorTs, + &affectedDisks, + &disksToReallocate)); + + auto d = state.GetDevice("uuid-1"); + UNIT_ASSERT_EQUAL(NProto::DEVICE_STATE_ERROR, d.GetState()); + UNIT_ASSERT_VALUES_EQUAL(errorTs.MicroSeconds(), d.GetStateTs()); + UNIT_ASSERT_VALUES_EQUAL(errorMessage, d.GetStateMessage()); + }); + + // Fix the device + agentConfig.MutableDevices(0)->SetState(NProto::DEVICE_STATE_ONLINE); + agentConfig.MutableDevices(0)->SetStateMessage(""); + + // Register the agent with fixed device. + // But we expect that the device state remains the same (error). + executor.WriteTx([&] (TDiskRegistryDatabase db) { + TVector affectedDisks; + TVector disksToReallocate; + UNIT_ASSERT_SUCCESS(state.RegisterAgent(db, + agentConfig, + Now(), + &affectedDisks, + &disksToReallocate)); + + auto d = state.GetDevice("uuid-1"); + UNIT_ASSERT_EQUAL(NProto::DEVICE_STATE_ERROR, d.GetState()); + UNIT_ASSERT_VALUES_EQUAL(errorTs.MicroSeconds(), d.GetStateTs()); + UNIT_ASSERT_VALUES_EQUAL(errorMessage, d.GetStateMessage()); + }); + } } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_migration.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_migration.cpp index 9b115f997a5..b21a2cb7a98 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_migration.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_migration.cpp @@ -1546,9 +1546,7 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) diskRegistry.UpdateConfig(CreateRegistryConfig(agents)); - RegisterAgents(*runtime, 1); - WaitForAgents(*runtime, 1); - WaitForSecureErase(*runtime, agents); + RegisterAndWaitForAgents(*runtime, agents); const TVector devices = [&] { auto response = diskRegistry.AllocateDisk("vol0", 20_GB); @@ -1577,6 +1575,7 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) const auto maxTime = TDuration::Seconds(maxMigrationTime->Val()); + UNIT_ASSERT_LT_C(TDuration::Zero(), maxTime, maxTime); UNIT_ASSERT_LE_C(maxTime, UpdateCountersInterval + dt, maxTime); } @@ -1591,6 +1590,7 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) const auto maxTime = TDuration::Seconds(maxMigrationTime->Val()); + UNIT_ASSERT_LT_C(TDuration::Zero(), maxTime, maxTime); UNIT_ASSERT_LE_C(maxTime, UpdateCountersInterval + dt, maxTime); } @@ -1607,6 +1607,7 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) const auto maxTime = TDuration::Seconds(maxMigrationTime->Val()); + UNIT_ASSERT_LT_C(TDuration::Zero(), maxTime, maxTime); UNIT_ASSERT_LE_C(maxTime, UpdateCountersInterval + dt, maxTime); } @@ -1622,6 +1623,7 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) const auto maxTime = TDuration::Seconds(maxMigrationTime->Val()); + UNIT_ASSERT_LT_C(TDuration::Zero(), maxTime, maxTime); UNIT_ASSERT_LE_C(maxTime, UpdateCountersInterval + dt, maxTime); } @@ -1636,6 +1638,7 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) const auto maxTime = TDuration::Seconds(maxMigrationTime->Val()); + UNIT_ASSERT_LT_C(TDuration::Zero(), maxTime, maxTime); UNIT_ASSERT_LE_C(maxTime, UpdateCountersInterval + dt, maxTime); } @@ -1668,9 +1671,80 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) const auto maxTime = TDuration::Seconds(maxMigrationTime->Val()); + UNIT_ASSERT_LT_C(TDuration::Zero(), maxTime, maxTime); UNIT_ASSERT_LE_C(maxTime, UpdateCountersInterval + dt, maxTime); } } + + Y_UNIT_TEST(ShouldResetMigrationTime) + { + const TVector agents { + CreateAgentConfig("agent-1", { + Device("dev-1", "uuid-1.1", "rack-1", 10_GB), + Device("dev-2", "uuid-1.2", "rack-1", 10_GB), + }) + }; + + auto runtime = TTestRuntimeBuilder() + .WithAgents(agents) + .Build(); + + TDiskRegistryClient diskRegistry(*runtime); + diskRegistry.WaitReady(); + diskRegistry.SetWritableState(true); + diskRegistry.UpdateConfig(CreateRegistryConfig(agents)); + + RegisterAndWaitForAgents(*runtime, agents); + + runtime->UpdateCurrentTime(TInstant::FromValue(1700000000000000)); // 2023-11-14T22:13:20 + runtime->DispatchEvents({}, 10ms); + + const auto uuid = diskRegistry.AllocateDisk("disk-1", 10_GB) + ->Record.GetDevices(0).GetDeviceUUID(); + + { + auto response = diskRegistry.BackupDiskRegistryState(true); + auto& backup = *response->Record.MutableBackup(); + UNIT_ASSERT_VALUES_EQUAL(1, backup.DisksSize()); + UNIT_ASSERT_VALUES_EQUAL("disk-1", backup.GetDisks(0).GetDiskId()); + + // set the migration start timestamp to a weird value + backup.MutableDisks(0)->SetMigrationStartTs(42); + + diskRegistry.RestoreDiskRegistryState( + std::move(backup), + true // force + ); + + diskRegistry.RebootTablet(); + diskRegistry.WaitReady(); + diskRegistry.SetWritableState(true); + RegisterAgent(*runtime, 0); + } + + const auto actualMigrationStartTs = runtime->GetCurrentTime(); + diskRegistry.ChangeDeviceState(uuid, NProto::DEVICE_STATE_WARNING); + + runtime->AdvanceCurrentTime(UpdateCountersInterval * 2); + runtime->DispatchEvents({}, 10ms); + + { + auto response = diskRegistry.AllocateDisk("disk-1", 10_GB); + UNIT_ASSERT_VALUES_EQUAL(1, response->Record.MigrationsSize()); + } + + const auto maxMigrationTime = runtime->GetAppData(0).Counters + ->GetSubgroup("counters", "blockstore") + ->GetSubgroup("component", "disk_registry") + ->GetCounter("MaxMigrationTime") + ->Val(); + + UNIT_ASSERT_LT(0, maxMigrationTime); + UNIT_ASSERT_GE( + runtime->GetCurrentTime() - actualMigrationStartTs, + TDuration::MicroSeconds(maxMigrationTime) + ); + } } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_session.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_session.cpp index ecc75c57e1b..a8446be4a7c 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_session.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_session.cpp @@ -543,26 +543,25 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) Y_UNIT_TEST(ShouldNotSendAcquireReleaseRequestsToUnavailableAgents) { - const auto agent1 = CreateAgentConfig("agent-1", { - Device("test", "uuid-1", "rack-1", 10_GB) - }); - - const auto agent2 = CreateAgentConfig("agent-2", { - Device("test", "uuid-2", "rack-2", 10_GB) - }); + const TVector agents { + CreateAgentConfig("agent-1", { + Device("test", "uuid-1", "rack-1", 10_GB) + }), + CreateAgentConfig("agent-2", { + Device("test", "uuid-2", "rack-2", 10_GB) + }) + }; auto runtime = TTestRuntimeBuilder() - .WithAgents({ agent1, agent2 }) + .WithAgents(agents) .Build(); TDiskRegistryClient diskRegistry(*runtime); diskRegistry.WaitReady(); diskRegistry.SetWritableState(true); - diskRegistry.UpdateConfig(CreateRegistryConfig(0, {agent1, agent2})); + diskRegistry.UpdateConfig(CreateRegistryConfig(0, agents)); - RegisterAgents(*runtime, 2); - WaitForAgents(*runtime, 2); - WaitForSecureErase(*runtime, {agent1, agent2}); + RegisterAndWaitForAgents(*runtime, agents); { auto response = diskRegistry.AllocateDisk("disk-1", 20_GB); @@ -615,15 +614,36 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) breakAgent("agent-2"); + // It is OK to successfully acquire empty device list { diskRegistry.SendAcquireDiskRequest("disk-1", "session-1"); auto response = diskRegistry.RecvAcquireDiskResponse(); - UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + + UNIT_ASSERT_VALUES_EQUAL(0, response->Record.DevicesSize()); + } + + // Let's acquire one more session + { + diskRegistry.SendAcquireDiskRequest( + "disk-1", + "session-2", + NProto::VOLUME_ACCESS_READ_ONLY); + + auto response = diskRegistry.RecvAcquireDiskResponse(); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); UNIT_ASSERT_VALUES_EQUAL(0, response->Record.DevicesSize()); } diskRegistry.ReleaseDisk("disk-1", "session-1"); + diskRegistry.ReleaseDisk("disk-1", "session-2"); + + diskRegistry.ChangeAgentState("agent-1", NProto::AGENT_STATE_WARNING); + { + auto response = diskRegistry.AcquireDisk("disk-1", "session-1"); + UNIT_ASSERT_VALUES_EQUAL(1, response->Record.DevicesSize()); + } } Y_UNIT_TEST(ShouldAcquireReleaseSession) diff --git a/cloud/blockstore/libs/storage/disk_registry/model/agent_list.cpp b/cloud/blockstore/libs/storage/disk_registry/model/agent_list.cpp index 6beecf85060..8e501efc767 100644 --- a/cloud/blockstore/libs/storage/disk_registry/model/agent_list.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/model/agent_list.cpp @@ -1,6 +1,7 @@ #include "agent_list.h" #include +#include #include @@ -238,9 +239,25 @@ void TAgentList::UpdateDevice( TInstant timestamp, const NProto::TDeviceConfig& oldConfig) { - device.SetState(oldConfig.GetState()); - device.SetStateTs(oldConfig.GetStateTs()); - device.SetStateMessage(oldConfig.GetStateMessage()); + STORAGE_CHECK_PRECONDITION( + device.GetState() == NProto::DEVICE_STATE_ERROR || + device.GetState() == NProto::DEVICE_STATE_ONLINE); + + // If DA reports broken device we should keep the state and state message of + // the device. At the same time we shouldn't remove the 'error' state + // automatically. + if (oldConfig.GetState() != NProto::DEVICE_STATE_ERROR + && device.GetState() == NProto::DEVICE_STATE_ERROR) + { + device.SetStateTs(timestamp.MicroSeconds()); + } else { + // Otherwise, we should keep the old state. + + device.SetState(oldConfig.GetState()); + device.SetStateTs(oldConfig.GetStateTs()); + device.SetStateMessage(oldConfig.GetStateMessage()); + } + device.SetCmsTs(oldConfig.GetCmsTs()); device.SetNodeId(agent.GetNodeId()); device.SetAgentId(agent.GetAgentId()); diff --git a/cloud/blockstore/libs/storage/disk_registry/model/agent_list_ut.cpp b/cloud/blockstore/libs/storage/disk_registry/model/agent_list_ut.cpp index d607017af32..c517c65bbcd 100644 --- a/cloud/blockstore/libs/storage/disk_registry/model/agent_list_ut.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/model/agent_list_ut.cpp @@ -1143,6 +1143,91 @@ Y_UNIT_TEST_SUITE(TAgentListTest) agentList.PublishCounters(now); UNIT_ASSERT_VALUES_EQUAL(30'000, c->Val()); } + + Y_UNIT_TEST_F(ShouldPreserveDeviceErrorState, TFixture) + { + const TString errorMessage = "broken device"; + + NProto::TAgentConfig agentConfig; + agentConfig.SetAgentId("foo-1"); + agentConfig.SetNodeId(1000); + *agentConfig.AddDevices() = CreateDevice("uuid-1", 1_GB, "rack-1"); + + TAgentList agentList = CreateAgentList(); + + const TKnownAgent knownAgent { + .Devices = {{ "uuid-1", CreateDevice("uuid-1", 0) }} + }; + + // Register new agent with one device. + { + const auto timestamp = TInstant::FromValue(100000); + + auto r = agentList.RegisterAgent( + agentConfig, + timestamp, + knownAgent); + + NProto::TAgentConfig& agent = r.Agent; + + UNIT_ASSERT_VALUES_EQUAL(1, r.NewDevices.size()); + + const auto& d = agent.GetDevices(0); + + UNIT_ASSERT_EQUAL(NProto::DEVICE_STATE_ONLINE, d.GetState()); + UNIT_ASSERT_VALUES_EQUAL(timestamp.MicroSeconds(), d.GetStateTs()); + } + + // Break the device. + agentConfig.MutableDevices(0)->SetState(NProto::DEVICE_STATE_ERROR); + agentConfig.MutableDevices(0)->SetStateMessage(errorMessage); + + const auto errorTs = TInstant::FromValue(200000); + + // Register the agent with the broken device. + // Now we expect to see our device in an error state. + { + auto r = agentList.RegisterAgent( + agentConfig, + errorTs, + knownAgent); + + NProto::TAgentConfig& agent = r.Agent; + + UNIT_ASSERT_VALUES_EQUAL(0, r.NewDevices.size()); + + const auto& d = agent.GetDevices(0); + + UNIT_ASSERT_VALUES_EQUAL(errorMessage, d.GetStateMessage()); + UNIT_ASSERT_EQUAL(NProto::DEVICE_STATE_ERROR, d.GetState()); + UNIT_ASSERT_VALUES_EQUAL(errorTs.MicroSeconds(), d.GetStateTs()); + } + + // Fix the device + agentConfig.MutableDevices(0)->SetState(NProto::DEVICE_STATE_ONLINE); + agentConfig.MutableDevices(0)->SetStateMessage(""); + + // Register the agent with fixed device. + // But we expect that the device state remains the same (error). + { + const auto timestamp = TInstant::FromValue(300000); + + auto r = agentList.RegisterAgent( + agentConfig, + timestamp, + knownAgent); + + NProto::TAgentConfig& agent = r.Agent; + + UNIT_ASSERT_VALUES_EQUAL(0, r.NewDevices.size()); + + const auto& d = agent.GetDevices(0); + + UNIT_ASSERT_VALUES_EQUAL(errorMessage, d.GetStateMessage()); + UNIT_ASSERT_EQUAL(NProto::DEVICE_STATE_ERROR, d.GetState()); + UNIT_ASSERT_VALUES_EQUAL(errorTs.MicroSeconds(), d.GetStateTs()); + } + } } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/disk_registry/testlib/test_env.h b/cloud/blockstore/libs/storage/disk_registry/testlib/test_env.h index 2f089dd79e4..0bb6d0c95e9 100644 --- a/cloud/blockstore/libs/storage/disk_registry/testlib/test_env.h +++ b/cloud/blockstore/libs/storage/disk_registry/testlib/test_env.h @@ -879,6 +879,17 @@ class TDiskRegistryClient return request; } + auto CreateRestoreDiskRegistryStateRequest( + NProto::TDiskRegistryStateBackup backup, + bool force) + { + auto request = std::make_unique(); + request->Record.MutableBackup()->Swap(&backup); + request->Record.SetForce(force); + + return request; + } + auto CreateFinishMigrationRequest( const TString& diskId, const TString& sourceId, diff --git a/cloud/blockstore/libs/storage/disk_registry/testlib/test_state.cpp b/cloud/blockstore/libs/storage/disk_registry/testlib/test_state.cpp index 93cc33b192c..858656e6d99 100644 --- a/cloud/blockstore/libs/storage/disk_registry/testlib/test_state.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/testlib/test_state.cpp @@ -597,6 +597,14 @@ TDiskRegistryStateBuilder& TDiskRegistryStateBuilder::WithPlacementGroups( return *this; } +TDiskRegistryStateBuilder& TDiskRegistryStateBuilder::WithErrorNotifications( + TVector notifications) +{ + ErrorNotifications = std::move(notifications); + + return *this; +} + TDiskRegistryStateBuilder& TDiskRegistryStateBuilder::AddDevicePoolConfig( TString name, ui64 allocationUnit, diff --git a/cloud/blockstore/libs/storage/disk_registry/testlib/test_state.h b/cloud/blockstore/libs/storage/disk_registry/testlib/test_state.h index aafa66947f7..6e239a8a7a6 100644 --- a/cloud/blockstore/libs/storage/disk_registry/testlib/test_state.h +++ b/cloud/blockstore/libs/storage/disk_registry/testlib/test_state.h @@ -284,6 +284,9 @@ struct TDiskRegistryStateBuilder TDiskRegistryStateBuilder& WithPlacementGroups( TVector groups); + TDiskRegistryStateBuilder& WithErrorNotifications( + TVector notifications); + TDiskRegistryStateBuilder& AddDevicePoolConfig( TString name, ui64 allocationUnit, diff --git a/cloud/blockstore/libs/storage/disk_registry/ya.make b/cloud/blockstore/libs/storage/disk_registry/ya.make index 411aaf8fd08..54ac45b9d8a 100644 --- a/cloud/blockstore/libs/storage/disk_registry/ya.make +++ b/cloud/blockstore/libs/storage/disk_registry/ya.make @@ -79,7 +79,6 @@ PEERDIR( contrib/ydb/core/scheme contrib/ydb/core/tablet contrib/ydb/core/tablet_flat - contrib/ydb/core/testlib/basics ) END() diff --git a/cloud/blockstore/libs/storage/disk_registry_proxy/ya.make b/cloud/blockstore/libs/storage/disk_registry_proxy/ya.make index c9a19178bb4..a6bc54ec493 100644 --- a/cloud/blockstore/libs/storage/disk_registry_proxy/ya.make +++ b/cloud/blockstore/libs/storage/disk_registry_proxy/ya.make @@ -17,8 +17,6 @@ PEERDIR( contrib/ydb/core/mon contrib/ydb/core/tablet contrib/ydb/core/tablet_flat - contrib/ydb/core/testlib - contrib/ydb/core/testlib/basics ) END() diff --git a/cloud/blockstore/libs/storage/init/common/ya.make b/cloud/blockstore/libs/storage/init/common/ya.make index 968038a336d..634f46f8d6d 100644 --- a/cloud/blockstore/libs/storage/init/common/ya.make +++ b/cloud/blockstore/libs/storage/init/common/ya.make @@ -9,6 +9,7 @@ PEERDIR( cloud/blockstore/libs/storage/api cloud/storage/core/libs/kikimr + cloud/storage/core/libs/version_ydb contrib/ydb/library/actors/core contrib/ydb/library/actors/util diff --git a/cloud/blockstore/libs/storage/init/disk_agent/actorsystem.cpp b/cloud/blockstore/libs/storage/init/disk_agent/actorsystem.cpp index 65e609a6486..4d5fa6e56fa 100644 --- a/cloud/blockstore/libs/storage/init/disk_agent/actorsystem.cpp +++ b/cloud/blockstore/libs/storage/init/disk_agent/actorsystem.cpp @@ -101,6 +101,7 @@ class TStorageServicesInitializer final auto diskAgent = CreateDiskAgent( Args.StorageConfig, Args.DiskAgentConfig, + Args.RdmaConfig, Args.Spdk, Args.Allocator, Args.AioStorageProvider, diff --git a/cloud/blockstore/libs/storage/init/disk_agent/actorsystem.h b/cloud/blockstore/libs/storage/init/disk_agent/actorsystem.h index f86f2c789a4..0fc8101ff3f 100644 --- a/cloud/blockstore/libs/storage/init/disk_agent/actorsystem.h +++ b/cloud/blockstore/libs/storage/init/disk_agent/actorsystem.h @@ -32,6 +32,7 @@ struct TDiskAgentActorSystemArgs TStorageConfigPtr StorageConfig; TDiskAgentConfigPtr DiskAgentConfig; + NRdma::TRdmaConfigPtr RdmaConfig; TDiskRegistryProxyConfigPtr DiskRegistryProxyConfig; ILoggingServicePtr Logging; diff --git a/cloud/blockstore/libs/storage/init/server/actorsystem.cpp b/cloud/blockstore/libs/storage/init/server/actorsystem.cpp index e59df3022e7..ceb72c27ead 100644 --- a/cloud/blockstore/libs/storage/init/server/actorsystem.cpp +++ b/cloud/blockstore/libs/storage/init/server/actorsystem.cpp @@ -228,6 +228,7 @@ class TStorageServicesInitializer final auto storageUserStats = NCloud::NStorage::NUserStats::CreateStorageUserStats( + TBlockStoreComponents::USER_STATS, "blockstore", "BlockStore", Args.UserCounterProviders); @@ -278,6 +279,7 @@ class TStorageServicesInitializer final auto diskAgent = CreateDiskAgent( Args.StorageConfig, Args.DiskAgentConfig, + Args.RdmaConfig, Args.Spdk, Args.Allocator, Args.AioStorageProvider, diff --git a/cloud/blockstore/libs/storage/init/server/actorsystem.h b/cloud/blockstore/libs/storage/init/server/actorsystem.h index 8d4ed0b387a..9b67ec4d937 100644 --- a/cloud/blockstore/libs/storage/init/server/actorsystem.h +++ b/cloud/blockstore/libs/storage/init/server/actorsystem.h @@ -41,6 +41,7 @@ struct TServerActorSystemArgs TDiagnosticsConfigPtr DiagnosticsConfig; TStorageConfigPtr StorageConfig; TDiskAgentConfigPtr DiskAgentConfig; + NRdma::TRdmaConfigPtr RdmaConfig; TDiskRegistryProxyConfigPtr DiskRegistryProxyConfig; ILoggingServicePtr Logging; diff --git a/cloud/blockstore/libs/storage/init/server/ya.make b/cloud/blockstore/libs/storage/init/server/ya.make index 25bd28585ba..1a711f60c86 100644 --- a/cloud/blockstore/libs/storage/init/server/ya.make +++ b/cloud/blockstore/libs/storage/init/server/ya.make @@ -39,7 +39,6 @@ PEERDIR( contrib/ydb/library/actors/util contrib/ydb/core/base - contrib/ydb/core/driver_lib/run contrib/ydb/core/load_test contrib/ydb/core/mind contrib/ydb/core/mon diff --git a/cloud/blockstore/libs/storage/partition/model/blob_unique_id_with_range.cpp b/cloud/blockstore/libs/storage/partition/model/blob_to_confirm.cpp similarity index 90% rename from cloud/blockstore/libs/storage/partition/model/blob_unique_id_with_range.cpp rename to cloud/blockstore/libs/storage/partition/model/blob_to_confirm.cpp index 7e1c2337d33..a409d24e0af 100644 --- a/cloud/blockstore/libs/storage/partition/model/blob_unique_id_with_range.cpp +++ b/cloud/blockstore/libs/storage/partition/model/blob_to_confirm.cpp @@ -1,11 +1,11 @@ -#include "blob_unique_id_with_range.h" +#include "blob_to_confirm.h" namespace NCloud::NBlockStore::NStorage::NPartition { //////////////////////////////////////////////////////////////////////////////// bool Overlaps( - const TCommitIdToBlobUniqueIdWithRange& blobs, + const TCommitIdToBlobsToConfirm& blobs, ui64 lowCommitId, ui64 highCommitId, const TBlockRange32& blockRange) diff --git a/cloud/blockstore/libs/storage/partition/model/blob_unique_id_with_range.h b/cloud/blockstore/libs/storage/partition/model/blob_to_confirm.h similarity index 61% rename from cloud/blockstore/libs/storage/partition/model/blob_unique_id_with_range.h rename to cloud/blockstore/libs/storage/partition/model/blob_to_confirm.h index d9aed76c194..e74e06ad714 100644 --- a/cloud/blockstore/libs/storage/partition/model/blob_unique_id_with_range.h +++ b/cloud/blockstore/libs/storage/partition/model/blob_to_confirm.h @@ -11,24 +11,27 @@ namespace NCloud::NBlockStore::NStorage::NPartition { //////////////////////////////////////////////////////////////////////////////// -struct TBlobUniqueIdWithRange +struct TBlobToConfirm { - ui64 UniqueId = 0; + ui64 UniqueId; TBlockRange32 BlockRange; + TVector Checksums; - TBlobUniqueIdWithRange() = default; - - TBlobUniqueIdWithRange(ui64 uniqueId, const TBlockRange32& blockRange) + TBlobToConfirm( + ui64 uniqueId, + const TBlockRange32& blockRange, + const TVector& checksums) : UniqueId(uniqueId) , BlockRange(blockRange) + , Checksums(checksums) {} + }; -using TCommitIdToBlobUniqueIdWithRange = - THashMap>; +using TCommitIdToBlobsToConfirm = THashMap>; bool Overlaps( - const TCommitIdToBlobUniqueIdWithRange& blobs, + const TCommitIdToBlobsToConfirm& blobs, ui64 lowCommitId, ui64 highCommitId, const TBlockRange32& blockRange); diff --git a/cloud/blockstore/libs/storage/partition/model/block.h b/cloud/blockstore/libs/storage/partition/model/block.h index 05994b7550d..069703120e0 100644 --- a/cloud/blockstore/libs/storage/partition/model/block.h +++ b/cloud/blockstore/libs/storage/partition/model/block.h @@ -82,4 +82,16 @@ struct IBlocksIndexVisitor ui16 blobOffset) = 0; }; +struct IExtendedBlocksIndexVisitor +{ + virtual ~IExtendedBlocksIndexVisitor() = default; + + virtual bool Visit( + ui32 blockIndex, + ui64 commitId, + const TPartialBlobId& blobId, + ui16 blobOffset, + ui32 checksum) = 0; +}; + } // namespace NCloud::NBlockStore::NStorage::NPartition diff --git a/cloud/blockstore/libs/storage/partition/model/ya.make b/cloud/blockstore/libs/storage/partition/model/ya.make index da9adb48031..21909ed34b9 100644 --- a/cloud/blockstore/libs/storage/partition/model/ya.make +++ b/cloud/blockstore/libs/storage/partition/model/ya.make @@ -8,7 +8,7 @@ GENERATE_ENUM_SERIALIZATION(operation_status.h) SRCS( barrier.cpp blob_index.cpp - blob_unique_id_with_range.cpp + blob_to_confirm.cpp block.cpp block_index.cpp block_mask.cpp diff --git a/cloud/blockstore/libs/storage/partition/part_actor.cpp b/cloud/blockstore/libs/storage/partition/part_actor.cpp index 8508e7c6159..405bd424542 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor.cpp @@ -3,7 +3,9 @@ #include #include #include + #include +#include #include #include @@ -351,20 +353,35 @@ void TPartitionActor::KillActors(const TActorContext& ctx) } } -void TPartitionActor::AddTransaction(TRequestInfo& requestInfo) +void TPartitionActor::AddTransaction( + TRequestInfo& transaction, + TRequestInfo::TCancelRoutine cancelRoutine) { - requestInfo.Ref(); + transaction.CancelRoutine = cancelRoutine; + + transaction.Ref(); + + STORAGE_VERIFY( + transaction.Empty(), + TWellKnownEntityTypes::TABLET, + TabletID()); - Y_ABORT_UNLESS(requestInfo.Empty()); - ActiveTransactions.PushBack(&requestInfo); + ActiveTransactions.PushBack(&transaction); } void TPartitionActor::RemoveTransaction(TRequestInfo& requestInfo) { - Y_ABORT_UNLESS(!requestInfo.Empty()); + STORAGE_VERIFY( + !requestInfo.Empty(), + TWellKnownEntityTypes::TABLET, + TabletID()); + requestInfo.Unlink(); - Y_ABORT_UNLESS(requestInfo.RefCount() > 1); + STORAGE_VERIFY( + requestInfo.RefCount() > 1, + TWellKnownEntityTypes::TABLET, + TabletID()); requestInfo.UnRef(); } @@ -372,9 +389,11 @@ void TPartitionActor::TerminateTransactions(const TActorContext& ctx) { while (ActiveTransactions) { TRequestInfo* requestInfo = ActiveTransactions.PopFront(); + STORAGE_VERIFY( + requestInfo->RefCount() >= 1, + TWellKnownEntityTypes::TABLET, + TabletID()); requestInfo->CancelRequest(ctx); - - Y_ABORT_UNLESS(requestInfo->RefCount() >= 1); requestInfo->UnRef(); } } @@ -383,7 +402,10 @@ void TPartitionActor::ReleaseTransactions() { while (ActiveTransactions) { TRequestInfo* requestInfo = ActiveTransactions.PopFront(); - Y_ABORT_UNLESS(requestInfo->RefCount() >= 1); + STORAGE_VERIFY( + requestInfo->RefCount() >= 1, + TWellKnownEntityTypes::TABLET, + TabletID()); requestInfo->UnRef(); } } diff --git a/cloud/blockstore/libs/storage/partition/part_actor.h b/cloud/blockstore/libs/storage/partition/part_actor.h index 168366ec1bc..112e5e73e26 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor.h +++ b/cloud/blockstore/libs/storage/partition/part_actor.h @@ -156,8 +156,8 @@ class TPartitionActor final NProto::TPartitionConfig partitionConfig, EStorageAccessMode storageAccessMode, ui32 siblingCount, - const NActors::TActorId& VolumeActorId); - ~TPartitionActor(); + const NActors::TActorId& volumeActorId); + ~TPartitionActor() override; static constexpr ui32 LogComponent = TBlockStoreComponents::PARTITION; using TCounters = TPartitionCounters; @@ -306,7 +306,25 @@ class TPartitionActor final bool deleteOnlyData); void KillActors(const NActors::TActorContext& ctx); - void AddTransaction(TRequestInfo& transaction); + void AddTransaction( + TRequestInfo& transaction, + TRequestInfo::TCancelRoutine cancelRoutine); + + template + void AddTransaction(TRequestInfo& transaction) + { + auto cancelRoutine = [] ( + const NActors::TActorContext& ctx, + TRequestInfo& requestInfo) + { + auto response = std::make_unique( + MakeError(E_REJECTED, "tablet is shutting down")); + + NCloud::Reply(ctx, requestInfo, std::move(response)); + }; + + AddTransaction(transaction, cancelRoutine); + } void RemoveTransaction(TRequestInfo& transaction); void TerminateTransactions(const NActors::TActorContext& ctx); void ReleaseTransactions(); diff --git a/cloud/blockstore/libs/storage/partition/part_actor_addblobs.cpp b/cloud/blockstore/libs/storage/partition/part_actor_addblobs.cpp index d9d42dd75ff..37237667a8b 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_addblobs.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_addblobs.cpp @@ -616,7 +616,7 @@ void TPartitionActor::HandleAddBlobs( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -629,7 +629,7 @@ void TPartitionActor::HandleAddBlobs( "AddBlobs", requestInfo->CallContext->RequestId); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition/part_actor_addconfirmedblobs.cpp b/cloud/blockstore/libs/storage/partition/part_actor_addconfirmedblobs.cpp index d588d335d06..a5e07c2af51 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_addconfirmedblobs.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_addconfirmedblobs.cpp @@ -160,7 +160,7 @@ void TAddConfirmedBlobsActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); ReplyAndDie(ctx, error); } @@ -286,7 +286,7 @@ void TPartitionActor::HandleAddConfirmedBlobs( MakePartialBlobId(commitId, blob.UniqueId), blob.BlockRange, TBlockMask(), // skipMask - TVector() /* TODO: checksums */); + blob.Checksums); } auto request = std::make_unique( diff --git a/cloud/blockstore/libs/storage/partition/part_actor_addgarbage.cpp b/cloud/blockstore/libs/storage/partition/part_actor_addgarbage.cpp index 0418b333c8b..c54e557288f 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_addgarbage.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_addgarbage.cpp @@ -23,7 +23,7 @@ void TPartitionActor::HandleAddGarbage( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -36,7 +36,7 @@ void TPartitionActor::HandleAddGarbage( "AddGarbage", requestInfo->CallContext->RequestId); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition/part_actor_addunconfirmedblobs.cpp b/cloud/blockstore/libs/storage/partition/part_actor_addunconfirmedblobs.cpp index bacfc21c900..61292a39a86 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_addunconfirmedblobs.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_addunconfirmedblobs.cpp @@ -24,7 +24,7 @@ void TPartitionActor::HandleAddUnconfirmedBlobs( auto* msg = ev->Get(); using TMethod = TEvPartitionPrivate::TAddUnconfirmedBlobsMethod; - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -37,7 +37,7 @@ void TPartitionActor::HandleAddUnconfirmedBlobs( "AddUnconfirmedBlobs", requestInfo->CallContext->RequestId); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition/part_actor_changedblocks.cpp b/cloud/blockstore/libs/storage/partition/part_actor_changedblocks.cpp index abf19235771..88af3dbaf25 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_changedblocks.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_changedblocks.cpp @@ -230,7 +230,7 @@ void TGetChangedBlocksActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); auto response = std::make_unique(error); @@ -305,7 +305,7 @@ void TPartitionActor::HandleGetChangedBlocks( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -426,7 +426,7 @@ void TPartitionActor::HandleGetChangedBlocks( highCommitId, DescribeRange(readRange).data()); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition/part_actor_checkpoint.cpp b/cloud/blockstore/libs/storage/partition/part_actor_checkpoint.cpp index 8b9b68b4d94..a223595bf87 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_checkpoint.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_checkpoint.cpp @@ -48,7 +48,7 @@ void TPartitionActor::HandleCreateCheckpoint( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -74,7 +74,7 @@ void TPartitionActor::HandleCreateCheckpoint( return; } - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); TString idempotenceId = GetIdempotenceId(*msg); @@ -188,7 +188,7 @@ void TPartitionActor::DeleteCheckpoint( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -221,7 +221,7 @@ void TPartitionActor::DeleteCheckpoint( return; } - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); auto tx = CreateTx( std::move(requestInfo), diff --git a/cloud/blockstore/libs/storage/partition/part_actor_cleanup.cpp b/cloud/blockstore/libs/storage/partition/part_actor_cleanup.cpp index ca23b58f1bf..d73f624aa4b 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_cleanup.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_cleanup.cpp @@ -97,7 +97,7 @@ void TPartitionActor::HandleCleanup( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -173,7 +173,7 @@ void TPartitionActor::HandleCleanup( State->GetCleanupState().SetStatus(EOperationStatus::Started); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition/part_actor_collectgarbage.cpp b/cloud/blockstore/libs/storage/partition/part_actor_collectgarbage.cpp index 47e4edb1881..f92944b9b47 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_collectgarbage.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_collectgarbage.cpp @@ -562,7 +562,7 @@ void TPartitionActor::HandleCollectGarbage( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -612,7 +612,7 @@ void TPartitionActor::HandleCollectGarbage( State->GetCollectGarbageState().SetStatus(EOperationStatus::Started); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx(ctx, requestInfo, commitId); return; diff --git a/cloud/blockstore/libs/storage/partition/part_actor_compaction.cpp b/cloud/blockstore/libs/storage/partition/part_actor_compaction.cpp index ced4ddf53d0..a32eb598475 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_compaction.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_compaction.cpp @@ -506,6 +506,7 @@ void TCompactionActor::WriteBlobs(const TActorContext& ctx) auto request = std::make_unique( rc.DataBlobId, rc.BlobContent.GetGuardedSgList(), + 0, // blockSizeForChecksums true); // async if (!RequestInfo->CallContext->LWOrbit.Fork(request->CallContext->LWOrbit)) { @@ -888,7 +889,7 @@ void TCompactionActor::HandlePoisonPill( Y_UNUSED(ev); auto response = std::make_unique( - MakeError(E_REJECTED, "Tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); ReplyAndDie(ctx, std::move(response)); } @@ -1199,7 +1200,7 @@ void TPartitionActor::HandleCompaction( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -1318,7 +1319,7 @@ void TPartitionActor::HandleCompaction( State->GetCleanupQueue().AcquireBarrier(commitId); State->GetGarbageQueue().AcquireBarrier(commitId); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); auto tx = CreateTx( requestInfo, diff --git a/cloud/blockstore/libs/storage/partition/part_actor_compactrange.cpp b/cloud/blockstore/libs/storage/partition/part_actor_compactrange.cpp index 3aa20087e75..ac5f341f8de 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_compactrange.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_compactrange.cpp @@ -165,7 +165,7 @@ void TForcedCompactionActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); ReplyAndDie(ctx, error); } diff --git a/cloud/blockstore/libs/storage/partition/part_actor_confirmblobs.cpp b/cloud/blockstore/libs/storage/partition/part_actor_confirmblobs.cpp index d6ecc55a430..2ce171a25cd 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_confirmblobs.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_confirmblobs.cpp @@ -12,6 +12,8 @@ #include #include +#include + namespace NCloud::NBlockStore::NStorage::NPartition { using namespace NActors; @@ -37,6 +39,7 @@ class TConfirmBlobsActor final : public TActorBootstrapped { private: + const ui64 StartCycleCount = GetCycleCount(); const ui64 TabletId = 0; const TActorId Tablet; const TVector Requests; @@ -108,6 +111,7 @@ void TConfirmBlobsActor::NotifyAndDie(const TActorContext& ctx) { auto ev = std::make_unique( std::move(Error), + StartCycleCount, std::move(UnrecoverableBlobs)); NCloud::Send(ctx, Tablet, std::move(ev)); Die(ctx); @@ -189,10 +193,8 @@ void TPartitionActor::ConfirmBlobs(const TActorContext& ctx) TVector requests; - for (const auto& entry: State->GetUnconfirmedBlobs()) { - auto commitId = entry.first; - - for (const auto& blob: entry.second) { + for (const auto& [commitId, blobs]: State->GetUnconfirmedBlobs()) { + for (const auto& blob: blobs) { auto blobId = MakePartialBlobId(commitId, blob.UniqueId); auto proxy = Info()->BSProxyIDForChannel( blobId.Channel(), blobId.Generation() @@ -233,7 +235,10 @@ void TPartitionActor::HandleConfirmBlobsCompleted( "[%lu] ConfirmBlobs: start tx", TabletID()); - ExecuteTx(ctx, std::move(msg->UnrecoverableBlobs)); + ExecuteTx( + ctx, + msg->StartCycleCount, + std::move(msg->UnrecoverableBlobs)); } bool TPartitionActor::PrepareConfirmBlobs( @@ -268,13 +273,33 @@ void TPartitionActor::CompleteConfirmBlobs( const TActorContext& ctx, TTxPartition::TConfirmBlobs& args) { - Y_UNUSED(args); - LOG_INFO(ctx, TBlockStoreComponents::PARTITION, "[%lu] ConfirmBlobs: complete tx", TabletID()); BlobsConfirmed(ctx); + + const auto duration = + CyclesToDurationSafe(GetCycleCount() - args.StartCycleCount); + + PartCounters->RequestCounters.ConfirmBlobs.AddRequest( + duration.MicroSeconds()); + + IProfileLog::TSysReadWriteRequest request; + request.RequestType = ESysRequestType::ConfirmBlobs; + request.Duration = duration; + + for (const auto& [_, blobs]: State->GetConfirmedBlobs()) { + for (const auto& blob: blobs) { + request.Ranges.push_back(ConvertRangeSafe(blob.BlockRange)); + } + } + + IProfileLog::TRecord record; + record.DiskId = State->GetConfig().GetDiskId(); + record.Ts = ctx.Now() - duration; + record.Request = request; + ProfileLog->Write(std::move(record)); } } // namespace NCloud::NBlockStore::NStorage::NPartition diff --git a/cloud/blockstore/libs/storage/partition/part_actor_deletegarbage.cpp b/cloud/blockstore/libs/storage/partition/part_actor_deletegarbage.cpp index ec9143bf081..978730c00f0 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_deletegarbage.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_deletegarbage.cpp @@ -19,7 +19,7 @@ void TPartitionActor::HandleDeleteGarbage( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -32,7 +32,7 @@ void TPartitionActor::HandleDeleteGarbage( "DeleteGarbage", requestInfo->CallContext->RequestId); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition/part_actor_describeblocks.cpp b/cloud/blockstore/libs/storage/partition/part_actor_describeblocks.cpp index 84a24f89421..035a49e0536 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_describeblocks.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_describeblocks.cpp @@ -107,7 +107,7 @@ void TPartitionActor::DescribeBlocks( commitId, DescribeRange(describeRange).data()); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, @@ -122,7 +122,7 @@ void TPartitionActor::HandleDescribeBlocks( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); diff --git a/cloud/blockstore/libs/storage/partition/part_actor_flush.cpp b/cloud/blockstore/libs/storage/partition/part_actor_flush.cpp index 59289f02104..d16b337b9e1 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_flush.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_flush.cpp @@ -194,6 +194,7 @@ void TFlushActor::WriteBlobs(const TActorContext& ctx) auto request = std::make_unique( req.BlobId, req.BlobContent.GetGuardedSgList(), + 0, // blockSizeForChecksums true); // async if (!RequestInfo->CallContext->LWOrbit.Fork(request->CallContext->LWOrbit)) { @@ -340,7 +341,7 @@ void TFlushActor::HandlePoisonPill( Y_UNUSED(ev); auto respose = std::make_unique( - MakeError(E_REJECTED, "Tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); ReplyAndDie(ctx, std::move(respose)); } diff --git a/cloud/blockstore/libs/storage/partition/part_actor_getusedblocks.cpp b/cloud/blockstore/libs/storage/partition/part_actor_getusedblocks.cpp index 54d5cbb1c75..97eef7e9393 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_getusedblocks.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_getusedblocks.cpp @@ -21,7 +21,7 @@ void TPartitionActor::HandleGetUsedBlocks( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -34,7 +34,7 @@ void TPartitionActor::HandleGetUsedBlocks( "GetUsedBlocks", requestInfo->CallContext->RequestId); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition/part_actor_loadstate.cpp b/cloud/blockstore/libs/storage/partition/part_actor_loadstate.cpp index 6b7551b06da..0b6c51bdf62 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_loadstate.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_loadstate.cpp @@ -228,7 +228,13 @@ void TPartitionActor::CompleteLoadState( Y_ABORT_UNLESS(State->GetGarbageQueue().AddNewBlobs(args.NewBlobs)); Y_ABORT_UNLESS(State->GetGarbageQueue().AddGarbageBlobs(args.GarbageBlobs)); - if (State->GetBaseDiskId()) { + // Logical used blocks calculation is not implemented for proxy overlay + // disks with multi-partition base disks, so we disable it. + // Proxy overlay disks are system disks that are used for snapshot creation + // or disk relocation. + // Also, logical used blocks calculation is not needed for such disks + // because they are temporary and do not belong to a user. + if (State->GetBaseDiskId() && !partitionConfig.GetIsSystem()) { if (args.ReadLogicalUsedBlocks) { State->GetLogicalUsedBlocks() = std::move(args.LogicalUsedBlocks); State->AccessStats().SetLogicalUsedBlocksCount( @@ -287,12 +293,12 @@ void TPartitionActor::HandleGetUsedBlocksResponse( Suicide(ctx); return; - } else { - LOG_DEBUG(ctx, TBlockStoreComponents::PARTITION, - "[%lu] LoadState completed", - TabletID()); } + LOG_DEBUG(ctx, TBlockStoreComponents::PARTITION, + "[%lu] LoadState completed", + TabletID()); + for (const auto& block: msg->Record.GetUsedBlocks()) { State->GetLogicalUsedBlocks().Merge( TCompressedBitmap::TSerializedChunk{ diff --git a/cloud/blockstore/libs/storage/partition/part_actor_metadata_rebuild.cpp b/cloud/blockstore/libs/storage/partition/part_actor_metadata_rebuild.cpp index ae7c2233479..6eebd8d88fc 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_metadata_rebuild.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_metadata_rebuild.cpp @@ -96,7 +96,7 @@ void TPartitionActor::HandleGetRebuildMetadataStatus( progress.SetIsCompleted(p.IsCompleted); } } else { - result = MakeError(E_REJECTED, "Tablet is dead"); + result = MakeError(E_REJECTED, "tablet is shutting down"); }; *response->Record.MutableError() = std::move(result); diff --git a/cloud/blockstore/libs/storage/partition/part_actor_metadata_rebuild_blockcount.cpp b/cloud/blockstore/libs/storage/partition/part_actor_metadata_rebuild_blockcount.cpp index ba5eb982b18..86b6950df1e 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_metadata_rebuild_blockcount.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_metadata_rebuild_blockcount.cpp @@ -220,7 +220,7 @@ void TMetadataRebuildBlockCountActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); NotifyCompleted(ctx, error); } @@ -261,7 +261,7 @@ void TPartitionActor::HandleMetadataRebuildBlockCount( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -311,7 +311,7 @@ void TPartitionActor::HandleMetadataRebuildBlockCount( gen, step); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx(ctx, CreateTx( requestInfo, diff --git a/cloud/blockstore/libs/storage/partition/part_actor_metadata_rebuild_usedblocks.cpp b/cloud/blockstore/libs/storage/partition/part_actor_metadata_rebuild_usedblocks.cpp index 90a0caaaf51..88b3f7652ac 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_metadata_rebuild_usedblocks.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_metadata_rebuild_usedblocks.cpp @@ -184,7 +184,7 @@ void TMetadataRebuildUsedBlocksActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); NotifyCompleted(ctx, error); } @@ -224,7 +224,7 @@ void TPartitionActor::HandleMetadataRebuildUsedBlocks( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -273,7 +273,7 @@ void TPartitionActor::HandleMetadataRebuildUsedBlocks( TabletID(), DescribeRange(blockRange).data()); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx(ctx, CreateTx(requestInfo, blockRange)); } diff --git a/cloud/blockstore/libs/storage/partition/part_actor_monitoring.cpp b/cloud/blockstore/libs/storage/partition/part_actor_monitoring.cpp index fa6d82b0b78..05ce4327029 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_monitoring.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_monitoring.cpp @@ -7,7 +7,9 @@ #include #include #include + #include +#include #include #include @@ -111,95 +113,37 @@ void DumpChannels( const TDiagnosticsConfig& config, ui64 hiveTabletId) { - HTML(out) { - DIV() { - out << "

Channel history

"; - } - - TABLE_CLASS("table table-condensed") { - TABLEBODY() { - for (const auto& channel: storage.Channels) { - TABLER() { - TABLED() { out << "Channel: " << channel.Channel; } - TABLED() { out << "StoragePool: " << channel.StoragePool; } - - if (auto latestEntry = channel.LatestEntry()) { - TABLED() { out << "Id: " << latestEntry->GroupID; } - TABLED() { out << "Gen: " << latestEntry->FromGeneration; } - const auto& cps = - state.GetConfig().GetExplicitChannelProfiles(); - if (cps.size()) { - // we need this check for legacy volumes - // see NBS-752 - if (channel.Channel < static_cast(cps.size())) { - const auto dataKind = static_cast( - cps[channel.Channel].GetDataKind()); - const auto& poolKind = - cps[channel.Channel].GetPoolKind(); - TABLED() { out << "PoolKind: " << poolKind; } - TABLED() { out << "DataKind: " << dataKind; } - } else { - // we need to output 2 cells, otherwise table - // markup will be a bit broken - TABLED() { out << "Ghost"; } - TABLED() { out << "Channel"; } - } - } - TABLED() { - TStringBuf label; - TStringBuf color; - if (state.CheckPermissions(channel.Channel, EChannelPermission::SystemWritesAllowed)) { - if (state.CheckPermissions(channel.Channel, EChannelPermission::UserWritesAllowed)) { - color = "green"; - label = "Writable"; - } else { - color = "yellow"; - label = "SystemWritable"; - } - } else { - if (state.CheckPermissions(channel.Channel, EChannelPermission::UserWritesAllowed)) { - color = "pink"; - label = "WeirdState"; - } else { - color = "orange"; - label = "Readonly"; - } - } - - SPAN_CLASS_STYLE("label", TStringBuilder() << "background-color: " << color) { - out << label; - } - } - TABLED() { - out << "Status"; - } - TABLED() { - out << "Graphs"; - } - TABLED() { - BuildReassignChannelButton( - out, - hiveTabletId, - storage.TabletID, - channel.Channel); - } - } - } - } - } - } + TVector channelInfos; + const auto& cps = state.GetConfig().GetExplicitChannelProfiles(); + for (int c = 0; c < cps.size(); ++c) { + const auto& cp = cps[c]; + const auto dataKind = + static_cast(cps[c].GetDataKind()); + channelInfos.push_back({ + cp.GetPoolKind(), + TStringBuilder() << dataKind, + state.CheckPermissions(c, EChannelPermission::UserWritesAllowed), + state.CheckPermissions(c, EChannelPermission::SystemWritesAllowed), + }); } + NCloud::NStorage::DumpChannels( + out, + channelInfos, + storage, + [&] (ui32 groupId, const TString& storagePool) { + return GetMonitoringYDBGroupUrl( + config, + groupId, + storagePool); + }, + [&] (IOutputStream& out, ui64 hiveTabletId, ui64 tabletId, ui32 c) { + BuildReassignChannelButton( + out, + hiveTabletId, + tabletId, + c); + }, + hiveTabletId); } void DumpCheckpoints( diff --git a/cloud/blockstore/libs/storage/partition/part_actor_monitoring_check.cpp b/cloud/blockstore/libs/storage/partition/part_actor_monitoring_check.cpp index 1fa412eb15f..bca30a37de4 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_monitoring_check.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_monitoring_check.cpp @@ -24,6 +24,7 @@ namespace { template class TCheckIndexVisitor final : public IBlocksIndexVisitor + , public IExtendedBlocksIndexVisitor { private: TTxPartition::TCheckIndex& Args; @@ -47,6 +48,18 @@ class TCheckIndexVisitor final return true; } + + bool Visit( + ui32 blockIndex, + ui64 commitId, + const TPartialBlobId& blobId, + ui16 blobOffset, + ui32 checksum) override + { + Y_UNUSED(checksum); + + return Visit(blockIndex, commitId, blobId, blobOffset); + } }; //////////////////////////////////////////////////////////////////////////////// @@ -165,7 +178,10 @@ bool TPartitionActor::PrepareCheckIndex( ); TCheckIndexVisitor visitorBlobs(args); - ready &= db.FindBlocksInBlobsIndex(visitorBlobs, args.BlockRange); + ready &= db.FindBlocksInBlobsIndex( + visitorBlobs, + State->GetMaxBlocksInBlob(), + args.BlockRange); return ready; } diff --git a/cloud/blockstore/libs/storage/partition/part_actor_monitoring_describe.cpp b/cloud/blockstore/libs/storage/partition/part_actor_monitoring_describe.cpp index 5f41ebda25f..c83f1111ad8 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_monitoring_describe.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_monitoring_describe.cpp @@ -59,7 +59,7 @@ class TDescribeRangeVisitor final //////////////////////////////////////////////////////////////////////////////// class TDescribeBlobVisitor final - : public IBlocksIndexVisitor + : public IExtendedBlocksIndexVisitor { private: TTxPartition::TDescribeBlob& Args; @@ -73,18 +73,18 @@ class TDescribeBlobVisitor final ui32 blockIndex, ui64 commitId, const TPartialBlobId& blobId, - ui16 blobOffset) override + ui16 blobOffset, + ui32 checksum) override { Y_UNUSED(blobId); - Args.MarkBlock(blockIndex, commitId, blobOffset); + Args.MarkBlock(blockIndex, commitId, blobOffset, checksum); return true; } }; } // namespace - //////////////////////////////////////////////////////////////////////////////// bool TPartitionActor::PrepareDescribeRange( @@ -138,23 +138,23 @@ void TPartitionActor::CompleteDescribeRange( { using namespace NMonitoringUtils; - auto comparer = [] ( + auto cmp = [] ( const TTxPartition::TDescribeRange::TBlockMark& l, const TTxPartition::TDescribeRange::TBlockMark& r) { - if (l.BlockIndex == r.BlockIndex) { - if (l.CommitId == r.CommitId) { - return l.BlobOffset < r.BlobOffset; - } else { - // last entries goes first - return l.CommitId > r.CommitId; - } - } else { + if (l.BlockIndex != r.BlockIndex) { return l.BlockIndex < r.BlockIndex; } + + if (l.CommitId != r.CommitId) { + // last entries go first + return l.CommitId > r.CommitId; + } + + return l.BlobOffset < r.BlobOffset; }; - Sort(args.BlockMarks, comparer); + Sort(args.BlockMarks, cmp); TStringStream out; DumpDefaultHeader(out, *Info(), SelfId().NodeId(), *DiagnosticsConfig); @@ -275,7 +275,10 @@ bool TPartitionActor::PrepareDescribeBlob( TPartitionDatabase db(tx.DB); TDescribeBlobVisitor visitor(args); - return db.FindBlocksInBlobsIndex(visitor, args.BlobId); + return db.FindBlocksInBlobsIndex( + visitor, + State->GetMaxBlocksInBlob(), + args.BlobId); } void TPartitionActor::ExecuteDescribeBlob( @@ -304,13 +307,26 @@ void TPartitionActor::CompleteDescribeBlob( TABLER() { TABLED() { out << "# Block"; } TABLED() { out << "Offset"; } + TABLED() { out << "Checksum"; } } } TABLEBODY() { - auto dump = [&] (const TTxPartition::TDescribeBlob::TBlockMark& mark) { + using TMark = TTxPartition::TDescribeBlob::TBlockMark; + auto dump = [&] (const TMark& mark) { TABLER() { - TABLED_CLASS("view") { DumpBlockIndex(out, *Info(), mark.BlockIndex, mark.CommitId); } - TABLED() { DumpBlobOffset(out, mark.BlobOffset); } + TABLED_CLASS("view") { + DumpBlockIndex( + out, + *Info(), + mark.BlockIndex, + mark.CommitId); + } + TABLED() { + DumpBlobOffset(out, mark.BlobOffset); + } + TABLED() { + out << mark.Checksum; + } } }; diff --git a/cloud/blockstore/libs/storage/partition/part_actor_monitoring_garbage.cpp b/cloud/blockstore/libs/storage/partition/part_actor_monitoring_garbage.cpp index d6ccb61a280..93719b60340 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_monitoring_garbage.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_monitoring_garbage.cpp @@ -172,7 +172,7 @@ void THttpGarbageActor::HandleAddGarbageRequest( { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "Tablet is dead")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } void THttpGarbageActor::HandleCollectGarbageRequest( @@ -181,7 +181,7 @@ void THttpGarbageActor::HandleCollectGarbageRequest( { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "Tablet is dead")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } STFUNC(THttpGarbageActor::StateWork) diff --git a/cloud/blockstore/libs/storage/partition/part_actor_monitoring_view.cpp b/cloud/blockstore/libs/storage/partition/part_actor_monitoring_view.cpp index 8745eb484d7..db2a2abad72 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_monitoring_view.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_monitoring_view.cpp @@ -248,7 +248,7 @@ void THttpReadBlockActor::HandleReadBlockRequest( { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "Tablet is dead")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } void THttpReadBlockActor::HandlePoisonPill( @@ -257,7 +257,7 @@ void THttpReadBlockActor::HandlePoisonPill( { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "Tablet is dead")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } STFUNC(THttpReadBlockActor::StateWork) diff --git a/cloud/blockstore/libs/storage/partition/part_actor_patchblob.cpp b/cloud/blockstore/libs/storage/partition/part_actor_patchblob.cpp index 5c41749bacf..de40a9d5e23 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_patchblob.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_patchblob.cpp @@ -217,7 +217,7 @@ void TPatchBlobActor::HandlePoisonPill( Y_UNUSED(ev); auto response = std::make_unique( - MakeError(E_REJECTED, "Tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); ReplyAndDie(ctx, std::move(response)); } diff --git a/cloud/blockstore/libs/storage/partition/part_actor_readblocks.cpp b/cloud/blockstore/libs/storage/partition/part_actor_readblocks.cpp index b353a01c3e0..33302c2c1bf 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_readblocks.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_readblocks.cpp @@ -654,7 +654,7 @@ void TReadBlocksActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); auto response = CreateReadBlocksResponse(ReplyLocal, error); ReplyAndDie(ctx, std::move(response), error); @@ -951,7 +951,7 @@ void TPartitionActor::ReadBlocks( commitId, DescribeRange(readRange).data()); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo, requestInfo->CancelRoutine); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition/part_actor_scan_disk.cpp b/cloud/blockstore/libs/storage/partition/part_actor_scan_disk.cpp index 31b4a77ff0c..8aec7828a91 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_scan_disk.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_scan_disk.cpp @@ -241,7 +241,7 @@ void TScanDiskActor::HandlePoisonPill( { Y_UNUSED(ev); - const auto error = MakeError(E_REJECTED, "Tablet is dead"); + const auto error = MakeError(E_REJECTED, "tablet is shutting down"); NotifyCompleted(ctx, error); } @@ -404,7 +404,7 @@ void TPartitionActor::HandleGetScanDiskStatus( } } } else { - result = MakeError(E_REJECTED, "Tablet is dead"); + result = MakeError(E_REJECTED, "tablet is shutting down"); }; *response->Record.MutableError() = std::move(result); @@ -417,7 +417,7 @@ void TPartitionActor::HandleScanDiskBatch( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -473,7 +473,7 @@ void TPartitionActor::HandleScanDiskBatch( gen, step); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx(ctx, CreateTx( requestInfo, diff --git a/cloud/blockstore/libs/storage/partition/part_actor_statpartition.cpp b/cloud/blockstore/libs/storage/partition/part_actor_statpartition.cpp index 7194ffc11b8..6cb21d95e8a 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_statpartition.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_statpartition.cpp @@ -94,6 +94,12 @@ void TPartitionActor::HandleStatPartition( response->Record.MutableStats()->SetCleanupQueueBytes( State->GetCleanupQueue().GetQueueBytes()); + response->Record.MutableStats()->SetUnconfirmedBlobCount( + State->GetUnconfirmedBlobCount()); + + response->Record.MutableStats()->SetConfirmedBlobCount( + State->GetConfirmedBlobCount()); + LWTRACK( ResponseSent_Partition, requestInfo->CallContext->LWOrbit, diff --git a/cloud/blockstore/libs/storage/partition/part_actor_stats.cpp b/cloud/blockstore/libs/storage/partition/part_actor_stats.cpp index fde1da90acb..9ffc4ca1809 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_stats.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_stats.cpp @@ -116,6 +116,9 @@ void TPartitionActor::SendStatsToService(const TActorContext& ctx) PartCounters->Simple.CheckpointBytes.Set(State->CalculateCheckpointBytes()); + PartCounters->Simple.UnconfirmedBlobCount.Set(State->GetUnconfirmedBlobCount()); + PartCounters->Simple.ConfirmedBlobCount.Set(State->GetConfirmedBlobCount()); + ui64 sysCpuConsumption = 0; for (ui32 tx = 0; tx < TPartitionCounters::ETransactionType::TX_SIZE; ++tx) { sysCpuConsumption += Counters->TxCumulative(tx, NKikimr::COUNTER_TT_EXECUTE_CPUTIME).Get(); diff --git a/cloud/blockstore/libs/storage/partition/part_actor_writeblob.cpp b/cloud/blockstore/libs/storage/partition/part_actor_writeblob.cpp index b54e88d33e5..df561ec6e04 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_writeblob.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_writeblob.cpp @@ -1,5 +1,6 @@ #include "part_actor.h" +#include #include #include #include @@ -42,6 +43,7 @@ class TWriteBlobActor final TInstant ResponseReceived; TStorageStatusFlags StorageStatusFlags; double ApproximateFreeSpaceShare = 0; + TVector BlockChecksums; public: TWriteBlobActor( @@ -141,7 +143,26 @@ void TWriteBlobActor::SendPutRequest(const TActorContext& ctx) blobContent = std::move(std::get(Request->Data)); } - Y_ABORT_UNLESS(!blobContent.Empty()); + STORAGE_VERIFY( + !blobContent.Empty(), + TWellKnownEntityTypes::TABLET, + TabletId); + + if (Request->BlockSizeForChecksums) { + STORAGE_VERIFY( + blobContent.Size() % Request->BlockSizeForChecksums == 0, + TWellKnownEntityTypes::TABLET, + TabletId); + + ui32 offset = 0; + while (offset < blobContent.Size()) { + BlockChecksums.push_back(ComputeDefaultDigest({ + blobContent.Data() + offset, + Request->BlockSizeForChecksums})); + + offset += Request->BlockSizeForChecksums; + } + } auto request = std::make_unique( MakeBlobId(TabletId, Request->BlobId), @@ -242,6 +263,7 @@ void TWriteBlobActor::HandlePutResult( auto response = std::make_unique(); response->ExecCycles = RequestInfo->GetExecCycles(); + response->BlockChecksums = std::move(BlockChecksums); ReplyAndDie(ctx, std::move(response)); } @@ -253,7 +275,7 @@ void TWriteBlobActor::HandlePoisonPill( Y_UNUSED(ev); auto response = std::make_unique( - MakeError(E_REJECTED, "Tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); ReplyAndDie(ctx, std::move(response)); } diff --git a/cloud/blockstore/libs/storage/partition/part_actor_writeblocks.cpp b/cloud/blockstore/libs/storage/partition/part_actor_writeblocks.cpp index a57f510641a..48fa51e7f08 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_writeblocks.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_writeblocks.cpp @@ -318,7 +318,7 @@ void TPartitionActor::HandleWriteBlocksCompleted( if (msg->UnconfirmedBlobsAdded) { // blobs are confirmed, but AddBlobs request will be executed // (for this commit) later - State->BlobsConfirmed(commitId); + State->BlobsConfirmed(commitId, std::move(msg->BlobsToConfirm)); Y_DEBUG_ABORT_UNLESS(msg->CollectGarbageBarrierAcquired); // commit & garbage queue barriers will be released when confirmed // blobs are added diff --git a/cloud/blockstore/libs/storage/partition/part_actor_writefreshblocks.cpp b/cloud/blockstore/libs/storage/partition/part_actor_writefreshblocks.cpp index ca999144057..3527989d136 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_writefreshblocks.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_writefreshblocks.cpp @@ -218,7 +218,8 @@ void TWriteFreshBlocksActor::WriteBlob(const TActorContext& ctx) CombinedContext, blobId, std::move(BlobContent), - false); // async + 0, // blockSizeForChecksums + false); // async NCloud::Send( ctx, @@ -251,8 +252,10 @@ void TWriteFreshBlocksActor::NotifyCompleted( using TEvent = TEvPartitionPrivate::TEvWriteBlocksCompleted; auto ev = std::make_unique( error, - false, // collectGarbageBarrierAcquired - false); // unconfirmedBlobsAdded + false, // collectGarbageBarrierAcquired + false, // unconfirmedBlobsAdded + TVector{} // blobsToConfirm + ); ev->ExecCycles = Requests.front().RequestInfo->GetExecCycles(); ev->TotalCycles = Requests.front().RequestInfo->GetTotalCycles(); @@ -342,7 +345,7 @@ void TWriteFreshBlocksActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); ReplyAllAndDie(ctx, error); } @@ -489,7 +492,7 @@ void TPartitionActor::WriteFreshBlocks( DescribeRange(r.Data.Range).data() ); - AddTransaction(*r.Data.RequestInfo); + AddTransaction(*r.Data.RequestInfo, r.Data.RequestInfo->CancelRoutine); subRequests.emplace_back( std::move(r.Data.RequestInfo), diff --git a/cloud/blockstore/libs/storage/partition/part_actor_writemergedblocks.cpp b/cloud/blockstore/libs/storage/partition/part_actor_writemergedblocks.cpp index b6c2f089a60..de166920a76 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_writemergedblocks.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_writemergedblocks.cpp @@ -52,15 +52,17 @@ class TWriteMergedBlocksActor final }; private: + const ui64 TabletId; const TActorId Tablet; const IBlockDigestGeneratorPtr BlockDigestGenerator; const ui64 CommitId; const TRequestInfoPtr RequestInfo; TVector WriteBlobRequests; + TVector BlobsToConfirm; const bool ReplyLocal; const bool ShouldAddUnconfirmedBlobs = false; const IWriteBlocksHandlerPtr WriteHandler; - const bool ChecksumsEnabled; + const ui32 BlockSizeForChecksums; TVector AffectedBlockInfos; size_t WriteBlobRequestsCompleted = 0; @@ -72,6 +74,7 @@ class TWriteMergedBlocksActor final public: TWriteMergedBlocksActor( + const ui64 tabletId, const TActorId& tablet, IBlockDigestGeneratorPtr blockDigestGenerator, ui64 commitId, @@ -80,7 +83,7 @@ class TWriteMergedBlocksActor final bool replyLocal, bool shouldAddUnconfirmedBlobs, IWriteBlocksHandlerPtr writeHandler, - bool checksumsEnabled); + ui32 blockSizeForChecksums); void Bootstrap(const TActorContext& ctx); @@ -98,7 +101,7 @@ class TWriteMergedBlocksActor final void Reply( const TActorContext& ctx, TRequestInfo& requestInfo, - IEventBasePtr response); + IEventBasePtr response) const; private: STFUNC(StateWork); @@ -123,6 +126,7 @@ class TWriteMergedBlocksActor final //////////////////////////////////////////////////////////////////////////////// TWriteMergedBlocksActor::TWriteMergedBlocksActor( + const ui64 tabletId, const TActorId& tablet, IBlockDigestGeneratorPtr blockDigestGenerator, ui64 commitId, @@ -131,8 +135,9 @@ TWriteMergedBlocksActor::TWriteMergedBlocksActor( bool replyLocal, bool shouldAddUnconfirmedBlobs, IWriteBlocksHandlerPtr writeHandler, - bool checksumsEnabled) - : Tablet(tablet) + ui32 blockSizeForChecksums) + : TabletId(tabletId) + , Tablet(tablet) , BlockDigestGenerator(std::move(blockDigestGenerator)) , CommitId(commitId) , RequestInfo(std::move(requestInfo)) @@ -140,7 +145,7 @@ TWriteMergedBlocksActor::TWriteMergedBlocksActor( , ReplyLocal(replyLocal) , ShouldAddUnconfirmedBlobs(shouldAddUnconfirmedBlobs) , WriteHandler(std::move(writeHandler)) - , ChecksumsEnabled(checksumsEnabled) + , BlockSizeForChecksums(blockSizeForChecksums) {} void TWriteMergedBlocksActor::Bootstrap(const TActorContext& ctx) @@ -181,10 +186,6 @@ TGuardedSgList TWriteMergedBlocksActor::BuildBlobContentAndComputeChecksums( if (digest.Defined()) { AffectedBlockInfos.push_back({blockIndex, *digest}); } - - if (ChecksumsEnabled) { - request.Checksums.push_back(ComputeDefaultDigest(block)); - } } } return guardedSgList; @@ -198,7 +199,9 @@ void TWriteMergedBlocksActor::WriteBlobs(const TActorContext& ctx) auto request = std::make_unique( req.BlobId, - std::move(guardedSglist)); + std::move(guardedSglist), + BlockSizeForChecksums, + false); // async if (!RequestInfo->CallContext->LWOrbit.Fork(request->CallContext->LWOrbit)) { LWTRACK( @@ -213,7 +216,8 @@ void TWriteMergedBlocksActor::WriteBlobs(const TActorContext& ctx) NCloud::Send( ctx, Tablet, - std::move(request)); + std::move(request), + i); } } @@ -245,16 +249,20 @@ void TWriteMergedBlocksActor::AddBlobs( ADD_WRITE_RESULT ); } else { - TVector blobs(Reserve(WriteBlobRequests.size())); + BlobsToConfirm.reserve(WriteBlobRequests.size()); for (const auto& req: WriteBlobRequests) { - blobs.emplace_back(req.BlobId.UniqueId(), req.WriteRange); + BlobsToConfirm.emplace_back( + req.BlobId.UniqueId(), + req.WriteRange, + // checksums are not ready at this point + TVector()); } request = std::make_unique( RequestInfo->CallContext, CommitId, - std::move(blobs)); + BlobsToConfirm); } SafeToUseOrbit = false; @@ -270,10 +278,18 @@ void TWriteMergedBlocksActor::NotifyCompleted( const NProto::TError& error) { using TEvent = TEvPartitionPrivate::TEvWriteBlocksCompleted; + if (!BlockSizeForChecksums) { + // this structure is needed only to transfer block checksums to + // PartState - passing it without checksums will trigger a couple + // of debug asserts and is actually pointless even if we didn't have + // those asserts + BlobsToConfirm.clear(); + } auto ev = std::make_unique( error, true, // collectGarbageBarrierAcquired - UnconfirmedBlobsAdded); + UnconfirmedBlobsAdded, + std::move(BlobsToConfirm)); ev->ExecCycles = RequestInfo->GetExecCycles(); ev->TotalCycles = RequestInfo->GetTotalCycles(); @@ -328,7 +344,7 @@ void TWriteMergedBlocksActor::ReplyAndDie( void TWriteMergedBlocksActor::Reply( const TActorContext& ctx, TRequestInfo& requestInfo, - IEventBasePtr response) + IEventBasePtr response) const { if (SafeToUseOrbit) { LWTRACK( @@ -347,7 +363,7 @@ void TWriteMergedBlocksActor::HandleWriteBlobResponse( const TEvPartitionPrivate::TEvWriteBlobResponse::TPtr& ev, const TActorContext& ctx) { - const auto* msg = ev->Get(); + auto* msg = ev->Get(); RequestInfo->AddExecCycles(msg->ExecCycles); @@ -355,12 +371,33 @@ void TWriteMergedBlocksActor::HandleWriteBlobResponse( return; } - Y_ABORT_UNLESS(WriteBlobRequestsCompleted < WriteBlobRequests.size()); + STORAGE_VERIFY( + ev->Cookie < WriteBlobRequests.size(), + TWellKnownEntityTypes::TABLET, + TabletId); + + if (BlobsToConfirm.empty()) { + WriteBlobRequests[ev->Cookie].Checksums = + std::move(msg->BlockChecksums); + } else { + STORAGE_VERIFY( + BlobsToConfirm.size() == WriteBlobRequests.size(), + TWellKnownEntityTypes::TABLET, + TabletId); + + BlobsToConfirm[ev->Cookie].Checksums = std::move(msg->BlockChecksums); + } + + STORAGE_VERIFY( + WriteBlobRequestsCompleted < WriteBlobRequests.size(), + TWellKnownEntityTypes::TABLET, + TabletId); + if (++WriteBlobRequestsCompleted < WriteBlobRequests.size()) { return; } - for (auto context: ForkedCallContexts) { + for (const auto& context: ForkedCallContexts) { RequestInfo->CallContext->LWOrbit.Join(context->LWOrbit); } @@ -421,7 +458,7 @@ void TWriteMergedBlocksActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); ReplyAndDie(ctx, error); } @@ -499,19 +536,25 @@ void TPartitionActor::WriteMergedBlocks( / State->GetBlockSize(); const bool checksumsEnabled = writeRange.Start < checksumBoundary; - bool shouldAddUnconfirmedBlobs = Config->GetAddingUnconfirmedBlobsEnabled(); + const bool addingUnconfirmedBlobsEnabledForCloud = Config->IsAddingUnconfirmedBlobsFeatureEnabled( + PartitionConfig.GetCloudId(), + PartitionConfig.GetFolderId(), + PartitionConfig.GetDiskId()); + bool shouldAddUnconfirmedBlobs = Config->GetAddingUnconfirmedBlobsEnabled() + || addingUnconfirmedBlobsEnabledForCloud; if (shouldAddUnconfirmedBlobs) { // we take confirmed blobs into account because they have not yet been // added to the index, so we treat them as unconfirmed while counting // the limit const ui32 blobCount = - State->GetUnconfirmedBlobs().size() + State->GetConfirmedBlobs().size(); + State->GetUnconfirmedBlobCount() + State->GetConfirmedBlobCount(); shouldAddUnconfirmedBlobs = blobCount < Config->GetUnconfirmedBlobCountHardLimit(); } auto actor = NCloud::Register( ctx, + TabletID(), SelfId(), BlockDigestGenerator, commitId, @@ -520,7 +563,7 @@ void TPartitionActor::WriteMergedBlocks( requestInBuffer.Data.ReplyLocal, shouldAddUnconfirmedBlobs, std::move(requestInBuffer.Data.Handler), - checksumsEnabled + checksumsEnabled ? State->GetBlockSize() : 0 ); Actors.Insert(actor); } diff --git a/cloud/blockstore/libs/storage/partition/part_actor_writemixedblocks.cpp b/cloud/blockstore/libs/storage/partition/part_actor_writemixedblocks.cpp index eff4548ebf3..a9365f35f0b 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_writemixedblocks.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_writemixedblocks.cpp @@ -47,33 +47,33 @@ class TWriteMixedBlocksActor final const bool Empty; const TRequestInfoPtr RequestInfo; const bool ReplyLocal; - const TVector Checksums; TSubRequest( const TBlockRange32 writeRange, const bool empty, TRequestInfoPtr requestInfo, - bool replyLocal, - TVector checksums) + bool replyLocal) : WriteRange(writeRange) , Empty(empty) , RequestInfo(std::move(requestInfo)) , ReplyLocal(replyLocal) - , Checksums(std::move(checksums)) { } }; TPartialBlobId BlobId; TVector SubRequests; + TVector Checksums; }; private: + const ui64 TabletId; const TActorId Tablet; const IBlockDigestGeneratorPtr BlockDigestGenerator; const ui64 CommitId; - const TVector Requests; + TVector Requests; const IWriteBlocksHandlerPtr WriteHandler; + const ui32 BlockSizeForChecksums; TVector AffectedBlockInfos; size_t RequestsCompleted = 0; @@ -83,11 +83,13 @@ class TWriteMixedBlocksActor final public: TWriteMixedBlocksActor( + const ui64 tabletId, const TActorId& tablet, IBlockDigestGeneratorPtr blockDigestGenerator, ui64 commitId, TVector requests, - IWriteBlocksHandlerPtr writeHandler); + IWriteBlocksHandlerPtr writeHandler, + ui32 blockSizeForChecksums); void Bootstrap(const TActorContext& ctx); @@ -126,16 +128,20 @@ class TWriteMixedBlocksActor final }; TWriteMixedBlocksActor::TWriteMixedBlocksActor( + const ui64 tabletId, const TActorId& tablet, IBlockDigestGeneratorPtr blockDigestGenerator, ui64 commitId, TVector requests, - IWriteBlocksHandlerPtr writeHandler) - : Tablet(tablet) + IWriteBlocksHandlerPtr writeHandler, + ui32 blockSizeForChecksums) + : TabletId(tabletId) + , Tablet(tablet) , BlockDigestGenerator(std::move(blockDigestGenerator)) , CommitId(commitId) , Requests(std::move(requests)) , WriteHandler(std::move(writeHandler)) + , BlockSizeForChecksums(blockSizeForChecksums) {} void TWriteMixedBlocksActor::Bootstrap(const TActorContext& ctx) @@ -225,7 +231,9 @@ void TWriteMixedBlocksActor::WriteBlobs(const TActorContext& ctx) auto request = std::make_unique( req.BlobId, - std::move(guardedSglist)); + std::move(guardedSglist), + BlockSizeForChecksums, + false); // async for (const auto& sr: req.SubRequests) { if (!sr.RequestInfo->CallContext->LWOrbit.Fork(request->CallContext->LWOrbit)) { @@ -251,22 +259,14 @@ void TWriteMixedBlocksActor::AddBlobs(const TActorContext& ctx) { TVector blobs(Reserve(Requests.size())); - for (const auto& req: Requests) { - ui32 blockCount = 0; - TVector blocks(Reserve(blockCount)); - TVector checksums(Reserve(blockCount)); + for (auto& req: Requests) { + TVector blocks; for (const auto& sr: req.SubRequests) { if (!sr.Empty) { - blockCount += sr.WriteRange.Size(); - for (ui32 idx: xrange(sr.WriteRange)) { blocks.push_back(idx); } - - for (const auto& checksum: sr.Checksums) { - checksums.push_back(checksum); - } } if (!sr.RequestInfo->CallContext->LWOrbit.Fork(CombinedContext->LWOrbit)) { LWTRACK( @@ -277,7 +277,10 @@ void TWriteMixedBlocksActor::AddBlobs(const TActorContext& ctx) } } - blobs.emplace_back(req.BlobId, std::move(blocks), std::move(checksums)); + blobs.emplace_back( + req.BlobId, + std::move(blocks), + std::move(req.Checksums)); } auto request = std::make_unique( @@ -302,8 +305,10 @@ void TWriteMixedBlocksActor::NotifyCompleted( using TEvent = TEvPartitionPrivate::TEvWriteBlocksCompleted; auto ev = std::make_unique( error, - true, // collectGarbageBarrierAcquired - false); // unconfirmedBlobsAdded + true, // collectGarbageBarrierAcquired + false, // unconfirmedBlobsAdded + TVector{} // blobsToConfirm + ); ui32 blocksCount = 0; ui64 waitCycles = 0; @@ -390,11 +395,18 @@ void TWriteMixedBlocksActor::HandleWriteBlobResponse( const TEvPartitionPrivate::TEvWriteBlobResponse::TPtr& ev, const TActorContext& ctx) { - const auto* msg = ev->Get(); + auto* msg = ev->Get(); - for (const auto& sr: Requests[ev->Cookie].SubRequests) { + STORAGE_VERIFY( + ev->Cookie < Requests.size(), + TWellKnownEntityTypes::TABLET, + TabletId); + + auto& request = Requests[ev->Cookie]; + for (const auto& sr: request.SubRequests) { sr.RequestInfo->AddExecCycles(msg->ExecCycles); } + request.Checksums = std::move(msg->BlockChecksums); if (HandleError(ctx, msg->GetError())) { return; @@ -442,7 +454,7 @@ void TWriteMixedBlocksActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); ReplyAllAndDie(ctx, error); } @@ -485,6 +497,8 @@ bool TPartitionActor::WriteMixedBlocks( State->GetCommitQueue().AcquireBarrier(commitId); State->GetGarbageQueue().AcquireBarrier(commitId); + bool checksumsEnabled = false; + for (const auto& group: groups) { requests.emplace_back(); @@ -496,37 +510,25 @@ bool TPartitionActor::WriteMixedBlocks( ); } - LOG_TRACE(ctx, TBlockStoreComponents::PARTITION, + LOG_DEBUG(ctx, TBlockStoreComponents::PARTITION, "[%lu] Writing mixed blocks @%lu (range: %s)", TabletID(), commitId, DescribeRange(request->Data.Range).data() ); - TVector checksums; - const ui32 checksumBoundary = Config->GetDiskPrefixLengthWithBlockChecksumsInBlobs() / State->GetBlockSize(); - const bool checksumsEnabled = - request->Data.Range.Start < checksumBoundary; - - if (checksumsEnabled) { - auto sgList = request->Data.Handler->GetBlocks( - ConvertRangeSafe(request->Data.Range)); - if (auto g = sgList.Acquire()) { - for (const auto& blockContent: g.Get()) { - checksums.push_back(ComputeDefaultDigest(blockContent)); - } - } + if (request->Data.Range.Start < checksumBoundary) { + checksumsEnabled = true; } requests.back().SubRequests.emplace_back( request->Data.Range, !request->Weight, request->Data.RequestInfo, - request->Data.ReplyLocal, - std::move(checksums) + request->Data.ReplyLocal ); } @@ -543,11 +545,13 @@ bool TPartitionActor::WriteMixedBlocks( auto actor = NCloud::Register( ctx, + TabletID(), SelfId(), BlockDigestGenerator, commitId, std::move(requests), - std::move(writeHandler) + std::move(writeHandler), + checksumsEnabled ? State->GetBlockSize() : 0 ); Actors.Insert(actor); diff --git a/cloud/blockstore/libs/storage/partition/part_actor_zeroblocks.cpp b/cloud/blockstore/libs/storage/partition/part_actor_zeroblocks.cpp index 4b52ce9aba2..24dc5529fc1 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_zeroblocks.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_zeroblocks.cpp @@ -183,7 +183,7 @@ void TZeroBlocksActor::HandlePoisonPill( Y_UNUSED(ev); auto respose = std::make_unique( - MakeError(E_REJECTED, "Tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); ReplyAndDie(ctx, std::move(respose)); } @@ -228,7 +228,7 @@ void TPartitionActor::HandleZeroBlocks( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -289,7 +289,7 @@ void TPartitionActor::HandleZeroBlocks( GetWriteBlobThreshold(*Config, PartitionConfig.GetStorageMediaKind()); if (requestSize < writeBlobThreshold) { // small writes will be accumulated in FreshBlocks table - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); auto tx = CreateTx( requestInfo, diff --git a/cloud/blockstore/libs/storage/partition/part_database.cpp b/cloud/blockstore/libs/storage/partition/part_database.cpp index 856aaa5912e..852e36d9890 100644 --- a/cloud/blockstore/libs/storage/partition/part_database.cpp +++ b/cloud/blockstore/libs/storage/partition/part_database.cpp @@ -669,9 +669,17 @@ bool TPartitionDatabase::ReadBlockMask( return true; } -// TODO: pass and use skipMask -static bool FindBlocksInBlobIndex( - IBlocksIndexVisitor& visitor, +enum class EIndexProcResult +{ + Ok, + NotReady, + Interrupted, +}; + +static EIndexProcResult FindBlocksInBlobIndex( + TPartitionDatabase& db, + IExtendedBlocksIndexVisitor& visitor, + const ui32 maxBlocksInBlob, const TPartialBlobId& blobId, const NProto::TBlobMeta& blobMeta, const TBlockMask& blockMask, @@ -687,7 +695,14 @@ static bool FindBlocksInBlobIndex( ? InvalidBlobOffset : blobOffset; - return visitor.Visit(blockIndex, commitId, blobId, maskedBlobOffset); + return visitor.Visit( + blockIndex, + commitId, + blobId, + maskedBlobOffset, + blobOffset < blobMeta.BlockChecksumsSize() + ? blobMeta.GetBlockChecksums(blobOffset) + : 0); }; if (blobMeta.HasMixedBlocks()) { @@ -701,7 +716,7 @@ static bool FindBlocksInBlobIndex( for (ui32 blockIndex: mixedBlocks.GetBlocks()) { if (blockRange.Contains(blockIndex)) { if (!visit(blockIndex, commitId, blobId, blobOffset)) { - return false; // interrupted + return EIndexProcResult::Interrupted; } } ++blobOffset; @@ -717,7 +732,7 @@ static bool FindBlocksInBlobIndex( if (blockRange.Contains(blockIndex)) { if (!visit(blockIndex, commitId, blobId, blobOffset)) { - return false; // interrupted + return EIndexProcResult::Interrupted; } } ++blobOffset; @@ -726,29 +741,84 @@ static bool FindBlocksInBlobIndex( } } else if (blobMeta.HasMergedBlocks()) { const auto& mergedBlocks = blobMeta.GetMergedBlocks(); + const auto blobRange = TBlockRange32::MakeClosedInterval( + mergedBlocks.GetStart(), + mergedBlocks.GetEnd()); + if (!blobRange.Overlaps(blockRange)) { + return EIndexProcResult::Ok; + } + const auto intersection = blobRange.Intersect(blockRange); + + struct TMark + { + ui64 CommitId = InvalidCommitId; + ui16 BlobOffset = InvalidBlobOffset; + }; + + struct TVisitor final + : public IBlocksIndexVisitor + { + TBlockRange32 Intersection; + TPartialBlobId BlobId; + + TVector Marks; + + TVisitor(TBlockRange32 intersection, TPartialBlobId blobId) + : Intersection(intersection) + , BlobId(blobId) + , Marks(intersection.Size()) + { + } - // if there is no intersection - just skip range - ui32 start = Max(blockRange.Start, mergedBlocks.GetStart()); - ui32 end = Min(blockRange.End, mergedBlocks.GetEnd()); + bool Visit( + ui32 blockIndex, + ui64 commitId, + const TPartialBlobId& blobId, + ui16 blobOffset) override + { + if (blobId == BlobId) { + Marks[blockIndex - Intersection.Start] = { + commitId, + blobOffset, + }; + } - if (start <= end) { - // every block shares the same commitId - ui64 commitId = blobId.CommitId(); + return true; + } + } helper(intersection, blobId); - for (ui32 blockIndex = start; blockIndex <= end; ++blockIndex) { - ui16 blobOffset = blockIndex - mergedBlocks.GetStart(); - if (!visit(blockIndex, commitId, blobId, blobOffset)) { - return false; // interrupted - } + // just using BlobMeta isn't enough - we need SkipMask to properly + // iterate blocks belonging to this merged blob + const bool ready = db.FindMergedBlocks( + helper, + blockRange, + true, // precharge + maxBlocksInBlob, + Max()); + + if (!ready) { + return EIndexProcResult::NotReady; + } + + for (ui32 i = 0; i < intersection.Size(); ++i) { + const auto res = visit( + intersection.Start + i, + helper.Marks[i].CommitId, + blobId, + helper.Marks[i].BlobOffset); + + if (!res) { + return EIndexProcResult::Interrupted; } } } - return true; + return EIndexProcResult::Ok; } bool TPartitionDatabase::FindBlocksInBlobsIndex( - IBlocksIndexVisitor& visitor, + IExtendedBlocksIndexVisitor& visitor, + const ui32 maxBlocksInBlob, const TBlockRange32& blockRange) { using TTable = TPartitionSchema::BlobsIndex; @@ -771,15 +841,19 @@ bool TPartitionDatabase::FindBlocksInBlobsIndex( auto blockMask = BlockMaskFromString( it.GetValueOrDefault()); - bool shouldContinue = FindBlocksInBlobIndex( + const auto res = FindBlocksInBlobIndex( + *this, visitor, + maxBlocksInBlob, blobId, blobMeta, blockMask, blockRange); - if (!shouldContinue) { - return true; // interrupted + switch (res) { + case EIndexProcResult::Ok: break; + case EIndexProcResult::NotReady: return false; + case EIndexProcResult::Interrupted: return true; } if (!it.Next()) { @@ -791,7 +865,8 @@ bool TPartitionDatabase::FindBlocksInBlobsIndex( } bool TPartitionDatabase::FindBlocksInBlobsIndex( - IBlocksIndexVisitor& visitor, + IExtendedBlocksIndexVisitor& visitor, + const ui32 maxBlocksInBlob, const TPartialBlobId& blobId) { using TTable = TPartitionSchema::BlobsIndex; @@ -810,12 +885,20 @@ bool TPartitionDatabase::FindBlocksInBlobsIndex( auto blockMask = BlockMaskFromString( it.GetValueOrDefault()); - FindBlocksInBlobIndex( + const auto res = FindBlocksInBlobIndex( + *this, visitor, + maxBlocksInBlob, blobId, blobMeta, blockMask, TBlockRange32::Max()); + + switch (res) { + case EIndexProcResult::Ok: break; + case EIndexProcResult::NotReady: return false; + case EIndexProcResult::Interrupted: return true; + } } return true; @@ -1194,10 +1277,11 @@ bool TPartitionDatabase::ReadGarbageBlobs(TVector& blobIds) void TPartitionDatabase::WriteUnconfirmedBlob( const TPartialBlobId& blobId, - const TBlobUniqueIdWithRange& blob) + const TBlobToConfirm& blob) { using TTable = TPartitionSchema::UnconfirmedBlobs; + // TODO: persist blob checksums (issue-122) Table() .Key(blobId.CommitId(), blobId.UniqueId()) .Update( @@ -1215,8 +1299,7 @@ void TPartitionDatabase::DeleteUnconfirmedBlob(const TPartialBlobId& blobId) .Delete(); } -bool TPartitionDatabase::ReadUnconfirmedBlobs( - TCommitIdToBlobUniqueIdWithRange& blobs) +bool TPartitionDatabase::ReadUnconfirmedBlobs(TCommitIdToBlobsToConfirm& blobs) { using TTable = TPartitionSchema::UnconfirmedBlobs; @@ -1235,7 +1318,11 @@ bool TPartitionDatabase::ReadUnconfirmedBlobs( it.GetValue(), it.GetValue()); - blobs[commitId].emplace_back(uniqueId, blockRange); + blobs[commitId].emplace_back( + uniqueId, + blockRange, + TVector() /* TODO: checksums */ + ); if (!it.Next()) { return false; // not ready diff --git a/cloud/blockstore/libs/storage/partition/part_database.h b/cloud/blockstore/libs/storage/partition/part_database.h index 1bcf3b5cea2..7b7d67eb6c0 100644 --- a/cloud/blockstore/libs/storage/partition/part_database.h +++ b/cloud/blockstore/libs/storage/partition/part_database.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include @@ -139,11 +139,13 @@ class TPartitionDatabase TMaybe& blockMask); bool FindBlocksInBlobsIndex( - IBlocksIndexVisitor& visitor, + IExtendedBlocksIndexVisitor& visitor, + const ui32 maxBlocksInBlob, const TBlockRange32& blockRange); bool FindBlocksInBlobsIndex( - IBlocksIndexVisitor& visitor, + IExtendedBlocksIndexVisitor& visitor, + const ui32 maxBlocksInBlob, const TPartialBlobId& blobId); EBlobIndexScanProgress FindBlocksInBlobsIndex( @@ -209,10 +211,10 @@ class TPartitionDatabase void WriteUnconfirmedBlob( const TPartialBlobId& blobId, - const TBlobUniqueIdWithRange& blob); + const TBlobToConfirm& blob); void DeleteUnconfirmedBlob(const TPartialBlobId& blobId); - bool ReadUnconfirmedBlobs(TCommitIdToBlobUniqueIdWithRange& blobs); + bool ReadUnconfirmedBlobs(TCommitIdToBlobsToConfirm& blobs); }; } // namespace NCloud::NBlockStore::NStorage::NPartition diff --git a/cloud/blockstore/libs/storage/partition/part_database_ut.cpp b/cloud/blockstore/libs/storage/partition/part_database_ut.cpp index d45b9961ef1..aaa20230cd5 100644 --- a/cloud/blockstore/libs/storage/partition/part_database_ut.cpp +++ b/cloud/blockstore/libs/storage/partition/part_database_ut.cpp @@ -15,6 +15,11 @@ TString GetBlockContent(char fill) return TString(DefaultBlockSize, fill); } +TString CommitId2Str(ui64 commitId) +{ + return commitId == InvalidCommitId ? "x" : ToString(commitId); +} + //////////////////////////////////////////////////////////////////////////////// struct TTestBlockVisitor final @@ -22,24 +27,46 @@ struct TTestBlockVisitor final { TStringBuilder Result; - void MarkBlock(ui32 blockIndex, ui64 commitId) + bool Visit( + ui32 blockIndex, + ui64 commitId, + const TPartialBlobId& blobId, + ui16 blobOffset) override { + Y_UNUSED(blobId); + Y_UNUSED(blobOffset); + if (Result) { Result << " "; } - Result << "#" << blockIndex << ":" << commitId; + Result << "#" << blockIndex << ":" << CommitId2Str(commitId); + return true; } +}; + +struct TTestBlockVisitorWithBlobOffset final + : public IExtendedBlocksIndexVisitor +{ + TStringBuilder Result; bool Visit( ui32 blockIndex, ui64 commitId, const TPartialBlobId& blobId, - ui16 blobOffset) override + ui16 blobOffset, + ui32 checksum) override { Y_UNUSED(blobId); - Y_UNUSED(blobOffset); - MarkBlock(blockIndex, commitId); + if (Result) { + Result << " "; + } + Result << "#" << blockIndex + << ":" << CommitId2Str(commitId) + << ":" << blobOffset; + if (checksum) { + Result << "##" << checksum; + } return true; } }; @@ -808,6 +835,85 @@ Y_UNIT_TEST_SUITE(TPartitionDatabaseTest) } }); } + + Y_UNIT_TEST(ShouldUseSkipMaskInFindBlocksInBlobsIndex) + { + TTestExecutor executor; + executor.WriteTx([&] (TPartitionDatabase db) { + db.InitSchema(); + }); + + TPartialBlobId blob1; + auto range1 = TBlockRange32::MakeClosedInterval(100, 120); + TBlockMask skipMask1; + skipMask1.Set(8, 14); + skipMask1.Set(21, skipMask1.Size()); + + TPartialBlobId blob2; + auto range2 = TBlockRange32::MakeClosedInterval(110, 140); + TBlockMask skipMask2; + skipMask2.Set(10, 20); + skipMask2.Set(31, skipMask2.Size()); + + executor.WriteTx([&] (TPartitionDatabase db) { + NProto::TBlobMeta meta; + + auto* mb = meta.MutableMergedBlocks(); + mb->SetStart(range1.Start); + mb->SetEnd(range1.End); + mb->SetSkipped(5); + blob1 = executor.MakeBlobId(); + db.WriteBlobMeta(blob1, meta); + db.WriteMergedBlocks(blob1, range1, skipMask1); + }); + + executor.WriteTx([&] (TPartitionDatabase db) { + NProto::TBlobMeta meta; + + auto* mb = meta.MutableMergedBlocks(); + mb->SetStart(range2.Start); + mb->SetEnd(range2.End); + mb->SetSkipped(10); + meta.AddBlockChecksums(111); + meta.AddBlockChecksums(222); + meta.AddBlockChecksums(333); + blob2 = executor.MakeBlobId(); + db.WriteBlobMeta(blob2, meta); + db.WriteMergedBlocks(blob2, range2, skipMask2); + }); + + executor.ReadTx([&] (TPartitionDatabase db) { + TTestBlockVisitorWithBlobOffset visitor; + db.FindBlocksInBlobsIndex( + visitor, + MaxBlocksCount, + blob1); + + UNIT_ASSERT_VALUES_EQUAL( + "#100:2:0 #101:2:1 #102:2:2 #103:2:3 #104:2:4 #105:2:5 #106:2:6" + " #107:2:7 #108:x:65535 #109:x:65535 #110:x:65535 #111:x:65535" + " #112:x:65535 #113:x:65535 #114:2:8 #115:2:9 #116:2:10" + " #117:2:11 #118:2:12 #119:2:13 #120:2:14", + visitor.Result); + }); + + executor.ReadTx([&] (TPartitionDatabase db) { + TTestBlockVisitorWithBlobOffset visitor; + db.FindBlocksInBlobsIndex( + visitor, + MaxBlocksCount, + blob2); + + UNIT_ASSERT_VALUES_EQUAL( + "#110:3:0##111 #111:3:1##222 #112:3:2##333 #113:3:3 #114:3:4" + " #115:3:5 #116:3:6 #117:3:7 #118:3:8 #119:3:9 #120:x:65535" + " #121:x:65535 #122:x:65535 #123:x:65535 #124:x:65535" + " #125:x:65535 #126:x:65535 #127:x:65535 #128:x:65535" + " #129:x:65535 #130:3:10 #131:3:11 #132:3:12 #133:3:13 #134:3:14" + " #135:3:15 #136:3:16 #137:3:17 #138:3:18 #139:3:19 #140:3:20", + visitor.Result); + }); + } } } // namespace NCloud::NBlockStore::NStorage::NPartition diff --git a/cloud/blockstore/libs/storage/partition/part_events_private.h b/cloud/blockstore/libs/storage/partition/part_events_private.h index 35afa546e32..d6af907df2e 100644 --- a/cloud/blockstore/libs/storage/partition/part_events_private.h +++ b/cloud/blockstore/libs/storage/partition/part_events_private.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include @@ -231,22 +231,24 @@ struct TEvPartitionPrivate { NActors::TActorId Proxy; - TPartialBlobId BlobId; + const TPartialBlobId BlobId; std::variant Data; - - bool Async = false; - TInstant Deadline; - - TWriteBlobRequest() = default; + // BlockSize is used to calculate checksums. If it's 0, checksums won't + // be calculated. + const ui32 BlockSizeForChecksums; + const bool Async; + const TInstant Deadline; template TWriteBlobRequest( TPartialBlobId blobId, TData data, - bool async = false, + ui32 blockSizeForChecksums, + bool async, TInstant deadline = TInstant::Max()) : BlobId(blobId) , Data(std::move(data)) + , BlockSizeForChecksums(blockSizeForChecksums) , Async(async) , Deadline(deadline) {} @@ -254,6 +256,7 @@ struct TEvPartitionPrivate struct TWriteBlobResponse { + TVector BlockChecksums; ui64 ExecCycles = 0; }; @@ -656,13 +659,13 @@ struct TEvPartitionPrivate struct TAddUnconfirmedBlobsRequest { ui64 CommitId = 0; - TVector Blobs; + TVector Blobs; TAddUnconfirmedBlobsRequest() = default; TAddUnconfirmedBlobsRequest( ui64 commitId, - TVector blobs) + TVector blobs) : CommitId(commitId) , Blobs(std::move(blobs)) {} @@ -726,12 +729,16 @@ struct TEvPartitionPrivate { bool CollectGarbageBarrierAcquired = false; bool UnconfirmedBlobsAdded = false; + // needed to pass block checksums to PartState + TVector BlobsToConfirm; TWriteBlocksCompleted( bool collectGarbageBarrierAcquired, - bool unconfirmedBlobsAdded) + bool unconfirmedBlobsAdded, + TVector blobsToConfirm) : CollectGarbageBarrierAcquired(collectGarbageBarrierAcquired) , UnconfirmedBlobsAdded(unconfirmedBlobsAdded) + , BlobsToConfirm(std::move(blobsToConfirm)) { } }; @@ -812,15 +819,15 @@ struct TEvPartitionPrivate // struct TConfirmBlobsCompleted - : TOperationCompleted { + const ui64 StartCycleCount; TVector UnrecoverableBlobs; - TConfirmBlobsCompleted() = default; - - explicit TConfirmBlobsCompleted( + TConfirmBlobsCompleted( + ui64 startCycleCount, TVector unrecoverableBlobs) - : UnrecoverableBlobs(std::move(unrecoverableBlobs)) + : StartCycleCount(startCycleCount) + , UnrecoverableBlobs(std::move(unrecoverableBlobs)) {} }; diff --git a/cloud/blockstore/libs/storage/partition/part_state.cpp b/cloud/blockstore/libs/storage/partition/part_state.cpp index 8a82682badb..fee55ae4233 100644 --- a/cloud/blockstore/libs/storage/partition/part_state.cpp +++ b/cloud/blockstore/libs/storage/partition/part_state.cpp @@ -617,11 +617,13 @@ bool TPartitionState::OverlapsConfirmedBlobs( } void TPartitionState::InitUnconfirmedBlobs( - TCommitIdToBlobUniqueIdWithRange blobs) + TCommitIdToBlobsToConfirm blobs) { UnconfirmedBlobs = std::move(blobs); - for (const auto& entry: UnconfirmedBlobs) { - auto commitId = entry.first; + UnconfirmedBlobCount = 0; + + for (const auto& [commitId, blobs]: UnconfirmedBlobs) { + UnconfirmedBlobCount += blobs.size(); CommitQueue.AcquireBarrier(commitId); GarbageQueue.AcquireBarrier(commitId); } @@ -630,11 +632,12 @@ void TPartitionState::InitUnconfirmedBlobs( void TPartitionState::WriteUnconfirmedBlob( TPartitionDatabase& db, ui64 commitId, - const TBlobUniqueIdWithRange& blob) + const TBlobToConfirm& blob) { auto blobId = MakePartialBlobId(commitId, blob.UniqueId); db.WriteUnconfirmedBlob(blobId, blob); UnconfirmedBlobs[commitId].push_back(blob); + UnconfirmedBlobCount++; } void TPartitionState::ConfirmedBlobsAdded( @@ -646,24 +649,49 @@ void TPartitionState::ConfirmedBlobsAdded( return; } - for (const auto& blob: it->second) { + auto& blobs = it->second; + const auto blobCount = blobs.size(); + + for (const auto& blob: blobs) { auto blobId = MakePartialBlobId(commitId, blob.UniqueId); db.DeleteUnconfirmedBlob(blobId); } ConfirmedBlobs.erase(it); + Y_DEBUG_ABORT_UNLESS(ConfirmedBlobCount >= blobCount); + ConfirmedBlobCount -= blobCount; GarbageQueue.ReleaseBarrier(commitId); CommitQueue.ReleaseBarrier(commitId); } -void TPartitionState::BlobsConfirmed(ui64 commitId) +void TPartitionState::BlobsConfirmed( + ui64 commitId, + TVector blobs) { auto it = UnconfirmedBlobs.find(commitId); Y_DEBUG_ABORT_UNLESS(it != UnconfirmedBlobs.end()); - ConfirmedBlobs[commitId] = std::move(it->second); + auto& dstBlobs = it->second; + const auto blobCount = dstBlobs.size(); + Y_DEBUG_ABORT_UNLESS(blobs.empty() || blobCount == blobs.size()); + for (ui32 i = 0; i < Min(blobCount, blobs.size()); ++i) { + const auto blockRange = dstBlobs[i].BlockRange; + Y_DEBUG_ABORT_UNLESS(dstBlobs[i].UniqueId == blobs[i].UniqueId); + Y_DEBUG_ABORT_UNLESS(blockRange.Start == blobs[i].BlockRange.Start); + Y_DEBUG_ABORT_UNLESS(blockRange.End == blobs[i].BlockRange.End); + Y_DEBUG_ABORT_UNLESS(blockRange.Size() == blobs[i].Checksums.size()); + if (dstBlobs[i].UniqueId == blobs[i].UniqueId) { + dstBlobs[i].Checksums = std::move(blobs[i]).Checksums; + } + } + + ConfirmedBlobs[commitId] = std::move(dstBlobs); + ConfirmedBlobCount += blobCount; + UnconfirmedBlobs.erase(it); + Y_DEBUG_ABORT_UNLESS(UnconfirmedBlobCount >= blobCount); + UnconfirmedBlobCount -= blobCount; } void TPartitionState::ConfirmBlobs( @@ -687,7 +715,10 @@ void TPartitionState::ConfirmBlobs( } ConfirmedBlobs = std::move(UnconfirmedBlobs); + ConfirmedBlobCount = UnconfirmedBlobCount; + UnconfirmedBlobs.clear(); + UnconfirmedBlobCount = 0; } #define BLOCKSTORE_PARTITION_IMPLEMENT_COUNTER(name) \ diff --git a/cloud/blockstore/libs/storage/partition/part_state.h b/cloud/blockstore/libs/storage/partition/part_state.h index cb161f07dcf..293cc2fb03a 100644 --- a/cloud/blockstore/libs/storage/partition/part_state.h +++ b/cloud/blockstore/libs/storage/partition/part_state.h @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -1298,22 +1298,34 @@ class TPartitionState } private: - TCommitIdToBlobUniqueIdWithRange UnconfirmedBlobs; + TCommitIdToBlobsToConfirm UnconfirmedBlobs; + ui32 UnconfirmedBlobCount = 0; // contains entries from UnconfirmedBlobs that have been confirmed but have // not yet been added to the index - TCommitIdToBlobUniqueIdWithRange ConfirmedBlobs; + TCommitIdToBlobsToConfirm ConfirmedBlobs; + ui32 ConfirmedBlobCount = 0; public: - const TCommitIdToBlobUniqueIdWithRange& GetUnconfirmedBlobs() const + const TCommitIdToBlobsToConfirm& GetUnconfirmedBlobs() const { return UnconfirmedBlobs; } - const TCommitIdToBlobUniqueIdWithRange& GetConfirmedBlobs() const + ui32 GetUnconfirmedBlobCount() const + { + return UnconfirmedBlobCount; + } + + const TCommitIdToBlobsToConfirm& GetConfirmedBlobs() const { return ConfirmedBlobs; } + ui32 GetConfirmedBlobCount() const + { + return ConfirmedBlobCount; + } + bool OverlapsUnconfirmedBlobs( ui64 lowCommitId, ui64 highCommitId, @@ -1324,16 +1336,16 @@ class TPartitionState ui64 highCommitId, const TBlockRange32& blockRange) const; - void InitUnconfirmedBlobs(TCommitIdToBlobUniqueIdWithRange blobs); + void InitUnconfirmedBlobs(TCommitIdToBlobsToConfirm blobs); void WriteUnconfirmedBlob( TPartitionDatabase& db, ui64 commitId, - const TBlobUniqueIdWithRange& blob); + const TBlobToConfirm& blob); void ConfirmedBlobsAdded(TPartitionDatabase& db, ui64 commitId); - void BlobsConfirmed(ui64 commitId); + void BlobsConfirmed(ui64 commitId, TVector blobs); // // AddConfirmedBlobs diff --git a/cloud/blockstore/libs/storage/partition/part_tx.h b/cloud/blockstore/libs/storage/partition/part_tx.h index e7428cb1fe4..2c65e545da2 100644 --- a/cloud/blockstore/libs/storage/partition/part_tx.h +++ b/cloud/blockstore/libs/storage/partition/part_tx.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include @@ -109,7 +109,7 @@ struct TTxPartition TVector CleanupQueue; TVector NewBlobs; TVector GarbageBlobs; - TCommitIdToBlobUniqueIdWithRange UnconfirmedBlobs; + TCommitIdToBlobsToConfirm UnconfirmedBlobs; explicit TLoadState(ui64 blocksCount) : UsedBlocks(blocksCount) @@ -781,7 +781,12 @@ struct TTxPartition ui16 blobOffset) { Y_DEBUG_ABORT_UNLESS(BlockRange.Contains(blockIndex)); - BlockMarks.push_back({ blobId, commitId, blockIndex, blobOffset }); + BlockMarks.push_back({ + blobId, + commitId, + blockIndex, + blobOffset, + }); } }; @@ -800,6 +805,7 @@ struct TTxPartition ui64 CommitId = 0; ui32 BlockIndex = 0; ui16 BlobOffset = 0; + ui32 Checksum = 0; }; TVector BlockMarks; @@ -816,9 +822,18 @@ struct TTxPartition BlockMarks.clear(); } - void MarkBlock(ui32 blockIndex, ui64 commitId, ui16 blobOffset) + void MarkBlock( + ui32 blockIndex, + ui64 commitId, + ui16 blobOffset, + ui32 checksum) { - BlockMarks.push_back({ commitId, blockIndex, blobOffset }); + BlockMarks.push_back({ + commitId, + blockIndex, + blobOffset, + checksum, + }); } }; @@ -1100,12 +1115,12 @@ struct TTxPartition { const TRequestInfoPtr RequestInfo; ui64 CommitId = 0; - TVector Blobs; + TVector Blobs; TAddUnconfirmedBlobs( TRequestInfoPtr requestInfo, ui64 commitId, - TVector blobs) + TVector blobs) : RequestInfo(std::move(requestInfo)) , CommitId(commitId) , Blobs(std::move(blobs)) @@ -1124,10 +1139,14 @@ struct TTxPartition struct TConfirmBlobs { const TRequestInfoPtr RequestInfo; + const ui64 StartCycleCount; TVector UnrecoverableBlobs; - explicit TConfirmBlobs(TVector unrecoverableBlobs) - : UnrecoverableBlobs(std::move(unrecoverableBlobs)) + TConfirmBlobs( + ui64 startCycleCount, + TVector unrecoverableBlobs) + : StartCycleCount(startCycleCount) + , UnrecoverableBlobs(std::move(unrecoverableBlobs)) {} void Clear() diff --git a/cloud/blockstore/libs/storage/partition/part_ut.cpp b/cloud/blockstore/libs/storage/partition/part_ut.cpp index 846c908d997..dea147bb46b 100644 --- a/cloud/blockstore/libs/storage/partition/part_ut.cpp +++ b/cloud/blockstore/libs/storage/partition/part_ut.cpp @@ -7181,7 +7181,7 @@ Y_UNIT_TEST_SUITE(TPartitionTest) BuildRemoteHttpQuery(TestTabletId, {{"action","collectGarbage"}}), HTTP_METHOD::HTTP_METHOD_POST); - UNIT_ASSERT_C(httpResponse->Html.Contains("Tablet is dead"), true); + UNIT_ASSERT_C(httpResponse->Html.Contains("tablet is shutting down"), true); } Y_UNIT_TEST(ShouldForgetTooOldCompactRangeOperations) @@ -9556,7 +9556,7 @@ Y_UNIT_TEST_SUITE(TPartitionTest) UNIT_ASSERT(!garbageCollectorFinished); } - Y_UNIT_TEST(ShouldWriteBlocksWhenAddingUnconfirmedBlobsEnabled) + Y_UNIT_TEST(ShouldWriteBlocksWhenAddingUnconfirmedBlobs) { auto config = DefaultConfig(); config.SetWriteBlobThreshold(1); @@ -9589,6 +9589,8 @@ Y_UNIT_TEST_SUITE(TPartitionTest) auto response = partition.StatPartition(); const auto& stats = response->Record.GetStats(); UNIT_ASSERT_VALUES_EQUAL(2, stats.GetMergedBlobsCount()); + UNIT_ASSERT_VALUES_EQUAL(0, stats.GetUnconfirmedBlobCount()); + UNIT_ASSERT_VALUES_EQUAL(0, stats.GetConfirmedBlobCount()); } UNIT_ASSERT_VALUES_EQUAL( GetBlockContent(1), @@ -9625,6 +9627,13 @@ Y_UNIT_TEST_SUITE(TPartitionTest) UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetError().GetCode()); } + { + auto response = partition.StatPartition(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL(0, stats.GetUnconfirmedBlobCount()); + UNIT_ASSERT_VALUES_EQUAL(1, stats.GetConfirmedBlobCount()); + } + // but we can work with other ranges UNIT_ASSERT_VALUES_EQUAL( GetBlockContent(1), @@ -9648,6 +9657,13 @@ Y_UNIT_TEST_SUITE(TPartitionTest) GetBlockContent(2), GetBlockContent(partition.ReadBlocks(11)) ); + + { + auto response = partition.StatPartition(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL(0, stats.GetUnconfirmedBlobCount()); + UNIT_ASSERT_VALUES_EQUAL(0, stats.GetConfirmedBlobCount()); + } } Y_UNIT_TEST(ShouldConfirmBlobs) @@ -9673,6 +9689,7 @@ Y_UNIT_TEST_SUITE(TPartitionTest) if (spoofWriteBlobs) { auto response = std::make_unique(); + response->BlockChecksums.resize(1); runtime->Send(new IEventHandle( event->Sender, event->Recipient, @@ -10693,6 +10710,235 @@ Y_UNIT_TEST_SUITE(TPartitionTest) response->GetErrorReason()); } } + + void EnableReadBlobCorruption(TTestActorRuntime& runtime) + { + std::shared_ptr sgList; + + runtime.SetEventFilter([=] ( + TTestActorRuntimeBase& runtime, + TAutoPtr& event) mutable + { + Y_UNUSED(runtime); + + switch (event->GetTypeRewrite()) { + case TEvPartitionCommonPrivate::EvReadBlobRequest: { + using TEv = + TEvPartitionCommonPrivate::TEvReadBlobRequest; + auto* msg = event->Get(); + sgList = std::make_shared(msg->Sglist); + + break; + } + case TEvPartitionCommonPrivate::EvReadBlobResponse: { + auto g = sgList->Acquire(); + UNIT_ASSERT(g); + UNIT_ASSERT_VALUES_UNEQUAL(0, g.Get().size()); + + const char* block = g.Get()[0].Data(); + memset(const_cast(block), '0', DefaultBlockSize); + break; + } + } + return false; + }); + + } + + Y_UNIT_TEST(ShouldDetectBlockCorruptionInBlobsWhenAddingUnconfirmedBlobs) + { + constexpr ui32 blockCount = 1024 * 1024; + auto config = DefaultConfig(); + config.SetCheckBlockChecksumsInBlobsUponRead(true); + config.SetAddingUnconfirmedBlobsEnabled(true); + auto runtime = PrepareTestActorRuntime(config, blockCount); + + TPartitionClient partition(*runtime); + partition.WaitReady(); + + const auto range = TBlockRange32::WithLength(0, 1024); + partition.WriteBlocks(range, 1); + + EnableReadBlobCorruption(*runtime); + + // direct read should fail + TVector blocks; + auto sglist = ResizeBlocks( + blocks, + range.Size(), + TString::TUninitialized(DefaultBlockSize)); + partition.SendReadBlocksLocalRequest(range, std::move(sglist)); + auto response = partition.RecvReadBlocksLocalResponse(); + UNIT_ASSERT_VALUES_EQUAL_C( + E_REJECTED, + response->GetStatus(), + response->GetErrorReason()); + } + + Y_UNIT_TEST(ShouldProperlyCalculateBlockChecksumsForBatchedWrites) + { + constexpr ui32 blockCount = 1024 * 1024; + auto config = DefaultConfig(); + // enabling batching + checksum checks + config.SetCheckBlockChecksumsInBlobsUponRead(true); + config.SetWriteRequestBatchingEnabled(true); + // removing the dependency on the current defaults + config.SetWriteBlobThreshold(128_KB); + config.SetMaxBlobRangeSize(128_MB); + // disabling flush + config.SetFlushThreshold(1_GB); + // enabling fresh channel writes to make stats check a bit more + // convenient + config.SetFreshChannelCount(1); + config.SetFreshChannelWriteRequestsEnabled(true); + auto runtime = PrepareTestActorRuntime(config, blockCount); + + TPartitionClient partition(*runtime); + partition.WaitReady(); + + std::unique_ptr processQueue; + bool intercept = true; + + runtime->SetEventFilter([&] ( + TTestActorRuntimeBase& runtime, + TAutoPtr& event) + { + Y_UNUSED(runtime); + + switch (event->GetTypeRewrite()) { + case TEvPartitionPrivate::EvProcessWriteQueue: { + if (intercept) { + processQueue.reset(event.Release()); + intercept = false; + + return true; + } + + break; + } + } + + return false; + }); + + // the following 14 blocks get batched into mixed blob 1 + partition.SendWriteBlocksRequest( + TBlockRange32::WithLength(208792, 1), + 'a'); + partition.SendWriteBlocksRequest( + TBlockRange32::WithLength(208796, 1), + 'b'); + partition.SendWriteBlocksRequest( + TBlockRange32::WithLength(208799, 1), + 'c'); + partition.SendWriteBlocksRequest( + TBlockRange32::WithLength(208864, 1), + 'd'); + partition.SendWriteBlocksRequest( + TBlockRange32::WithLength(208866, 10), + 'e'); + // the following 17 blocks get batched into mixed blob 2 + partition.SendWriteBlocksRequest( + TBlockRange32::WithLength(251925, 17), + 'f'); + // and the remaining block goes to fresh blocks + partition.SendWriteBlocksRequest( + TBlockRange32::WithLength(800000, 1), + 'g'); + + runtime->DispatchEvents({}, TDuration::Seconds(2)); + runtime->Send(processQueue.release()); + partition.RecvWriteBlocksResponse(); + + // checking that mixed batching has actually worked + { + const auto stats = partition.StatPartition()->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL(2, stats.GetMixedBlobsCount()); + UNIT_ASSERT_VALUES_EQUAL(31, stats.GetMixedBlocksCount()); + UNIT_ASSERT_VALUES_EQUAL(1, stats.GetFreshBlobsCount()); + UNIT_ASSERT_VALUES_EQUAL(1, stats.GetFreshBlocksCount()); + } + + // checking that we don't run into a failed checksum check upon read + { + TVector blocks; + auto sglist = ResizeBlocks( + blocks, + 1024, + TString::TUninitialized(DefaultBlockSize)); + partition.SendReadBlocksLocalRequest( + TBlockRange32::WithLength(208792, 1024), + std::move(sglist)); + auto response = partition.RecvReadBlocksLocalResponse(); + UNIT_ASSERT_VALUES_EQUAL_C( + S_OK, + response->GetStatus(), + response->GetErrorReason()); + } + + EnableReadBlobCorruption(*runtime); + + // checking that we actually saved the checksums + { + TVector blocks; + auto sglist = ResizeBlocks( + blocks, + 1024, + TString::TUninitialized(DefaultBlockSize)); + partition.SendReadBlocksLocalRequest( + TBlockRange32::WithLength(208792, 1024), + std::move(sglist)); + auto response = partition.RecvReadBlocksLocalResponse(); + UNIT_ASSERT_VALUES_EQUAL_C( + E_REJECTED, + response->GetStatus(), + response->GetErrorReason()); + } + } + + Y_UNIT_TEST(ShouldCancelRequestsOnTabletRestart) + { + constexpr ui32 blockCount = 1024 * 1024; + auto runtime = PrepareTestActorRuntime({}, blockCount); + + TPartitionClient partition(*runtime); + partition.WaitReady(); + + const auto range = TBlockRange32::WithLength(0, 1024); + partition.SendWriteBlocksRequest(range, 1); + + bool putSeen = false; + runtime->SetEventFilter([&] (auto& runtime, auto& event) { + Y_UNUSED(runtime); + + switch (event->GetTypeRewrite()) { + case TEvBlobStorage::EvPutResult: { + putSeen = true; + return true; + } + } + + return false; + }); + + TDispatchOptions options; + options.CustomFinalCondition = [&] { + return putSeen; + }; + runtime->DispatchEvents(options); + + partition.RebootTablet(); + + auto response = partition.RecvWriteBlocksResponse(); + UNIT_ASSERT_VALUES_EQUAL_C( + E_REJECTED, + response->GetStatus(), + response->GetErrorReason()); + + UNIT_ASSERT_VALUES_EQUAL( + "tablet is shutting down", + response->GetErrorReason()); + } } } // namespace NCloud::NBlockStore::NStorage::NPartition diff --git a/cloud/blockstore/libs/storage/partition/ya.make b/cloud/blockstore/libs/storage/partition/ya.make index a213e9e1b16..ffdf7b50245 100644 --- a/cloud/blockstore/libs/storage/partition/ya.make +++ b/cloud/blockstore/libs/storage/partition/ya.make @@ -64,6 +64,7 @@ PEERDIR( cloud/storage/core/libs/api cloud/storage/core/libs/common cloud/storage/core/libs/tablet + cloud/storage/core/libs/viewer library/cpp/blockcodecs library/cpp/cgiparam @@ -76,7 +77,6 @@ PEERDIR( contrib/ydb/core/scheme contrib/ydb/core/tablet contrib/ydb/core/tablet_flat - contrib/ydb/core/testlib/basics contrib/ydb/library/actors/core ) diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor.cpp index 3b537006596..f1df67abb75 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor.cpp @@ -1,9 +1,11 @@ #include "part2_actor.h" #include -#include #include +#include +#include + #include #include #include @@ -340,20 +342,35 @@ void TPartitionActor::KillActors(const TActorContext& ctx) } } -void TPartitionActor::AddTransaction(TRequestInfo& requestInfo) +void TPartitionActor::AddTransaction( + TRequestInfo& transaction, + TRequestInfo::TCancelRoutine cancelRoutine) { - requestInfo.Ref(); + transaction.CancelRoutine = cancelRoutine; - Y_ABORT_UNLESS(requestInfo.Empty()); - ActiveTransactions.PushBack(&requestInfo); + transaction.Ref(); + + STORAGE_VERIFY( + transaction.Empty(), + TWellKnownEntityTypes::TABLET, + TabletID()); + ActiveTransactions.PushBack(&transaction); } void TPartitionActor::RemoveTransaction(TRequestInfo& requestInfo) { - Y_ABORT_UNLESS(!requestInfo.Empty()); + STORAGE_VERIFY( + !requestInfo.Empty(), + TWellKnownEntityTypes::TABLET, + TabletID()); + requestInfo.Unlink(); - Y_ABORT_UNLESS(requestInfo.RefCount() > 1); + STORAGE_VERIFY( + requestInfo.RefCount() > 1, + TWellKnownEntityTypes::TABLET, + TabletID()); + requestInfo.UnRef(); } @@ -361,9 +378,12 @@ void TPartitionActor::TerminateTransactions(const TActorContext& ctx) { while (ActiveTransactions) { auto* requestInfo = ActiveTransactions.PopFront(); - requestInfo->CancelRequest(ctx); + STORAGE_VERIFY( + requestInfo->RefCount() >= 1, + TWellKnownEntityTypes::TABLET, + TabletID()); - Y_ABORT_UNLESS(requestInfo->RefCount() >= 1); + requestInfo->CancelRequest(ctx); requestInfo->UnRef(); } } diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor.h b/cloud/blockstore/libs/storage/partition2/part2_actor.h index 8bfb656e169..6ab78f66978 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor.h +++ b/cloud/blockstore/libs/storage/partition2/part2_actor.h @@ -185,9 +185,28 @@ class TPartitionActor final void BeforeDie(const NActors::TActorContext& ctx); void KillActors(const NActors::TActorContext& ctx); - void AddTransaction(TRequestInfo& transaction); + void AddTransaction( + TRequestInfo& transaction, + TRequestInfo::TCancelRoutine cancelRoutine); + + template + void AddTransaction(TRequestInfo& transaction) + { + auto cancelRoutine = [] ( + const NActors::TActorContext& ctx, + TRequestInfo& requestInfo) + { + auto response = std::make_unique( + MakeError(E_REJECTED, "tablet is shutting down")); + + NCloud::Reply(ctx, requestInfo, std::move(response)); + }; + + AddTransaction(transaction, cancelRoutine); + } void RemoveTransaction(TRequestInfo& transaction); void TerminateTransactions(const NActors::TActorContext& ctx); + void ReleaseTransactions(); ui64 CalcChannelHistorySize() const; diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_addblobs.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_addblobs.cpp index 7ea16b380b2..2c253943a72 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_addblobs.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_addblobs.cpp @@ -25,7 +25,7 @@ void TPartitionActor::HandleAddBlobs( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -42,7 +42,7 @@ void TPartitionActor::HandleAddBlobs( "[%lu] Start adding blobs", TabletID()); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_addgarbage.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_addgarbage.cpp index 66fb21ed788..51974315e39 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_addgarbage.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_addgarbage.cpp @@ -23,7 +23,7 @@ void TPartitionActor::HandleAddGarbage( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -36,7 +36,7 @@ void TPartitionActor::HandleAddGarbage( "AddGarbage", requestInfo->CallContext->RequestId); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_changedblocks.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_changedblocks.cpp index f34280aaf9e..014c5b32f52 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_changedblocks.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_changedblocks.cpp @@ -232,7 +232,7 @@ void TGetChangedBlocksActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); auto response = std::make_unique(error); @@ -313,7 +313,7 @@ void TPartitionActor::HandleGetChangedBlocks( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -434,7 +434,7 @@ void TPartitionActor::HandleGetChangedBlocks( highCommitId, DescribeRange(readRange).data()); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_cleanup.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_cleanup.cpp index 9be2c4cafc2..36bdfa654b4 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_cleanup.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_cleanup.cpp @@ -43,7 +43,7 @@ void TPartitionActor::HandleCleanup( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -87,7 +87,7 @@ void TPartitionActor::HandleCleanup( auto createTxAndEnqueueIntoCCCQueue = [&] (ui64 commitId, auto onStartProcessing) { State->SetCleanupStatus(EOperationStatus::Delayed); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); auto tx = CreateTx(requestInfo, commitId, msg->Mode); auto& queue = State->GetCCCRequestQueue(); diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_collectgarbage.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_collectgarbage.cpp index 8f605b9bb67..e613aa1133e 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_collectgarbage.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_collectgarbage.cpp @@ -562,7 +562,7 @@ void TPartitionActor::HandleCollectGarbage( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -612,7 +612,7 @@ void TPartitionActor::HandleCollectGarbage( State->SetCollectGarbageStatus(EOperationStatus::Started); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx(ctx, requestInfo, commitId); return; diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_compaction.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_compaction.cpp index 202aa0252ac..1e5a68b127e 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_compaction.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_compaction.cpp @@ -530,7 +530,7 @@ void TCompactionActor::HandlePoisonPill( Y_UNUSED(ev); auto response = std::make_unique( - MakeError(E_REJECTED, "Tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); ReplyAndDie(ctx, std::move(response)); } @@ -703,7 +703,7 @@ void TPartitionActor::HandleCompaction( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -825,7 +825,7 @@ void TPartitionActor::HandleCompaction( State->SetCompactionStatus(EOperationStatus::Started); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); auto& queue = State->GetCCCRequestQueue(); // shouldn't wait for inflight fresh blocks to complete (commitID == 0) diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_compactrange.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_compactrange.cpp index f73787e568c..8b9e82bfab2 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_compactrange.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_compactrange.cpp @@ -163,7 +163,7 @@ void TForcedCompactionActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); ReplyAndDie(ctx, error); } diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_createcheckpoint.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_createcheckpoint.cpp index af25568d702..c7a3ece2c91 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_createcheckpoint.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_createcheckpoint.cpp @@ -21,7 +21,7 @@ void TPartitionActor::HandleCreateCheckpoint( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -84,7 +84,7 @@ void TPartitionActor::HandleCreateCheckpoint( State->WriteCheckpoint(meta); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); auto tx = CreateTx( requestInfo, diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_deletecheckpoint.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_deletecheckpoint.cpp index aa292cd7a1c..80471fcee56 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_deletecheckpoint.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_deletecheckpoint.cpp @@ -21,7 +21,7 @@ void TPartitionActor::HandleDeleteCheckpoint( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -65,7 +65,7 @@ void TPartitionActor::HandleDeleteCheckpoint( return; } - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); auto tx = CreateTx( requestInfo, diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_deletegarbage.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_deletegarbage.cpp index 06341ddc26d..1a7116a054d 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_deletegarbage.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_deletegarbage.cpp @@ -19,7 +19,7 @@ void TPartitionActor::HandleDeleteGarbage( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -32,7 +32,7 @@ void TPartitionActor::HandleDeleteGarbage( "DeleteGarbage", requestInfo->CallContext->RequestId); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_describeblocks.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_describeblocks.cpp index 895aa7d0887..2ee172b5cfc 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_describeblocks.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_describeblocks.cpp @@ -68,7 +68,7 @@ void TPartitionActor::DescribeBlocks( commitId, DescribeRange(describeRange).data()); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, @@ -83,7 +83,7 @@ void TPartitionActor::HandleDescribeBlocks( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_flush.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_flush.cpp index 2278dd54eb3..ab68b67594d 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_flush.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_flush.cpp @@ -264,7 +264,7 @@ void TFlushActor::HandlePoisonPill( Y_UNUSED(ev); auto respose = std::make_unique( - MakeError(E_REJECTED, "Tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); ReplyAndDie(ctx, std::move(respose)); } diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_monitoring_cleanup.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_monitoring_cleanup.cpp index d721edb01ea..a589487b758 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_monitoring_cleanup.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_monitoring_cleanup.cpp @@ -147,7 +147,7 @@ void TForcedCleanupActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); NotifyCompleted(ctx, error); } diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_monitoring_garbage.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_monitoring_garbage.cpp index c9bc715fcf3..a5632cc4a78 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_monitoring_garbage.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_monitoring_garbage.cpp @@ -88,8 +88,7 @@ THttpGarbageActor::THttpGarbageActor( , TabletId(tabletId) , Action(action) , BlobIds(std::move(blobIds)) -{ -} +{} void THttpGarbageActor::Bootstrap(const TActorContext& ctx) { @@ -174,7 +173,7 @@ void THttpGarbageActor::HandleAddGarbageRequest( { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "Tablet is dead")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } void THttpGarbageActor::HandleCollectGarbageRequest( @@ -183,7 +182,7 @@ void THttpGarbageActor::HandleCollectGarbageRequest( { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "Tablet is dead")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } STFUNC(THttpGarbageActor::StateWork) diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_monitoring_view.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_monitoring_view.cpp index e02a55a12d7..7552b012d74 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_monitoring_view.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_monitoring_view.cpp @@ -242,7 +242,7 @@ void THttpReadBlockActor::HandlePoisonPill( { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "Tablet is dead")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } STFUNC(THttpReadBlockActor::StateWork) diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_readblob.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_readblob.cpp index dd0a107064d..4344b58cf8d 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_readblob.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_readblob.cpp @@ -311,7 +311,7 @@ void TReadBlobActor::HandlePoisonPill( Y_UNUSED(ev); auto response = std::make_unique( - MakeError(E_REJECTED, "Tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); ReplyAndDie(ctx, std::move(response)); } diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_readblocks.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_readblocks.cpp index bc87bd357e5..cb5f43ebd2d 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_readblocks.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_readblocks.cpp @@ -670,7 +670,7 @@ void TReadBlocksActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); auto response = CreateReadBlocksResponse(ReplyLocal, error); ReplyAndDie(ctx, std::move(response), error); @@ -942,7 +942,7 @@ void TPartitionActor::ReadBlocks( commitId, DescribeRange(readRange).data()); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo, requestInfo->CancelRoutine); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_update_index_structures.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_update_index_structures.cpp index c640ad41ec0..2cefe4f26c4 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_update_index_structures.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_update_index_structures.cpp @@ -57,7 +57,7 @@ void TPartitionActor::HandleUpdateIndexStructures( auto* msg = ev->Get(); auto requestInfo = - CreateRequestInfo( + CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -103,7 +103,7 @@ void TPartitionActor::HandleUpdateIndexStructures( requestInfo, msg->BlockRange); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx(ctx, std::move(tx)); } diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_writeblob.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_writeblob.cpp index b26b3253a59..c9d3f07ad3f 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_writeblob.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_writeblob.cpp @@ -234,7 +234,7 @@ void TWriteBlobActor::HandlePoisonPill( Y_UNUSED(ev); auto response = std::make_unique( - MakeError(E_REJECTED, "Tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); ReplyAndDie(ctx, std::move(response)); } diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_writefreshblocks.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_writefreshblocks.cpp index a6941919520..a47cbbb39ad 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_writefreshblocks.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_writefreshblocks.cpp @@ -401,7 +401,7 @@ void TWriteFreshBlocksActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); ReplyAllAndDie(ctx, error); } diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_writemergedblocks.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_writemergedblocks.cpp index fae7dde77f3..27ab1a39e17 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_writemergedblocks.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_writemergedblocks.cpp @@ -356,7 +356,7 @@ void TWriteMergedBlocksActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); ReplyAndDie(ctx, error); } diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_writemixedblocks.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_writemixedblocks.cpp index 5199fc7b92e..e2ed12bc55d 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_writemixedblocks.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_writemixedblocks.cpp @@ -446,7 +446,7 @@ void TWriteMixedBlocksActor::HandlePoisonPill( { Y_UNUSED(ev); - auto error = MakeError(E_REJECTED, "Tablet is dead"); + auto error = MakeError(E_REJECTED, "tablet is shutting down"); ReplyAllAndDie(ctx, error); } diff --git a/cloud/blockstore/libs/storage/partition2/part2_actor_zeroblocks.cpp b/cloud/blockstore/libs/storage/partition2/part2_actor_zeroblocks.cpp index 4680e89eefd..b6805a1cebe 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_actor_zeroblocks.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_actor_zeroblocks.cpp @@ -186,7 +186,7 @@ void TZeroBlocksActor::HandlePoisonPill( Y_UNUSED(ev); auto respose = std::make_unique( - MakeError(E_REJECTED, "Tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); ReplyAndDie(ctx, std::move(respose)); } @@ -214,7 +214,7 @@ void TPartitionActor::HandleZeroBlocks( { auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( + auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, msg->CallContext); @@ -332,7 +332,7 @@ void TPartitionActor::HandleZeroBlocks( TabletID(), DescribeRange(writeRange).data()); - AddTransaction(*requestInfo); + AddTransaction(*requestInfo); ExecuteTx( ctx, diff --git a/cloud/blockstore/libs/storage/partition2/part2_ut.cpp b/cloud/blockstore/libs/storage/partition2/part2_ut.cpp index b0ae02c5942..0f7f92b5dd9 100644 --- a/cloud/blockstore/libs/storage/partition2/part2_ut.cpp +++ b/cloud/blockstore/libs/storage/partition2/part2_ut.cpp @@ -4852,7 +4852,7 @@ Y_UNIT_TEST_SUITE(TPartition2Test) BuildRemoteHttpQuery(TestTabletId, {{"action","collectGarbage"}}), HTTP_METHOD::HTTP_METHOD_POST); - UNIT_ASSERT_C(httpResponse->Html.Contains("Tablet is dead"), true); + UNIT_ASSERT_C(httpResponse->Html.Contains("tablet is shutting down"), true); } Y_UNIT_TEST(ShouldNotDeleteBlobsMarkedAsGarbageByUncommittedTransactions) @@ -6979,6 +6979,50 @@ Y_UNIT_TEST_SUITE(TPartition2Test) response->GetStatus(), response->GetErrorReason()); } + + Y_UNIT_TEST(ShouldCancelRequestsOnTabletRestart) + { + constexpr ui32 blockCount = 1024 * 1024; + auto runtime = PrepareTestActorRuntime({}, blockCount); + + TPartitionClient partition(*runtime); + partition.WaitReady(); + + const auto range = TBlockRange32::WithLength(0, 1024); + partition.SendWriteBlocksRequest(range, 1); + + bool putSeen = false; + runtime->SetEventFilter([&] (auto& runtime, auto& event) { + Y_UNUSED(runtime); + + switch (event->GetTypeRewrite()) { + case TEvBlobStorage::EvPutResult: { + putSeen = true; + return true; + } + } + + return false; + }); + + TDispatchOptions options; + options.CustomFinalCondition = [&] { + return putSeen; + }; + runtime->DispatchEvents(options); + + partition.RebootTablet(); + + auto response = partition.RecvWriteBlocksResponse(); + UNIT_ASSERT_VALUES_EQUAL_C( + E_REJECTED, + response->GetStatus(), + response->GetErrorReason()); + + UNIT_ASSERT_VALUES_EQUAL( + "tablet is shutting down", + response->GetErrorReason()); + } } } // namespace NCloud::NBlockStore::NStorage::NPartition2 diff --git a/cloud/blockstore/libs/storage/partition2/ya.make b/cloud/blockstore/libs/storage/partition2/ya.make index 93d3e274a7e..63fb42fb752 100644 --- a/cloud/blockstore/libs/storage/partition2/ya.make +++ b/cloud/blockstore/libs/storage/partition2/ya.make @@ -72,7 +72,6 @@ PEERDIR( contrib/ydb/core/scheme contrib/ydb/core/tablet contrib/ydb/core/tablet_flat - contrib/ydb/core/testlib/basics contrib/ydb/library/actors/core ) diff --git a/cloud/blockstore/libs/storage/partition_common/actor_describe_base_disk_blocks.cpp b/cloud/blockstore/libs/storage/partition_common/actor_describe_base_disk_blocks.cpp index 9a4f66b8d46..6548136e945 100644 --- a/cloud/blockstore/libs/storage/partition_common/actor_describe_base_disk_blocks.cpp +++ b/cloud/blockstore/libs/storage/partition_common/actor_describe_base_disk_blocks.cpp @@ -226,7 +226,7 @@ void TDescribeBaseDiskBlocksActor::HandlePoisonPill( { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "Tablet is dead")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } STFUNC(TDescribeBaseDiskBlocksActor::StateWork) diff --git a/cloud/blockstore/libs/storage/partition_common/actor_read_blob.cpp b/cloud/blockstore/libs/storage/partition_common/actor_read_blob.cpp index 16cfb1de120..1cb604292e3 100644 --- a/cloud/blockstore/libs/storage/partition_common/actor_read_blob.cpp +++ b/cloud/blockstore/libs/storage/partition_common/actor_read_blob.cpp @@ -275,7 +275,7 @@ void TReadBlobActor::HandlePoisonPill( Y_UNUSED(ev); auto response = std::make_unique( - MakeError(E_REJECTED, "Tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); ReplyAndDie(ctx, std::move(response)); } diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/checksum_range.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/checksum_range.cpp new file mode 100644 index 00000000000..0621665b15b --- /dev/null +++ b/cloud/blockstore/libs/storage/partition_nonrepl/checksum_range.cpp @@ -0,0 +1,114 @@ +#include "checksum_range.h" + +#include +#include + +#include + +namespace NCloud::NBlockStore::NStorage { + +using namespace NActors; + +//////////////////////////////////////////////////////////////////////////////// + +TChecksumRangeActorCompanion::TChecksumRangeActorCompanion( + TBlockRange64 range, + TVector replicas) + : Range(range) + , Replicas(std::move(replicas)) +{ + Checksums.resize(Replicas.size()); +} + +bool TChecksumRangeActorCompanion::IsFinished() const +{ + return Finished; +} + +const TVector& TChecksumRangeActorCompanion::GetChecksums() const +{ + return Checksums; +} + +NProto::TError TChecksumRangeActorCompanion::GetError() const +{ + return Error; +} + +TInstant TChecksumRangeActorCompanion::GetChecksumStartTs() const +{ + return ChecksumStartTs; +} + +TDuration TChecksumRangeActorCompanion::GetChecksumDuration() const +{ + return ChecksumDuration; +} + +void TChecksumRangeActorCompanion::CalculateChecksums(const TActorContext& ctx) +{ + for (size_t i = 0; i < Replicas.size(); ++i) { + CalculateReplicaChecksum(ctx, i); + } + ChecksumStartTs = ctx.Now(); +} + +void TChecksumRangeActorCompanion::CalculateReplicaChecksum(const TActorContext& ctx, int idx) +{ + auto request = std::make_unique(); + request->Record.SetStartIndex(Range.Start); + request->Record.SetBlocksCount(Range.Size()); + + auto* headers = request->Record.MutableHeaders(); + headers->SetIsBackgroundRequest(true); + headers->SetClientId(TString(BackgroundOpsClientId)); + + auto event = std::make_unique( + Replicas[idx].ActorId, + ctx.SelfID, + request.release(), + IEventHandle::FlagForwardOnNondelivery, + idx, // cookie + &ctx.SelfID // forwardOnNondelivery + ); + + ctx.Send(event.release()); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TChecksumRangeActorCompanion::HandleChecksumResponse( + const TEvNonreplPartitionPrivate::TEvChecksumBlocksResponse::TPtr& ev, + const TActorContext& ctx) +{ + auto* msg = ev->Get(); + + Error = msg->Record.GetError(); + + if (HasError(Error)) { + LOG_WARN(ctx, TBlockStoreComponents::PARTITION, + "[%s] Checksum error %s", + Replicas[0].Name.c_str(), + FormatError(Error).c_str()); + + ChecksumDuration = ctx.Now() - ChecksumStartTs; + Finished = true; + return; + } + + Checksums[ev->Cookie] = msg->Record.GetChecksum(); + if (++CalculatedChecksumsCount == Replicas.size()) { + ChecksumDuration = ctx.Now() - ChecksumStartTs; + Finished = true; + } +} + +void TChecksumRangeActorCompanion::HandleChecksumUndelivery( + const NActors::TActorContext& ctx) +{ + ChecksumDuration = ctx.Now() - ChecksumStartTs; + + Error = MakeError(E_REJECTED, "ChecksumBlocks request undelivered"); +} + +} // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/checksum_range.h b/cloud/blockstore/libs/storage/partition_nonrepl/checksum_range.h new file mode 100644 index 00000000000..1bdf4a1baf2 --- /dev/null +++ b/cloud/blockstore/libs/storage/partition_nonrepl/checksum_range.h @@ -0,0 +1,49 @@ +#pragma once + +#include "part_mirror_resync_util.h" +#include "part_nonrepl_events_private.h" + +#include + +#include + +namespace NCloud::NBlockStore::NStorage { + +//////////////////////////////////////////////////////////////////////////////// + +class TChecksumRangeActorCompanion +{ +private: + const TBlockRange64 Range; + const TVector Replicas; + + TInstant ChecksumStartTs; + TDuration ChecksumDuration; + ui32 CalculatedChecksumsCount = 0; + bool Finished = false; + TVector Checksums; + NProto::TError Error; + +public: + TChecksumRangeActorCompanion( + TBlockRange64 range, + TVector replicas); + + void CalculateChecksums(const NActors::TActorContext& ctx); + + void HandleChecksumResponse( + const TEvNonreplPartitionPrivate::TEvChecksumBlocksResponse::TPtr& ev, + const NActors::TActorContext& ctx); + void HandleChecksumUndelivery(const NActors::TActorContext& ctx); + + bool IsFinished() const; + const TVector& GetChecksums() const; + NProto::TError GetError() const; + TInstant GetChecksumStartTs() const; + TDuration GetChecksumDuration() const; + +private: + void CalculateReplicaChecksum(const NActors::TActorContext& ctx, int idx); +}; + +} // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/config.h b/cloud/blockstore/libs/storage/partition_nonrepl/config.h index 71ef5ddfb7c..909e4695e9a 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/config.h +++ b/cloud/blockstore/libs/storage/partition_nonrepl/config.h @@ -245,9 +245,7 @@ class TNonreplicatedPartitionConfig void AugmentErrorFlags(NCloud::NProto::TError& error) const { if (MuteIOErrors) { - ui32 flags = error.GetFlags(); - SetProtoFlag(flags, NCloud::NProto::EF_HW_PROBLEMS_DETECTED); - error.SetFlags(flags); + SetErrorProtoFlag(error, NCloud::NProto::EF_HW_PROBLEMS_DETECTED); } } diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator.cpp index d3ca11eb6cd..2eb4cdaf8e2 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator.cpp @@ -8,20 +8,19 @@ namespace NCloud::NBlockStore::NStorage { /////////////////////////////////////////////////////////////////////////////// TMigrationTimeoutCalculator::TMigrationTimeoutCalculator( - TStorageConfigPtr config, + ui32 maxMigrationBandwidthMiBs, + ui32 expectedDiskAgentSize, TNonreplicatedPartitionConfigPtr partitionConfig) - : Config(std::move(config)) + : MaxMigrationBandwidthMiBs(maxMigrationBandwidthMiBs) + , ExpectedDiskAgentSize(expectedDiskAgentSize) , PartitionConfig(std::move(partitionConfig)) {} TDuration TMigrationTimeoutCalculator::CalculateTimeout( TBlockRange64 nextProcessingRange) const { - const ui32 maxMigrationBandwidthMiBs = Config->GetMaxMigrationBandwidth(); - const ui32 expectedDiskAgentSize = Config->GetExpectedDiskAgentSize(); - // migration range is 4_MB - const auto migrationFactorPerAgent = maxMigrationBandwidthMiBs / 4; + const auto migrationFactorPerAgent = MaxMigrationBandwidthMiBs / 4.0; if (PartitionConfig->GetUseSimpleMigrationBandwidthLimiter()) { return TDuration::Seconds(1) / migrationFactorPerAgent; @@ -40,8 +39,8 @@ TDuration TMigrationTimeoutCalculator::CalculateTimeout( } const auto factor = - Max(migrationFactorPerAgent * agentDeviceCount / expectedDiskAgentSize, - 1U); + Max(migrationFactorPerAgent * agentDeviceCount / ExpectedDiskAgentSize, + 1.0); return TDuration::Seconds(1) / factor; } diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator.h b/cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator.h index 2376de99c75..1b5ab82618a 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator.h +++ b/cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator.h @@ -12,16 +12,18 @@ namespace NCloud::NBlockStore::NStorage { class TMigrationTimeoutCalculator { private: - const TStorageConfigPtr Config; + const ui32 MaxMigrationBandwidthMiBs = 0; + const ui32 ExpectedDiskAgentSize = 0; TNonreplicatedPartitionConfigPtr PartitionConfig; public: TMigrationTimeoutCalculator( - TStorageConfigPtr config, + ui32 maxMigrationBandwidthMiBs, + ui32 expectedDiskAgentSize, TNonreplicatedPartitionConfigPtr partitionConfig); - [[nodiscard]] TDuration CalculateTimeout( - TBlockRange64 nextProcessingRange) const; + [[nodiscard]] TDuration + CalculateTimeout(TBlockRange64 nextProcessingRange) const; }; } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator_ut.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator_ut.cpp index 6a507f0e1bd..28745ee1afc 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator_ut.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/migration_timeout_calculator_ut.cpp @@ -42,15 +42,6 @@ TDevices MakeDevices() return result; } -TStorageConfigPtr MakeStorageConfig(ui32 expectedDiskAgentSize) -{ - NProto::TStorageServiceConfig storageConfig; - storageConfig.SetMaxMigrationBandwidth(16); - storageConfig.SetExpectedDiskAgentSize(expectedDiskAgentSize); - - return std::make_shared(std::move(storageConfig), nullptr); -} - TNonreplicatedPartitionConfigPtr MakePartitionConfig( TDevices devices, bool useSimpleMigrationBandwidthLimiter) @@ -80,7 +71,8 @@ Y_UNIT_TEST_SUITE(TMigrationCalculatorTest) Y_UNIT_TEST(ShouldCalculateMigrationTimeout) { TMigrationTimeoutCalculator timeoutCalculator( - MakeStorageConfig(4), + 16, + 4, MakePartitionConfig(MakeDevices(), false)); // Devices #1, #2, #4 belong to Agent#1, device #3 belong to Agent#2. @@ -112,7 +104,8 @@ Y_UNIT_TEST_SUITE(TMigrationCalculatorTest) Y_UNIT_TEST(ShouldCalculateMigrationTimeoutWithSimpleLimiter) { TMigrationTimeoutCalculator timeoutCalculator( - MakeStorageConfig(100500), + 16, + 100500, MakePartitionConfig(MakeDevices(), true)); // When UseSimpleMigrationBandwidthLimiter enabled we expect the same diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor.cpp index 48f65e1c2b6..b93981dbf8d 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor.cpp @@ -110,15 +110,16 @@ void TMirrorPartitionResyncActor::SetupPartitions(const TActorContext& ctx) SelfId(), SelfId())); - for (const auto& replicaInfo: State.GetReplicaInfos()) { + const auto& replicaInfos = State.GetReplicaInfos(); + for (ui32 i = 0; i < replicaInfos.size(); i++) { IActorPtr actor = CreateNonreplicatedPartition( Config, - replicaInfo.Config, + replicaInfos[i].Config, TActorId(), // do not send stats RdmaClient); TActorId actorId = NCloud::Register(ctx, std::move(actor)); - Replicas.push_back({replicaInfo.Config->GetName(), actorId}); + Replicas.push_back({replicaInfos[i].Config->GetName(), i, actorId}); } } diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor.h b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor.h index 03bd5c8bfd2..ec89364e7c8 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor.h +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor.h @@ -48,7 +48,7 @@ class TMirrorPartitionResyncActor final TMirrorPartitionResyncState State; NActors::TActorId MirrorActorId; - TVector Replicas; + TVector Replicas; TPartitionDiskCountersPtr MirrorCounters; bool UpdateCountersScheduled = false; @@ -175,13 +175,13 @@ class TMirrorPartitionResyncActor final void ProcessReadRequestFastPath( const TEvService::TEvReadBlocksRequest::TPtr& ev, - TVector&& replicas, + TVector&& replicas, TBlockRange64 range, const NActors::TActorContext& ctx); void ProcessReadRequestFastPath( const TEvService::TEvReadBlocksLocalRequest::TPtr& ev, - TVector&& replicas, + TVector&& replicas, TBlockRange64 range, const NActors::TActorContext& ctx); diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor_readblocks.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor_readblocks.cpp index 4a52a568c54..807c4f27841 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor_readblocks.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor_readblocks.cpp @@ -37,7 +37,7 @@ void TMirrorPartitionResyncActor::ProcessReadRequestSyncPath( return; } - TVector replicas; + TVector replicas; // filtering out replicas with fresh devices for (ui32 i = 0; i < Replicas.size(); ++i) { if (State.GetReplicaInfos()[i].Config->DevicesReadyForReading(range)) { @@ -50,7 +50,7 @@ void TMirrorPartitionResyncActor::ProcessReadRequestSyncPath( void TMirrorPartitionResyncActor::ProcessReadRequestFastPath( const TEvService::TEvReadBlocksRequest::TPtr& ev, - TVector&& replicas, + TVector&& replicas, TBlockRange64 range, const TActorContext& ctx) { @@ -95,7 +95,7 @@ void TMirrorPartitionResyncActor::ProcessReadRequestFastPath( void TMirrorPartitionResyncActor::ProcessReadRequestFastPath( const TEvService::TEvReadBlocksLocalRequest::TPtr& ev, - TVector&& replicas, + TVector&& replicas, TBlockRange64 range, const TActorContext& ctx) { diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor_resync.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor_resync.cpp index a5314620665..754ee660a35 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor_resync.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_actor_resync.cpp @@ -105,7 +105,7 @@ void TMirrorPartitionResyncActor::ResyncNextRange(const TActorContext& ctx) MakeIntrusive() ); - TVector replicas; + TVector replicas; // filtering out replicas with fresh devices for (ui32 i = 0; i < Replicas.size(); ++i) { if (State.GetReplicaInfos()[i].Config->DevicesReadyForReading( diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_fastpath_actor.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_fastpath_actor.cpp index fe82f541ad7..ee94807a370 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_fastpath_actor.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_fastpath_actor.cpp @@ -22,7 +22,7 @@ TMirrorPartitionResyncFastPathActor::TMirrorPartitionResyncFastPathActor( ui32 blockSize, TBlockRange64 range, TGuardedSgList sgList, - TVector replicas, + TVector replicas, TString clientId) : RequestInfo(std::move(requestInfo)) , BlockSize(blockSize) diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_fastpath_actor.h b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_fastpath_actor.h index 19749aac017..1a67a2c5bd4 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_fastpath_actor.h +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_fastpath_actor.h @@ -31,7 +31,7 @@ class TMirrorPartitionResyncFastPathActor final const TRequestInfoPtr RequestInfo; const ui32 BlockSize; const TBlockRange64 Range; - const TVector Replicas; + const TVector Replicas; const TString ClientId; TGuardedSgList SgList; @@ -44,7 +44,7 @@ class TMirrorPartitionResyncFastPathActor final ui32 blockSize, TBlockRange64 range, TGuardedSgList sgList, - TVector replicas, + TVector replicas, TString clientId); void Bootstrap(const NActors::TActorContext& ctx); diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_util.h b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_util.h index c6a2fc348af..1f8ecec4856 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_util.h +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_mirror_resync_util.h @@ -2,6 +2,8 @@ #include +#include + #include #include @@ -10,6 +12,15 @@ namespace NCloud::NBlockStore::NStorage { //////////////////////////////////////////////////////////////////////////////// +struct TReplicaDescriptor +{ + TString Name; + ui32 ReplicaIndex = 0; + NActors::TActorId ActorId; +}; + +//////////////////////////////////////////////////////////////////////////////// + // TODO: increase x4? constexpr ui64 ResyncRangeSize = 4_MB; diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor_writeblocks.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor_writeblocks.cpp index f9bfd83c865..1479bfac54c 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor_writeblocks.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor_writeblocks.cpp @@ -34,7 +34,7 @@ class TDiskAgentWriteActor final const TVector DeviceRequests; const TNonreplicatedPartitionConfigPtr PartConfig; const TActorId Part; - const bool AssignIdToWriteAndZeroRequestsEnabled; + const bool AssignVolumeRequestId; TInstant StartTime; ui32 RequestsCompleted = 0; @@ -48,7 +48,7 @@ class TDiskAgentWriteActor final TVector deviceRequests, TNonreplicatedPartitionConfigPtr partConfig, const TActorId& part, - bool assignIdToWriteAndZeroRequestsEnabled, + bool assignVolumeRequestId, bool replyLocal); void Bootstrap(const TActorContext& ctx); @@ -86,15 +86,14 @@ TDiskAgentWriteActor::TDiskAgentWriteActor( TVector deviceRequests, TNonreplicatedPartitionConfigPtr partConfig, const TActorId& part, - bool assignIdToWriteAndZeroRequestsEnabled, + bool assignVolumeRequestId, bool replyLocal) : RequestInfo(std::move(requestInfo)) , Request(std::move(request)) , DeviceRequests(std::move(deviceRequests)) , PartConfig(std::move(partConfig)) , Part(part) - , AssignIdToWriteAndZeroRequestsEnabled( - assignIdToWriteAndZeroRequestsEnabled) + , AssignVolumeRequestId(assignVolumeRequestId) , ReplyLocal(replyLocal) {} @@ -130,7 +129,7 @@ void TDiskAgentWriteActor::WriteBlocks(const TActorContext& ctx) request->Record.SetDeviceUUID(deviceRequest.Device.GetDeviceUUID()); request->Record.SetStartIndex(deviceRequest.DeviceBlockRange.Start); request->Record.SetBlockSize(PartConfig->GetBlockSize()); - if (AssignIdToWriteAndZeroRequestsEnabled) { + if (AssignVolumeRequestId) { request->Record.SetVolumeRequestId(RequestInfo->Cookie); request->Record.SetMultideviceRequest(DeviceRequests.size() > 1); } @@ -351,6 +350,10 @@ void TNonreplicatedPartitionActor::HandleWriteBlocks( return; } + const bool assignVolumeRequestId = + Config->GetAssignIdToWriteAndZeroRequestsEnabled() && + !msg->Record.GetHeaders().GetIsBackgroundRequest(); + auto actorId = NCloud::Register( ctx, requestInfo, @@ -358,7 +361,7 @@ void TNonreplicatedPartitionActor::HandleWriteBlocks( std::move(deviceRequests), PartConfig, SelfId(), - Config->GetAssignIdToWriteAndZeroRequestsEnabled(), + assignVolumeRequestId, false); // replyLocal RequestsInProgress.AddWriteRequest(actorId, std::move(request)); @@ -440,6 +443,10 @@ void TNonreplicatedPartitionActor::HandleWriteBlocksLocal( msg->Record.BlocksCount, PartConfig->GetBlockSize())); + const bool assignVolumeRequestId = + Config->GetAssignIdToWriteAndZeroRequestsEnabled() && + !msg->Record.GetHeaders().GetIsBackgroundRequest(); + auto actorId = NCloud::Register( ctx, requestInfo, @@ -447,7 +454,7 @@ void TNonreplicatedPartitionActor::HandleWriteBlocksLocal( std::move(deviceRequests), PartConfig, SelfId(), - Config->GetAssignIdToWriteAndZeroRequestsEnabled(), + assignVolumeRequestId, true); // replyLocal RequestsInProgress.AddWriteRequest(actorId, std::move(request)); diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor_zeroblocks.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor_zeroblocks.cpp index 7b64de81bd9..47b507ec329 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor_zeroblocks.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor_zeroblocks.cpp @@ -33,7 +33,7 @@ class TDiskAgentZeroActor final const TNonreplicatedPartitionConfigPtr PartConfig; const TActorId Part; const ui32 BlockSize; - const bool AssignIdToWriteAndZeroRequestsEnabled; + const bool AssignVolumeRequestId; TInstant StartTime; ui32 RequestsCompleted = 0; @@ -48,7 +48,7 @@ class TDiskAgentZeroActor final TNonreplicatedPartitionConfigPtr partConfig, const TActorId& part, ui32 blockSize, - bool assignIdToWriteAndZeroRequestsEnabled); + bool assignVolumeRequestId); void Bootstrap(const TActorContext& ctx); @@ -84,15 +84,14 @@ TDiskAgentZeroActor::TDiskAgentZeroActor( TNonreplicatedPartitionConfigPtr partConfig, const TActorId& part, ui32 blockSize, - bool assignIdToWriteAndZeroRequestsEnabled) + bool assignVolumeRequestId) : RequestInfo(std::move(requestInfo)) , Request(std::move(request)) , DeviceRequests(std::move(deviceRequests)) , PartConfig(std::move(partConfig)) , Part(part) , BlockSize(blockSize) - , AssignIdToWriteAndZeroRequestsEnabled( - assignIdToWriteAndZeroRequestsEnabled) + , AssignVolumeRequestId(assignVolumeRequestId) {} void TDiskAgentZeroActor::Bootstrap(const TActorContext& ctx) @@ -123,7 +122,7 @@ void TDiskAgentZeroActor::ZeroBlocks(const TActorContext& ctx) request->Record.SetStartIndex(deviceRequest.DeviceBlockRange.Start); request->Record.SetBlockSize(BlockSize); request->Record.SetBlocksCount(deviceRequest.DeviceBlockRange.Size()); - if (AssignIdToWriteAndZeroRequestsEnabled) { + if (AssignVolumeRequestId) { request->Record.SetVolumeRequestId(RequestInfo->Cookie); request->Record.SetMultideviceRequest(DeviceRequests.size() > 1); } @@ -301,6 +300,10 @@ void TNonreplicatedPartitionActor::HandleZeroBlocks( return; } + const bool assignVolumeRequestId = + Config->GetAssignIdToWriteAndZeroRequestsEnabled() && + !msg->Record.GetHeaders().GetIsBackgroundRequest(); + auto actorId = NCloud::Register( ctx, requestInfo, @@ -309,7 +312,7 @@ void TNonreplicatedPartitionActor::HandleZeroBlocks( PartConfig, SelfId(), PartConfig->GetBlockSize(), - Config->GetAssignIdToWriteAndZeroRequestsEnabled()); + assignVolumeRequestId); RequestsInProgress.AddWriteRequest(actorId, std::move(request)); } diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.cpp index 7b71b9c9ebc..4f727ec37d8 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.cpp @@ -40,7 +40,10 @@ TNonreplicatedPartitionMigrationActor::TNonreplicatedPartitionMigrationActor( , SrcConfig(std::move(srcConfig)) , Migrations(std::move(migrations)) , RdmaClient(std::move(rdmaClient)) - , TimeoutCalculator(Config, SrcConfig) + , TimeoutCalculator( + Config->GetMaxMigrationBandwidth(), + Config->GetExpectedDiskAgentSize(), + SrcConfig) {} void TNonreplicatedPartitionMigrationActor::OnBootstrap( diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp index 02db4097990..e5b3311e0cf 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp @@ -239,6 +239,10 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocks( TVector requests; + const bool assignVolumeRequestId = + AssignIdToWriteAndZeroRequestsEnabled && + !msg->Record.GetHeaders().GetIsBackgroundRequest(); + for (auto& r: deviceRequests) { auto ep = AgentId2Endpoint[r.Device.GetAgentId()]; Y_ABORT_UNLESS(ep); @@ -248,7 +252,7 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocks( deviceRequest.SetDeviceUUID(r.Device.GetDeviceUUID()); deviceRequest.SetStartIndex(r.DeviceBlockRange.Start); deviceRequest.SetBlockSize(PartConfig->GetBlockSize()); - if (AssignIdToWriteAndZeroRequestsEnabled) { + if (assignVolumeRequestId) { deviceRequest.SetVolumeRequestId(requestInfo->Cookie); deviceRequest.SetMultideviceRequest(deviceRequests.size() > 1); } @@ -379,6 +383,10 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocksLocal( TVector requests; + const bool assignVolumeRequestId = + AssignIdToWriteAndZeroRequestsEnabled && + !msg->Record.GetHeaders().GetIsBackgroundRequest(); + ui64 blocks = 0; for (auto& r: deviceRequests) { auto ep = AgentId2Endpoint[r.Device.GetAgentId()]; @@ -389,7 +397,7 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocksLocal( deviceRequest.SetDeviceUUID(r.Device.GetDeviceUUID()); deviceRequest.SetStartIndex(r.DeviceBlockRange.Start); deviceRequest.SetBlockSize(PartConfig->GetBlockSize()); - if (AssignIdToWriteAndZeroRequestsEnabled) { + if (assignVolumeRequestId) { deviceRequest.SetVolumeRequestId(requestInfo->Cookie); deviceRequest.SetMultideviceRequest(deviceRequests.size() > 1); } diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp index e78adeb1bb5..e80c8a265b6 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp @@ -187,6 +187,10 @@ void TNonreplicatedPartitionRdmaActor::HandleZeroBlocks( TVector requests; + const bool assignVolumeRequestId = + AssignIdToWriteAndZeroRequestsEnabled && + !msg->Record.GetHeaders().GetIsBackgroundRequest(); + for (auto& r: deviceRequests) { auto ep = AgentId2Endpoint[r.Device.GetAgentId()]; Y_ABORT_UNLESS(ep); @@ -197,7 +201,7 @@ void TNonreplicatedPartitionRdmaActor::HandleZeroBlocks( deviceRequest.SetStartIndex(r.DeviceBlockRange.Start); deviceRequest.SetBlocksCount(r.DeviceBlockRange.Size()); deviceRequest.SetBlockSize(PartConfig->GetBlockSize()); - if (AssignIdToWriteAndZeroRequestsEnabled) { + if (assignVolumeRequestId) { deviceRequest.SetVolumeRequestId(requestInfo->Cookie); deviceRequest.SetMultideviceRequest(deviceRequests.size() > 1); } diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_ut.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_ut.cpp index b6172c77186..944481fd1c6 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_ut.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_ut.cpp @@ -715,7 +715,7 @@ Y_UNIT_TEST_SUITE(TNonreplicatedPartitionRdmaTest) const auto blockRange = TBlockRange64::WithLength(1024, 3072); - { + { // non-background WriteBlocksLocal should pass volume request id. TString data(DefaultBlockSize, 'A'); client.SendRequest( client.GetActorId(), @@ -730,7 +730,22 @@ Y_UNIT_TEST_SUITE(TNonreplicatedPartitionRdmaTest) UNIT_ASSERT_VALUES_EQUAL(10, writeRequestId); } - { + { // background WriteBlocksLocal should NOT pass volume request id. + TString data(DefaultBlockSize, 'A'); + auto request = + client.CreateWriteBlocksLocalRequest(blockRange, data); + request->Record.MutableHeaders()->SetIsBackgroundRequest(true); + client.SendRequest(client.GetActorId(), std::move(request), 10); + runtime.DispatchEvents({}, TDuration::Seconds(1)); + auto response = + client.RecvResponse(); + UNIT_ASSERT_C( + SUCCEEDED(response->GetStatus()), + response->GetErrorReason()); + UNIT_ASSERT_VALUES_EQUAL(0, writeRequestId); + } + + { // non-background WriteBlocks should pass volume request id. client.SendRequest( client.GetActorId(), client.CreateWriteBlocksRequest(blockRange, 'A'), @@ -744,7 +759,20 @@ Y_UNIT_TEST_SUITE(TNonreplicatedPartitionRdmaTest) UNIT_ASSERT_VALUES_EQUAL(20, writeRequestId); } - { + { // background WriteBlocks should NOT pass volume request id. + auto request = client.CreateWriteBlocksRequest(blockRange, 'A'); + request->Record.MutableHeaders()->SetIsBackgroundRequest(true); + client.SendRequest(client.GetActorId(), std::move(request), 20); + runtime.DispatchEvents({}, TDuration::Seconds(1)); + auto response = + client.RecvResponse(); + UNIT_ASSERT_C( + SUCCEEDED(response->GetStatus()), + response->GetErrorReason()); + UNIT_ASSERT_VALUES_EQUAL(0, writeRequestId); + } + + { // non-background ZeroBlocks should pass volume request id. client.SendRequest( client.GetActorId(), client.CreateZeroBlocksRequest(blockRange), @@ -757,6 +785,19 @@ Y_UNIT_TEST_SUITE(TNonreplicatedPartitionRdmaTest) response->GetErrorReason()); UNIT_ASSERT_VALUES_EQUAL(30, zeroRequestId); } + + { // background ZeroBlocks should NOT pass volume request id. + auto request = client.CreateZeroBlocksRequest(blockRange); + request->Record.MutableHeaders()->SetIsBackgroundRequest(true); + client.SendRequest(client.GetActorId(), std::move(request), 30); + runtime.DispatchEvents({}, TDuration::Seconds(1)); + auto response = + client.RecvResponse(); + UNIT_ASSERT_C( + SUCCEEDED(response->GetStatus()), + response->GetErrorReason()); + UNIT_ASSERT_VALUES_EQUAL(0, zeroRequestId); + } } Y_UNIT_TEST(ShouldSupportReadOnlyMode) diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_ut.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_ut.cpp index f32ac91d805..7478bd2bf71 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_ut.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_ut.cpp @@ -143,6 +143,7 @@ struct TTestEnv } storageConfig.SetExpectedClientBackoffIncrement(500); storageConfig.SetNonReplicatedAgentMaxTimeout(300'000); + storageConfig.SetAssignIdToWriteAndZeroRequestsEnabled(true); auto config = std::make_shared( std::move(storageConfig), @@ -1503,6 +1504,118 @@ Y_UNIT_TEST_SUITE(TNonreplicatedPartitionTest) UNIT_ASSERT_VALUES_EQUAL(2048 * 4096, counters.BytesWritten.Value); UNIT_ASSERT_VALUES_EQUAL(512 * 4096, counters.BytesRead.Value); } + + Y_UNIT_TEST(ShouldSetVolumeRequestIdForNonBackgroundRequests) + { + TTestBasicRuntime runtime; + + TTestEnv env(runtime); + + TPartitionClient client(runtime, env.ActorId); + + TActorId reacquireDiskRecipient; + + ui64 interceptedVolumeRequestId = 0; + auto takeVolumeRequestId = [&](TAutoPtr& event) + { + switch (event->GetTypeRewrite()) { + case TEvDiskAgent::EvWriteDeviceBlocksRequest: { + using TEvent = TEvDiskAgent::TEvWriteDeviceBlocksRequest; + auto* msg = event->template Get(); + interceptedVolumeRequestId = + msg->Record.GetVolumeRequestId(); + } break; + case TEvDiskAgent::EvZeroDeviceBlocksRequest: { + using TEvent = TEvDiskAgent::TEvZeroDeviceBlocksRequest; + auto* msg = event->template Get(); + interceptedVolumeRequestId = + msg->Record.GetVolumeRequestId(); + } break; + } + + return TTestActorRuntime::DefaultObserverFunc(event); + }; + runtime.SetObserverFunc(takeVolumeRequestId); + + { // Check WriteBlocks + auto doWriteBlocks = [&](bool isBackground, ui64 volumeRequestId) + { + interceptedVolumeRequestId = 0; + auto request = client.CreateWriteBlocksRequest( + TBlockRange64::WithLength(1024, 3072), + 'A'); + request->Record.MutableHeaders()->SetIsBackgroundRequest( + isBackground); + client.SendRequest( + client.GetActorId(), + std::move(request), + volumeRequestId); + runtime.AdvanceCurrentTime(TDuration::Seconds(1)); + runtime.DispatchEvents(); + auto response = client.RecvWriteBlocksResponse(); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + }; + + doWriteBlocks(false, 101); + UNIT_ASSERT_VALUES_EQUAL(101, interceptedVolumeRequestId); + + doWriteBlocks(true, 102); + UNIT_ASSERT_VALUES_EQUAL(0, interceptedVolumeRequestId); + } + + { // Check WriteBlocksLocal + auto doWriteBlocksLocal = + [&](bool isBackground, ui64 volumeRequestId) + { + const TString data(DefaultBlockSize, 'B'); + interceptedVolumeRequestId = 0; + auto request = client.CreateWriteBlocksLocalRequest( + TBlockRange64::WithLength(1024, 1024), + data); + request->Record.MutableHeaders()->SetIsBackgroundRequest( + isBackground); + client.SendRequest( + client.GetActorId(), + std::move(request), + volumeRequestId); + runtime.AdvanceCurrentTime(TDuration::Seconds(1)); + runtime.DispatchEvents(); + auto response = client.RecvWriteBlocksLocalResponse(); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + }; + + doWriteBlocksLocal(false, 101); + UNIT_ASSERT_VALUES_EQUAL(101, interceptedVolumeRequestId); + + doWriteBlocksLocal(true, 102); + UNIT_ASSERT_VALUES_EQUAL(0, interceptedVolumeRequestId); + } + + { // Check ZeroBlocks + auto doZeroBlocks = [&](bool isBackground, ui64 volumeRequestId) + { + interceptedVolumeRequestId = 0; + auto request = client.CreateZeroBlocksRequest( + TBlockRange64::WithLength(1024, 3072)); + request->Record.MutableHeaders()->SetIsBackgroundRequest( + isBackground); + client.SendRequest( + client.GetActorId(), + std::move(request), + volumeRequestId); + runtime.AdvanceCurrentTime(TDuration::Seconds(1)); + runtime.DispatchEvents(); + auto response = client.RecvZeroBlocksResponse(); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + }; + + doZeroBlocks(false, 101); + UNIT_ASSERT_VALUES_EQUAL(101, interceptedVolumeRequestId); + + doZeroBlocks(true, 102); + UNIT_ASSERT_VALUES_EQUAL(0, interceptedVolumeRequestId); + } + } } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/resync_range.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/resync_range.cpp index 8e47ee8cd6b..11afffd793b 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/resync_range.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/resync_range.cpp @@ -21,7 +21,7 @@ TResyncRangeActor::TResyncRangeActor( TRequestInfoPtr requestInfo, ui32 blockSize, TBlockRange64 range, - TVector replicas, + TVector replicas, TString writerClientId, IBlockDigestGeneratorPtr blockDigestGenerator) : RequestInfo(std::move(requestInfo)) @@ -44,50 +44,23 @@ void TResyncRangeActor::Bootstrap(const TActorContext& ctx) "ResyncRange", RequestInfo->CallContext->RequestId); - ChecksumBlocks(ctx); + ChecksumRangeActorCompanion.CalculateChecksums(ctx); } -void TResyncRangeActor::ChecksumBlocks(const TActorContext& ctx) { - for (size_t i = 0; i < Replicas.size(); ++i) { - ChecksumReplicaBlocks(ctx, i); - } - - ChecksumStartTs = ctx.Now(); -} - -void TResyncRangeActor::ChecksumReplicaBlocks(const TActorContext& ctx, int idx) +void TResyncRangeActor::CompareChecksums(const TActorContext& ctx) { - auto request = std::make_unique(); - request->Record.SetStartIndex(Range.Start); - request->Record.SetBlocksCount(Range.Size()); - - auto* headers = request->Record.MutableHeaders(); - headers->SetIsBackgroundRequest(true); - headers->SetClientId(TString(BackgroundOpsClientId)); - - auto event = std::make_unique( - Replicas[idx].ActorId, - ctx.SelfID, - request.release(), - IEventHandle::FlagForwardOnNondelivery, - idx, // cookie - &ctx.SelfID // forwardOnNondelivery - ); - - ctx.Send(event.release()); -} - -void TResyncRangeActor::CompareChecksums(const TActorContext& ctx) { + const auto& checksums = ChecksumRangeActorCompanion.GetChecksums(); THashMap checksumCount; ui32 majorCount = 0; ui64 majorChecksum = 0; int majorIdx = 0; - for (const auto& [idx, checksum]: Checksums) { + for (size_t i = 0; i < checksums.size(); i++) { + ui64 checksum = checksums[i]; if (++checksumCount[checksum] > majorCount) { majorCount = checksumCount[checksum]; majorChecksum = checksum; - majorIdx = idx; + majorIdx = i; } } @@ -106,17 +79,18 @@ void TResyncRangeActor::CompareChecksums(const TActorContext& ctx) { majorCount, Replicas.size()); - for (const auto& [idx, checksum]: Checksums) { + for (size_t i = 0; i < checksums.size(); i++) { + ui64 checksum = checksums[i]; if (checksum != majorChecksum) { LOG_WARN(ctx, TBlockStoreComponents::PARTITION, "[%s] Replica %lu block range %s checksum %lu differs from majority checksum %lu", Replicas[0].Name.c_str(), - idx, + Replicas[i].ReplicaIndex, DescribeRange(Range).c_str(), checksum, majorChecksum); - ActorsToResync.push_back(idx); + ActorsToResync.push_back(i); } } @@ -203,7 +177,7 @@ void TResyncRangeActor::WriteReplicaBlocks(const TActorContext& ctx, int idx) LOG_WARN(ctx, TBlockStoreComponents::PARTITION, "[%s] Replica %lu Overwrite block range %s during resync", Replicas[0].Name.c_str(), - idx, + Replicas[idx].ReplicaIndex, DescribeRange(Range).c_str()); ctx.Send(event.release()); @@ -214,8 +188,8 @@ void TResyncRangeActor::Done(const TActorContext& ctx) auto response = std::make_unique( std::move(Error), Range, - ChecksumStartTs, - ChecksumDuration, + ChecksumRangeActorCompanion.GetChecksumStartTs(), + ChecksumRangeActorCompanion.GetChecksumDuration(), ReadStartTs, ReadDuration, WriteStartTs, @@ -240,11 +214,10 @@ void TResyncRangeActor::HandleChecksumUndelivery( const TEvNonreplPartitionPrivate::TEvChecksumBlocksRequest::TPtr& ev, const TActorContext& ctx) { - ChecksumDuration = ctx.Now() - ChecksumStartTs; - Y_UNUSED(ev); - Error = MakeError(E_REJECTED, "ChecksumBlocks request undelivered"); + ChecksumRangeActorCompanion.HandleChecksumUndelivery(ctx); + Error = ChecksumRangeActorCompanion.GetError(); Done(ctx); } @@ -253,26 +226,20 @@ void TResyncRangeActor::HandleChecksumResponse( const TEvNonreplPartitionPrivate::TEvChecksumBlocksResponse::TPtr& ev, const TActorContext& ctx) { - ChecksumDuration = ctx.Now() - ChecksumStartTs; + ChecksumRangeActorCompanion.HandleChecksumResponse(ev, ctx); - auto* msg = ev->Get(); + if (!ChecksumRangeActorCompanion.IsFinished()) { + return; + } - Error = msg->Record.GetError(); + Error = ChecksumRangeActorCompanion.GetError(); if (HasError(Error)) { - LOG_WARN(ctx, TBlockStoreComponents::PARTITION, - "[%s] Checksum error %s", - Replicas[0].Name.c_str(), - FormatError(Error).c_str()); - Done(ctx); return; } - Checksums.insert({ev->Cookie, msg->Record.GetChecksum()}); - if (Checksums.size() == Replicas.size()) { - CompareChecksums(ctx); - } + CompareChecksums(ctx); } void TResyncRangeActor::HandleReadUndelivery( @@ -356,8 +323,12 @@ STFUNC(TResyncRangeActor::StateWork) switch (ev->GetTypeRewrite()) { HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); - HFunc(TEvNonreplPartitionPrivate::TEvChecksumBlocksRequest, HandleChecksumUndelivery); - HFunc(TEvNonreplPartitionPrivate::TEvChecksumBlocksResponse, HandleChecksumResponse); + HFunc( + TEvNonreplPartitionPrivate::TEvChecksumBlocksRequest, + HandleChecksumUndelivery); + HFunc( + TEvNonreplPartitionPrivate::TEvChecksumBlocksResponse, + HandleChecksumResponse); HFunc(TEvService::TEvReadBlocksLocalRequest, HandleReadUndelivery); HFunc(TEvService::TEvReadBlocksLocalResponse, HandleReadResponse); HFunc(TEvService::TEvWriteBlocksLocalRequest, HandleWriteUndelivery); diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/resync_range.h b/cloud/blockstore/libs/storage/partition_nonrepl/resync_range.h index 32bbabb7ed6..a7ce5a05b02 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/resync_range.h +++ b/cloud/blockstore/libs/storage/partition_nonrepl/resync_range.h @@ -1,5 +1,6 @@ #pragma once +#include "checksum_range.h" #include "part_nonrepl_events_private.h" #include @@ -17,14 +18,6 @@ namespace NCloud::NBlockStore::NStorage { //////////////////////////////////////////////////////////////////////////////// -struct TResyncReplica -{ - TString Name; - NActors::TActorId ActorId; -}; - -//////////////////////////////////////////////////////////////////////////////// - class TResyncRangeActor final : public NActors::TActorBootstrapped { @@ -32,39 +25,38 @@ class TResyncRangeActor final const TRequestInfoPtr RequestInfo; const ui32 BlockSize; const TBlockRange64 Range; - const TVector Replicas; + const TVector Replicas; const TString WriterClientId; const IBlockDigestGeneratorPtr BlockDigestGenerator; - THashMap Checksums; TVector ActorsToResync; ui32 ResyncedCount = 0; TGuardedBuffer Buffer; TGuardedSgList SgList; NProto::TError Error; - TInstant ChecksumStartTs; - TDuration ChecksumDuration; TInstant ReadStartTs; TDuration ReadDuration; TInstant WriteStartTs; TDuration WriteDuration; TVector AffectedBlockInfos; + TChecksumRangeActorCompanion ChecksumRangeActorCompanion{ + Range, + Replicas}; + public: TResyncRangeActor( TRequestInfoPtr requestInfo, ui32 blockSize, TBlockRange64 range, - TVector replicas, + TVector replicas, TString writerClientId, IBlockDigestGeneratorPtr blockDigestGenerator); void Bootstrap(const NActors::TActorContext& ctx); private: - void ChecksumBlocks(const NActors::TActorContext& ctx); - void ChecksumReplicaBlocks(const NActors::TActorContext& ctx, int idx); void CompareChecksums(const NActors::TActorContext& ctx); void ReadBlocks(const NActors::TActorContext& ctx, int idx); void WriteBlocks(const NActors::TActorContext& ctx); diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/resync_range_ut.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/resync_range_ut.cpp index 882f46fa7d6..59a6353faf8 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/resync_range_ut.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/resync_range_ut.cpp @@ -36,7 +36,7 @@ namespace { struct TTestEnv { TTestActorRuntime& Runtime; - TVector Replicas; + TVector Replicas; TActorId VolumeActorId; TStorageStatsServiceStatePtr StorageStatsServiceState; TDiskAgentStatePtr DiskAgentState; @@ -174,7 +174,7 @@ struct TTestEnv actorId, TActorSetupCmd(part.release(), TMailboxType::Simple, 0) ); - Replicas.push_back({name, actorId}); + Replicas.push_back({name, static_cast(i), actorId}); } auto dummy = std::make_unique(); @@ -237,7 +237,7 @@ struct TTestEnv MakeIntrusive() ); - TVector replicas; + TVector replicas; for (int idx: idxs) { replicas.push_back(Replicas[idx]); } diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/ya.make b/cloud/blockstore/libs/storage/partition_nonrepl/ya.make index 1f5198b6cb0..7a2fbfe8a2f 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/ya.make +++ b/cloud/blockstore/libs/storage/partition_nonrepl/ya.make @@ -1,6 +1,7 @@ LIBRARY() SRCS( + checksum_range.cpp config.cpp copy_range.cpp migration_timeout_calculator.cpp @@ -71,8 +72,6 @@ PEERDIR( library/cpp/containers/ring_buffer contrib/ydb/core/base - contrib/ydb/core/testlib - contrib/ydb/core/testlib/basics contrib/ydb/library/actors/core ) diff --git a/cloud/blockstore/libs/storage/protos/part.proto b/cloud/blockstore/libs/storage/protos/part.proto index 09fb1263638..4834bc0fd40 100644 --- a/cloud/blockstore/libs/storage/protos/part.proto +++ b/cloud/blockstore/libs/storage/protos/part.proto @@ -73,6 +73,10 @@ message TPartitionConfig // Optional tablet id of base volume. uint64 BaseDiskTabletId = 17; + + // Indicates that partition does not belong to the user directly, but is + // used for system needs. + bool IsSystem = 18; } //////////////////////////////////////////////////////////////////////////////// @@ -120,6 +124,9 @@ message TPartitionStats uint64 CompactionByBlobCountPerDisk = 19; uint64 CompactionByGarbageBlocksPerRange = 20; uint64 CompactionByGarbageBlocksPerDisk = 21; + + uint32 UnconfirmedBlobCount = 22; + uint32 ConfirmedBlobCount = 23; } //////////////////////////////////////////////////////////////////////////////// @@ -170,7 +177,8 @@ message TBlobMeta // corresponding flag in TStorageServiceConfig is set. // May contain zeroes. Zeroes are treated as "no checksum" and thus // verification doesn't work for actual zero checksums. - repeated uint32 BlockChecksums = 3; + reserved 3; + repeated uint32 BlockChecksums = 4; } //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/blockstore/libs/storage/protos/volume.proto b/cloud/blockstore/libs/storage/protos/volume.proto index e16afecad1a..d91db31e439 100644 --- a/cloud/blockstore/libs/storage/protos/volume.proto +++ b/cloud/blockstore/libs/storage/protos/volume.proto @@ -366,6 +366,8 @@ message TCachedPartStats uint64 GarbageQueueBytes = 10; uint64 ChannelHistorySize = 11; uint64 LogicalUsedBytesCount = 12; + uint32 UnconfirmedBlobCount = 13; + uint32 ConfirmedBlobCount = 14; } //////////////////////////////////////////////////////////////////////////////// @@ -603,7 +605,7 @@ message TChangeStorageConfigRequest // Label of volume to wait readiness of. string DiskId = 2; - + // Storage config NProto.TStorageServiceConfig StorageConfig = 3; diff --git a/cloud/blockstore/libs/storage/service/service_actor_actions_get_dependent_disks.cpp b/cloud/blockstore/libs/storage/service/service_actor_actions_get_dependent_disks.cpp index 713bd78aba2..4bb5c231f0b 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_actions_get_dependent_disks.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_actions_get_dependent_disks.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -148,8 +149,12 @@ void TGetDependentDisksActor::HandleGetDependentDisksResponse( { auto* msg = ev->Get(); - const auto& error = msg->GetError(); - if (HasError(error) && error.GetCode() != E_NOT_FOUND) { + auto error = msg->GetError(); + if (error.GetCode() == E_NOT_FOUND) { + SetErrorProtoFlag(error, NCloud::NProto::EF_SILENT); + } + + if (HasError(error)) { HandleError(ctx, error); return; } diff --git a/cloud/blockstore/libs/storage/service/service_actor_actions_update_disk_registry_params.cpp b/cloud/blockstore/libs/storage/service/service_actor_actions_update_disk_registry_params.cpp index e2ce6a0cedb..9076546f06a 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_actions_update_disk_registry_params.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_actions_update_disk_registry_params.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -128,9 +129,12 @@ void TUpdateDiskRegistryAgentListParamsActor::HandleUpdateDiskRegistryAgentListP const TActorContext& ctx) { auto* msg = ev->Get(); - const auto& error = msg->GetError(); + auto error = msg->GetError(); + if (error.GetCode() == E_NOT_FOUND) { + SetErrorProtoFlag(error, NCloud::NProto::EF_SILENT); + } - if (FAILED(error.GetCode())) { + if (HasError(error)) { HandleError(ctx, error); return; } diff --git a/cloud/blockstore/libs/storage/service/service_actor_actions_writable_state.cpp b/cloud/blockstore/libs/storage/service/service_actor_actions_writable_state.cpp index 7b7d61797d5..cdc5a73b824 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_actions_writable_state.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_actions_writable_state.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -63,15 +64,18 @@ TWritableStateActor::TWritableStateActor( void TWritableStateActor::Bootstrap(const TActorContext& ctx) { - auto request = - std::make_unique(); + NPrivateProto::TDiskRegistrySetWritableStateRequest proto; - if (!google::protobuf::util::JsonStringToMessage(Input, &request->Record).ok()) { + if (!google::protobuf::util::JsonStringToMessage(Input, &proto).ok()) { Error = MakeError(E_ARGUMENT, "Failed to parse input"); ReplyAndDie(ctx); return; } + auto request = + std::make_unique(); + request->Record.SetState(proto.GetState()); + Become(&TThis::StateWork); NCloud::Send(ctx, MakeDiskRegistryProxyServiceId(), std::move(request)); diff --git a/cloud/blockstore/libs/storage/service/service_actor_monitoring.cpp b/cloud/blockstore/libs/storage/service/service_actor_monitoring.cpp index bf87183f04f..cb0e1fede2a 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_monitoring.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_monitoring.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -154,6 +155,11 @@ void TServiceActor::RenderHtmlInfo(IOutputStream& out) const RenderDownDisks(out); RenderVolumeList(out); + if (RdmaClient) { + TAG(TH3) { out << "RdmaClient"; } + RdmaClient->DumpHtml(out); + } + TAG(TH3) { out << "Config"; } Config->DumpHtml(out); diff --git a/cloud/blockstore/libs/storage/service/service_ut_actions.cpp b/cloud/blockstore/libs/storage/service/service_ut_actions.cpp index 8619ef697d4..f220df374d7 100644 --- a/cloud/blockstore/libs/storage/service/service_ut_actions.cpp +++ b/cloud/blockstore/libs/storage/service/service_ut_actions.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -322,7 +323,7 @@ Y_UNIT_TEST_SUITE(TServiceActionsTest) UNIT_ASSERT(!drState->WritableState); - NProto::TSetWritableStateRequest request; + NPrivateProto::TDiskRegistrySetWritableStateRequest request; request.SetState(true); TString buf; @@ -971,11 +972,11 @@ Y_UNIT_TEST_SUITE(TServiceActionsTest) UNIT_ASSERT(describeReceived); } - Y_UNIT_TEST(ShouldForwardUpdateParamsRequestsToDiskRegistry) + void UpdateDiskRegistryAgentListParamsTest( + ui32 returnedCode, + ui32 expectedErrorFlags) { - auto drState = MakeIntrusive(); - TTestEnv env(1, 1, 4, 1, {drState}); - auto& runtime = env.GetRuntime(); + TTestEnv env(1, 1, 4, 1, MakeIntrusive()); NProto::TStorageServiceConfig config; ui32 nodeIdx = SetupTestEnv(env, config); @@ -988,13 +989,19 @@ Y_UNIT_TEST_SUITE(TServiceActionsTest) requestParams.SetNewNonReplicatedAgentMaxTimeoutMs(456); bool updateParamsReceived = false; - runtime.SetObserverFunc([&] (TAutoPtr& event) { + env.GetRuntime().SetObserverFunc([&] (TAutoPtr& event) { switch (event->GetTypeRewrite()) { case TEvDiskRegistry::EvUpdateDiskRegistryAgentListParamsRequest: { updateParamsReceived = true; auto* msg = event->Get(); const auto& params = msg->Record.GetParams(); UNIT_ASSERT(google::protobuf::util::MessageDifferencer::Equals(requestParams, params)); + break; + } + case TEvDiskRegistry::EvUpdateDiskRegistryAgentListParamsResponse: { + auto* msg = event->Get(); + msg->Record.MutableError()->SetCode(returnedCode); + break; } } return TTestActorRuntime::DefaultObserverFunc(event); @@ -1003,10 +1010,20 @@ Y_UNIT_TEST_SUITE(TServiceActionsTest) TString buf; google::protobuf::util::MessageToJsonString(requestParams, &buf); - const auto response = service.ExecuteAction("updatediskregistryagentlistparams", buf); + service.SendExecuteActionRequest("updatediskregistryagentlistparams", buf); + const auto response = service.RecvExecuteActionResponse(); - UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); UNIT_ASSERT(updateParamsReceived); + UNIT_ASSERT_VALUES_EQUAL(returnedCode, response->GetStatus()); + UNIT_ASSERT_VALUES_EQUAL(expectedErrorFlags, response->GetError().GetFlags()); + } + + Y_UNIT_TEST(ShouldForwardUpdateParamsRequestsToDiskRegistry) { + UpdateDiskRegistryAgentListParamsTest(S_OK, NProto::EF_NONE); + } + + Y_UNIT_TEST(ShouldForwardUpdateParamsRequestsToDiskRegistryAndSilentNotFound) { + UpdateDiskRegistryAgentListParamsTest(E_NOT_FOUND, NProto::EF_SILENT); } Y_UNIT_TEST(ShouldFinishFillDisk) @@ -1165,7 +1182,10 @@ Y_UNIT_TEST_SUITE(TServiceActionsTest) service.DestroyVolume(); } - void ShouldGetDependentDevicesTest(TVector returnedDiskIds) + void ShouldGetDependentDevicesTest( + TVector returnedDiskIds, + ui32 returnedCode, + ui32 expectedErrorFlags) { TTestEnv env(1, 1, 4, 1, MakeIntrusive()); @@ -1189,10 +1209,10 @@ Y_UNIT_TEST_SUITE(TServiceActionsTest) } case TEvDiskRegistry::EvGetDependentDisksResponse: { auto* msg = event->Get(); - msg->Record.ClearDependentDiskIds(); - for (const auto& id : returnedDiskIds) { - msg->Record.AddDependentDiskIds(id); - } + msg->Record.MutableDependentDiskIds()->Add( + returnedDiskIds.begin(), + returnedDiskIds.end()); + msg->Record.MutableError()->SetCode(returnedCode); break; } } @@ -1200,31 +1220,41 @@ Y_UNIT_TEST_SUITE(TServiceActionsTest) } ); - const auto response = service.ExecuteAction("GetDependentDisks", R"({"Host":"localhost"})"); - UNIT_ASSERT(requestReceived); + NProto::TGetDependentDisksRequest request; + request.SetHost("localhost"); + TString jsonInput; + google::protobuf::util::MessageToJsonString(request, &jsonInput); - NJson::TJsonValue responseJson; - if (!NJson::ReadJsonTree(response->Record.GetOutput(), &responseJson, false)) { - UNIT_ASSERT(false); - } + service.SendExecuteActionRequest("GetDependentDisks", jsonInput); + const auto response = service.RecvExecuteActionResponse(); + + UNIT_ASSERT(requestReceived); + UNIT_ASSERT_VALUES_EQUAL(returnedCode, response->GetStatus()); + UNIT_ASSERT_VALUES_EQUAL(expectedErrorFlags, response->GetError().GetFlags()); - const auto diskIdsJson = responseJson["DependentDiskIds"].GetArray(); - TVector responseDiskIds; - for (const auto& diskIdJson: diskIdsJson) { - responseDiskIds.emplace_back(diskIdJson.GetString()); + if (returnedCode == S_OK) { + NProto::TGetDependentDisksResponse output; + UNIT_ASSERT(google::protobuf::util::JsonStringToMessage( + response->Record.GetOutput(), &output).ok()); + ASSERT_VECTORS_EQUAL(returnedDiskIds, output.GetDependentDiskIds()); + } else { + UNIT_ASSERT_VALUES_EQUAL("", response->Record.GetOutput()); } - UNIT_ASSERT_VALUES_EQUAL(returnedDiskIds, responseDiskIds); } - Y_UNIT_TEST(ShouldGetDependentDevicesNoDisks) { - ShouldGetDependentDevicesTest({}); + ShouldGetDependentDevicesTest({}, S_OK, NProto::EF_NONE); } Y_UNIT_TEST(ShouldGetDependentDevicesMultipleDisks) { - ShouldGetDependentDevicesTest({"disk1", "disk2"}); + ShouldGetDependentDevicesTest({"disk1", "disk2"}, S_OK, NProto::EF_NONE); + } + + Y_UNIT_TEST(ShouldGetDependentDevicesAndSilentNotFound) + { + ShouldGetDependentDevicesTest({}, E_NOT_FOUND, NProto::EF_SILENT); } NProto::TChangeStorageConfigResponse ExecuteChangeStorageConfig( diff --git a/cloud/blockstore/libs/storage/service/service_ut_create.cpp b/cloud/blockstore/libs/storage/service/service_ut_create.cpp index 72669bf984a..6ff62794628 100644 --- a/cloud/blockstore/libs/storage/service/service_ut_create.cpp +++ b/cloud/blockstore/libs/storage/service/service_ut_create.cpp @@ -700,6 +700,64 @@ Y_UNIT_TEST_SUITE(TServiceCreateVolumeTest) } } + Y_UNIT_TEST(ShouldCreateSystemVolumeWithMultipartitionedBaseDisk) + { + TTestEnv env; + NProto::TStorageServiceConfig config; + config.SetBytesPerPartition(2_GB); + config.SetBytesPerPartitionSSD(1_GB); + config.SetMultipartitionVolumesEnabled(true); + config.SetBytesPerStripe(16_MB); + config.SetMaxPartitionsPerVolume(2); + ui32 nodeIdx = SetupTestEnv(env, config); + + auto& runtime = env.GetRuntime(); + + TServiceClient service(runtime, nodeIdx); + + { + service.CreateVolume( + "baseDisk", + 2_GB / DefaultBlockSize, + DefaultBlockSize, + "", // folderId + "", // cloudId + NCloud::NProto::STORAGE_MEDIA_SSD, + NProto::TVolumePerformanceProfile(), + TString(), // placementGroupId + 0, // placementPartitionIndex + 0, // partitionsCount + NProto::TEncryptionSpec() + ); + + auto response = service.DescribeVolume("baseDisk"); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + UNIT_ASSERT_VALUES_EQUAL(2, response->Record.GetVolume().GetPartitionsCount()); + + service.CreateVolume( + "vol0", + 2_GB / DefaultBlockSize, + DefaultBlockSize, + TString(), // folderId + TString(), // cloudId + NCloud::NProto::STORAGE_MEDIA_SSD, + NProto::TVolumePerformanceProfile(), + TString(), // placementGroupId + 0, // placementPartitionIndex + 0, // partitionsCount + NProto::TEncryptionSpec(), + true, // isSystem + "baseDisk", + "baseDiskCheckpointId", + 0 // fillGeneration + ); + + response = service.DescribeVolume("vol0"); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + UNIT_ASSERT_VALUES_EQUAL(1, response->Record.GetVolume().GetPartitionsCount()); + } + } + Y_UNIT_TEST(ShouldCreateEncryptedVolume) { TTestEnv env; diff --git a/cloud/blockstore/libs/storage/service/volume_session_actor_mount.cpp b/cloud/blockstore/libs/storage/service/volume_session_actor_mount.cpp index 92d3b47291a..145c3a39d14 100644 --- a/cloud/blockstore/libs/storage/service/volume_session_actor_mount.cpp +++ b/cloud/blockstore/libs/storage/service/volume_session_actor_mount.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -50,9 +51,7 @@ using TEvInternalMountVolumeResponsePtr = NProto::TError MakeErrorSilent(const NProto::TError& error) { auto result = error; - ui32 flags = error.GetFlags(); - SetProtoFlag(flags, NProto::EF_SILENT); - result.SetFlags(flags); + SetErrorProtoFlag(result, NProto::EF_SILENT); return result; } diff --git a/cloud/blockstore/libs/storage/ss_proxy/ss_proxy_actor_describescheme.cpp b/cloud/blockstore/libs/storage/ss_proxy/ss_proxy_actor_describescheme.cpp index d0e671cac41..c5f2145a07d 100644 --- a/cloud/blockstore/libs/storage/ss_proxy/ss_proxy_actor_describescheme.cpp +++ b/cloud/blockstore/libs/storage/ss_proxy/ss_proxy_actor_describescheme.cpp @@ -141,9 +141,7 @@ void TDescribeSchemeActor::HandleDescribeSchemeResult( // TODO: return E_NOT_FOUND instead of StatusPathDoesNotExist if (status == NKikimrScheme::StatusPathDoesNotExist) { - ui32 flags = error.GetFlags(); - SetProtoFlag(flags, NProto::EF_SILENT); - error.SetFlags(flags); + SetErrorProtoFlag(error, NCloud::NProto::EF_SILENT); } } diff --git a/cloud/blockstore/libs/storage/testlib/disk_registry_proxy_mock.h b/cloud/blockstore/libs/storage/testlib/disk_registry_proxy_mock.h index 2b59b838dfd..f531f573b25 100644 --- a/cloud/blockstore/libs/storage/testlib/disk_registry_proxy_mock.h +++ b/cloud/blockstore/libs/storage/testlib/disk_registry_proxy_mock.h @@ -10,7 +10,7 @@ #include #include - +#include #include #include @@ -139,6 +139,10 @@ class TDiskRegistryProxyMock final TEvService::TEvCmsActionRequest, HandleCmsAction); + HFunc( + TEvDiskRegistryProxy::TEvGetDrTabletInfoRequest, + HandleGetDrTabletInfo); + IgnoreFunc(NKikimr::TEvLocal::TEvTabletMetrics); default: @@ -966,6 +970,19 @@ class TDiskRegistryProxyMock final std::make_unique< TEvService::TEvCmsActionResponse>()); } + + void HandleGetDrTabletInfo( + const TEvDiskRegistryProxy::TEvGetDrTabletInfoRequest::TPtr& ev, + const NActors::TActorContext& ctx) + { + const ui64 testDiskRegistryTabletId = + NKikimr::MakeTabletID(0, NKikimr::MakeDefaultHiveID(0), 1); + NCloud::Reply( + ctx, + *ev, + std::make_unique( + testDiskRegistryTabletId)); + } }; } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/undelivered/ya.make b/cloud/blockstore/libs/storage/undelivered/ya.make index ea0be972a2d..a9e25b1bfe0 100644 --- a/cloud/blockstore/libs/storage/undelivered/ya.make +++ b/cloud/blockstore/libs/storage/undelivered/ya.make @@ -8,8 +8,6 @@ PEERDIR( cloud/blockstore/libs/kikimr cloud/blockstore/libs/storage/api contrib/ydb/library/actors/core - contrib/ydb/core/testlib - contrib/ydb/core/testlib/basics ) END() diff --git a/cloud/blockstore/libs/storage/volume/actors/read_disk_registry_based_overlay.cpp b/cloud/blockstore/libs/storage/volume/actors/read_disk_registry_based_overlay.cpp index da65ff61456..eae6c7da954 100644 --- a/cloud/blockstore/libs/storage/volume/actors/read_disk_registry_based_overlay.cpp +++ b/cloud/blockstore/libs/storage/volume/actors/read_disk_registry_based_overlay.cpp @@ -441,7 +441,7 @@ void TReadDiskRegistryBasedOverlayActor::HandlePoisonPill( { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "Tablet is dead")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/blockstore/libs/storage/volume/actors/shadow_disk_actor.cpp b/cloud/blockstore/libs/storage/volume/actors/shadow_disk_actor.cpp new file mode 100644 index 00000000000..a881af433e6 --- /dev/null +++ b/cloud/blockstore/libs/storage/volume/actors/shadow_disk_actor.cpp @@ -0,0 +1,1084 @@ +#include "shadow_disk_actor.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace NActors; + +namespace NCloud::NBlockStore::NStorage { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +template +void ForwardMessageToActor( + TEvent& ev, + const NActors::TActorContext& ctx, + TActorId destActor) +{ + NActors::TActorId nondeliveryActor = ev->GetForwardOnNondeliveryRecipient(); + auto message = std::make_unique( + destActor, + ev->Sender, + ev->ReleaseBase().Release(), + ev->Flags, + ev->Cookie, + ev->Flags & NActors::IEventHandle::FlagForwardOnNondelivery + ? &nondeliveryActor + : nullptr); + ctx.Send(std::move(message)); +} + +TString GetDeviceUUIDs(const TDevices& devices) +{ + TString result; + for (const auto& device: devices) { + if (result.empty()) { + result += device.GetDeviceUUID(); + } else { + result += ", " + device.GetDeviceUUID(); + } + } + return result; +} + +bool CheckDeviceUUIDsIdentical( + const TDevices& described, + const TDevices& acquired) +{ + if (described.size() != acquired.size()) { + return false; + } + + for (int i = 0; i < described.size(); ++i) { + if (described[i].GetDeviceUUID() != acquired[i].GetDeviceUUID()) { + return false; + } + } + return true; +} + +/////////////////////////////////////////////////////////////////////////////// + +class TAcquireShadowDiskActor + : public NActors::TActorBootstrapped +{ +private: + const TString ShadowDiskId; + const TShadowDiskActor::EAcquireReason AcquireReason; + const bool ReadOnlyMount; + const TDuration TotalTimeout; + const TActorId ParentActor; + const ui64 MountSeqNumber = 0; + const ui32 Generation = 0; + const TString ShadowDiskClientId = "shadow-disk-client-id"; + + TInstant AcquireStartedAt = {}; + // The list of devices received via the describe request. + // This is necessary to check that all disk devices have been acquired. + TDevices ShadowDiskDevices; + TDevices AcquiredShadowDiskDevices; + + // Delay provider when retrying describe and acquire requests to disk + // registry. + TBackoffDelayProvider RetryDelayProvider; + +public: + TAcquireShadowDiskActor( + const TStorageConfigPtr config, + TString shadowDiskId, + const TDevices& shadowDiskDevices, + TShadowDiskActor::EAcquireReason acquireReason, + bool readOlyMount, + bool areWritesToSourceBlocked, + ui64 mountSeqNumber, + ui32 generation, + TActorId parentActor); + + void Bootstrap(const TActorContext& ctx); + +private: + void DescribeShadowDisk(const NActors::TActorContext& ctx); + void AcquireShadowDisk(const NActors::TActorContext& ctx); + + std::unique_ptr + MakeDescribeDiskRequest() const; + std::unique_ptr + MakeAcquireDiskRequest() const; + + void HandleDiskRegistryError( + const NActors::TActorContext& ctx, + const NProto::TError& error, + std::unique_ptr retryEvent, + const TString& actionName); + + void MaybeReady(const TActorContext& ctx); + void ReplyAndDie(const TActorContext& ctx, const NProto::TError& error); + +private: + STFUNC(Work); + + void HandleDescribeDiskResponse( + const TEvDiskRegistry::TEvDescribeDiskResponse::TPtr& ev, + const NActors::TActorContext& ctx); + + void HandleAcquireDiskResponse( + const TEvDiskRegistry::TEvAcquireDiskResponse::TPtr& ev, + const NActors::TActorContext& ctx); + + void HandleDescribeDiskRequestUndelivery( + const TEvDiskRegistry::TEvDescribeDiskRequest::TPtr& ev, + const NActors::TActorContext& ctx); + + void HandleAcquireDiskRequestUndelivery( + const TEvDiskRegistry::TEvAcquireDiskRequest::TPtr& ev, + const NActors::TActorContext& ctx); + + void HandleWakeup( + const NActors::TEvents::TEvWakeup::TPtr& ev, + const NActors::TActorContext& ctx); +}; + +/////////////////////////////////////////////////////////////////////////////// + +TAcquireShadowDiskActor::TAcquireShadowDiskActor( + const TStorageConfigPtr config, + TString shadowDiskId, + const TDevices& shadowDiskDevices, + TShadowDiskActor::EAcquireReason acquireReason, + bool readOnlyMount, + bool areWritesToSourceBlocked, + ui64 mountSeqNumber, + ui32 generation, + TActorId parentActor) + : ShadowDiskId(std::move(shadowDiskId)) + , AcquireReason(acquireReason) + , ReadOnlyMount(readOnlyMount) + , TotalTimeout( + areWritesToSourceBlocked + ? config->GetMaxAcquireShadowDiskTotalTimeoutWhenBlocked() + : config->GetMaxAcquireShadowDiskTotalTimeoutWhenNonBlocked()) + , ParentActor(parentActor) + , MountSeqNumber(mountSeqNumber) + , Generation(generation) + , ShadowDiskDevices(shadowDiskDevices) + , RetryDelayProvider( + areWritesToSourceBlocked + ? config->GetMinAcquireShadowDiskRetryDelayWhenBlocked() + : config->GetMinAcquireShadowDiskRetryDelayWhenNonBlocked(), + areWritesToSourceBlocked + ? config->GetMaxAcquireShadowDiskRetryDelayWhenBlocked() + : config->GetMaxAcquireShadowDiskRetryDelayWhenNonBlocked()) +{ + if (AcquireReason != TShadowDiskActor::EAcquireReason::FirstAcquire) { + STORAGE_CHECK_PRECONDITION(!ShadowDiskDevices.empty()); + } +} + +void TAcquireShadowDiskActor::Bootstrap(const TActorContext& ctx) +{ + Become(&TThis::Work); + + DescribeShadowDisk(ctx); + AcquireShadowDisk(ctx); + + AcquireStartedAt = ctx.Now(); + ctx.Schedule(TotalTimeout, new TEvents::TEvWakeup()); +} + +void TAcquireShadowDiskActor::DescribeShadowDisk( + const NActors::TActorContext& ctx) +{ + if (!ShadowDiskDevices.empty()) { + // We will not describe devices if this is not the first acquire and we + // already know them. + return; + } + LOG_INFO_S( + ctx, + TBlockStoreComponents::VOLUME, + "Describing shadow disk " << ShadowDiskId.Quote()); + + NCloud::SendWithUndeliveryTracking( + ctx, + MakeDiskRegistryProxyServiceId(), + MakeDescribeDiskRequest()); +} + +void TAcquireShadowDiskActor::AcquireShadowDisk( + const NActors::TActorContext& ctx) +{ + if (AcquireReason != TShadowDiskActor::EAcquireReason::PeriodicalReAcquire) + { + LOG_INFO_S( + ctx, + TBlockStoreComponents::VOLUME, + "Acquiring shadow disk " << ShadowDiskId.Quote() << " with timeout " + << TotalTimeout.ToString()); + } + + NCloud::SendWithUndeliveryTracking( + ctx, + MakeDiskRegistryProxyServiceId(), + MakeAcquireDiskRequest()); +} + +auto TAcquireShadowDiskActor::MakeDescribeDiskRequest() const + -> std::unique_ptr +{ + auto request = std::make_unique(); + request->Record.SetDiskId(ShadowDiskId); + return request; +} + +auto TAcquireShadowDiskActor::MakeAcquireDiskRequest() const + -> std::unique_ptr +{ + auto request = std::make_unique(); + request->Record.SetDiskId(ShadowDiskId); + request->Record.MutableHeaders()->SetClientId(ShadowDiskClientId); + request->Record.SetAccessMode( + ReadOnlyMount ? NProto::EVolumeAccessMode::VOLUME_ACCESS_READ_ONLY + : NProto::EVolumeAccessMode::VOLUME_ACCESS_READ_WRITE); + request->Record.SetMountSeqNumber(MountSeqNumber); + request->Record.SetVolumeGeneration(Generation); + return request; +} + +void TAcquireShadowDiskActor::HandleDiskRegistryError( + const NActors::TActorContext& ctx, + const NProto::TError& error, + std::unique_ptr retryEvent, + const TString& actionName) +{ + LOG_DEBUG_S( + ctx, + TBlockStoreComponents::VOLUME, + "Can't " << actionName << " shadow disk " << ShadowDiskId.Quote() + << " Error: " << FormatError(error)); + + const TInstant timeoutElapsedAt = AcquireStartedAt + TotalTimeout; + + const bool canRetry = GetErrorKind(error) == EErrorKind::ErrorRetriable && + timeoutElapsedAt > ctx.Now(); + + if (canRetry) { + LOG_WARN_S( + ctx, + TBlockStoreComponents::VOLUME, + "Will soon retry " << actionName << " shadow disk " + << ShadowDiskId.Quote() + << " Error: " << FormatError(error) + << " with delay " << RetryDelayProvider.GetDelay().ToString()); + + TActivationContext::Schedule( + RetryDelayProvider.GetDelayAndIncrease(), + std::move(retryEvent), + nullptr); + } else { + LOG_ERROR_S( + ctx, + TBlockStoreComponents::VOLUME, + "Will not retry " << actionName << " shadow disk " + << ShadowDiskId.Quote() + << " Error: " << FormatError(error)); + ReplyAndDie(ctx, error); + } +} + +void TAcquireShadowDiskActor::ReplyAndDie( + const TActorContext& ctx, + const NProto::TError& error) +{ + auto response = + std::make_unique(error); + if (!HasError(response->GetError())) { + response->Record.MutableDevices()->Swap(&AcquiredShadowDiskDevices); + } + + NCloud::Send( + ctx, + ParentActor, + std::move(response), + static_cast(AcquireReason)); + + Die(ctx); +} + +STFUNC(TAcquireShadowDiskActor::Work) +{ + switch (ev->GetTypeRewrite()) { + HFunc( + TEvDiskRegistry::TEvDescribeDiskResponse, + HandleDescribeDiskResponse); + HFunc( + TEvDiskRegistry::TEvAcquireDiskResponse, + HandleAcquireDiskResponse); + HFunc( + TEvDiskRegistry::TEvDescribeDiskRequest, + HandleDescribeDiskRequestUndelivery); + HFunc( + TEvDiskRegistry::TEvAcquireDiskRequest, + HandleAcquireDiskRequestUndelivery); + HFunc(NActors::TEvents::TEvWakeup, HandleWakeup); + + default: + HandleUnexpectedEvent(ev, TBlockStoreComponents::SERVICE); + break; + } +} + +void TAcquireShadowDiskActor::HandleDescribeDiskResponse( + const TEvDiskRegistry::TEvDescribeDiskResponse::TPtr& ev, + const NActors::TActorContext& ctx) +{ + auto* msg = ev->Get(); + auto& record = msg->Record; + if (HasError(record.GetError())) { + HandleDiskRegistryError( + ctx, + record.GetError(), + std::make_unique( + MakeDiskRegistryProxyServiceId(), + ctx.SelfID, + MakeDescribeDiskRequest().release()), + "describe"); + return; + } + + ShadowDiskDevices.Swap(record.MutableDevices()); + MaybeReady(ctx); +} + +void TAcquireShadowDiskActor::HandleAcquireDiskResponse( + const TEvDiskRegistry::TEvAcquireDiskResponse::TPtr& ev, + const NActors::TActorContext& ctx) +{ + auto* msg = ev->Get(); + auto& record = msg->Record; + + if (HasError(record.GetError())) { + HandleDiskRegistryError( + ctx, + record.GetError(), + std::make_unique( + MakeDiskRegistryProxyServiceId(), + ctx.SelfID, + MakeAcquireDiskRequest().release()), + "acquire"); + return; + } + + AcquiredShadowDiskDevices.Swap(record.MutableDevices()); + MaybeReady(ctx); +} + +void TAcquireShadowDiskActor::HandleDescribeDiskRequestUndelivery( + const TEvDiskRegistry::TEvDescribeDiskRequest::TPtr& ev, + const NActors::TActorContext& ctx) +{ + Y_UNUSED(ev); + DescribeShadowDisk(ctx); +} + +void TAcquireShadowDiskActor::HandleAcquireDiskRequestUndelivery( + const TEvDiskRegistry::TEvAcquireDiskRequest::TPtr& ev, + const NActors::TActorContext& ctx) +{ + Y_UNUSED(ev); + AcquireShadowDisk(ctx); +} + +void TAcquireShadowDiskActor::HandleWakeup( + const NActors::TEvents::TEvWakeup::TPtr& ev, + const NActors::TActorContext& ctx) +{ + Y_UNUSED(ev); + + LOG_ERROR_S( + ctx, + TBlockStoreComponents::VOLUME, + "Acquire timeout. Shadow disk " << ShadowDiskId.Quote()); + + ReplyAndDie(ctx, MakeError(E_TIMEOUT)); +} + +void TAcquireShadowDiskActor::MaybeReady(const NActors::TActorContext& ctx) +{ + bool gotDescribeAndAcquireResponses = + !ShadowDiskDevices.empty() && !AcquiredShadowDiskDevices.empty(); + if (!gotDescribeAndAcquireResponses) { + return; + } + + // Check all shadow disk devices have been acquired. + if (!CheckDeviceUUIDsIdentical( + ShadowDiskDevices, + AcquiredShadowDiskDevices)) + { + HandleDiskRegistryError( + ctx, + MakeError( + E_REJECTED, + TStringBuilder() + << "The acquired devices are not identical to described. " + "Described [" + << GetDeviceUUIDs(ShadowDiskDevices) << "] acquired [" + << GetDeviceUUIDs(AcquiredShadowDiskDevices) << "]"), + std::make_unique( + MakeDiskRegistryProxyServiceId(), + ctx.SelfID, + MakeAcquireDiskRequest().release(), + 0, // flags + static_cast( + TShadowDiskActor::EAcquireReason::FirstAcquire)), + "acquire"); + AcquiredShadowDiskDevices.Clear(); + return; + }; + + if (AcquireReason != TShadowDiskActor::EAcquireReason::PeriodicalReAcquire) + { + LOG_INFO_S( + ctx, + TBlockStoreComponents::VOLUME, + "Acquired shadow disk " << ShadowDiskId.Quote()); + } + + ReplyAndDie(ctx, MakeError(S_OK)); +} + +} // namespace + +/////////////////////////////////////////////////////////////////////////////// + +TShadowDiskActor::TShadowDiskActor( + TStorageConfigPtr config, + NRdma::IClientPtr rdmaClient, + IProfileLogPtr profileLog, + IBlockDigestGeneratorPtr digestGenerator, + TString rwClientId, + ui64 mountSeqNumber, + ui32 generation, + TNonreplicatedPartitionConfigPtr srcConfig, + TActorId volumeActorId, + TActorId srcActorId, + const TActiveCheckpointInfo& checkpointInfo) + : TNonreplicatedPartitionMigrationCommonActor( + static_cast(this), + config, + srcConfig->GetName(), + srcConfig->GetBlockCount(), + srcConfig->GetBlockSize(), + std::move(profileLog), + std::move(digestGenerator), + checkpointInfo.ProcessedBlockCount, + std::move(rwClientId), + volumeActorId) + , Config(std::move(config)) + , RdmaClient(std::move(rdmaClient)) + , SrcConfig(std::move(srcConfig)) + , CheckpointId(checkpointInfo.CheckpointId) + , ShadowDiskId(checkpointInfo.ShadowDiskId) + , MountSeqNumber(mountSeqNumber) + , Generation(generation) + , VolumeActorId(volumeActorId) + , SrcActorId(srcActorId) + , ProcessedBlockCount(checkpointInfo.ProcessedBlockCount) +{ + STORAGE_CHECK_PRECONDITION( + checkpointInfo.Data == ECheckpointData::DataPresent); + + switch (checkpointInfo.ShadowDiskState) { + case EShadowDiskState::None: + STORAGE_CHECK_PRECONDITION( + checkpointInfo.ShadowDiskState != EShadowDiskState::None); + break; + case EShadowDiskState::New: + State = EActorState::WaitAcquireForPrepareStart; + break; + case EShadowDiskState::Preparing: + State = checkpointInfo.ProcessedBlockCount == 0 + ? EActorState::WaitAcquireForPrepareStart + : EActorState::WaitAcquireForPrepareContinue; + break; + case EShadowDiskState::Ready: + State = EActorState::WaitAcquireForRead; + break; + case EShadowDiskState::Error: + State = EActorState::Error; + break; + } +} + +TShadowDiskActor::~TShadowDiskActor() = default; + +void TShadowDiskActor::OnBootstrap(const NActors::TActorContext& ctx) +{ + PoisonPillHelper.TakeOwnership(ctx, SrcActorId); + AcquireShadowDisk(ctx, EAcquireReason::FirstAcquire); +} + +bool TShadowDiskActor::OnMessage( + const TActorContext& ctx, + TAutoPtr& ev) +{ + switch (ev->GetTypeRewrite()) { + HFunc( + TEvDiskRegistry::TEvAcquireDiskResponse, + HandleAcquireDiskResponse); + HFunc( + TEvNonreplPartitionPrivate::TEvUpdateCounters, + HandleUpdateCounters); + HFunc( + TEvVolume::TEvDiskRegistryBasedPartitionCounters, + HandleShadowDiskCounters); + HFunc(NActors::TEvents::TEvWakeup, HandleWakeup); + HFunc(TEvVolume::TEvReacquireDisk, HandleReacquireDisk); + HFunc(TEvVolume::TEvRdmaUnavailable, HandleRdmaUnavailable); + HFunc( + TEvVolumePrivate::TEvUpdateShadowDiskStateResponse, + HandleUpdateShadowDiskStateResponse); + + // Read request. + HFunc( + TEvService::TEvReadBlocksRequest, + HandleReadBlocks); + HFunc( + TEvService::TEvReadBlocksLocalRequest, + HandleReadBlocks); + + // Write/zero request. + case TEvService::TEvWriteBlocksRequest::EventType: { + return HandleWriteZeroBlocks( + *reinterpret_cast( + &ev), + ctx); + } + case TEvService::TEvWriteBlocksLocalRequest::EventType: { + return HandleWriteZeroBlocks( + *reinterpret_cast< + TEvService::TEvWriteBlocksLocalRequest::TPtr*>(&ev), + ctx); + } + case TEvService::TEvZeroBlocksRequest::EventType: { + return HandleWriteZeroBlocks( + *reinterpret_cast(&ev), + ctx); + } + + default: + // Message processing by the base class is required. + return false; + } + + // We get here if we have processed an incoming message. And its processing + // by the base class is not required. + return true; +} + +TDuration TShadowDiskActor::CalculateMigrationTimeout() +{ + STORAGE_CHECK_PRECONDITION(TimeoutCalculator); + + if (TimeoutCalculator) { + return TimeoutCalculator->CalculateTimeout(GetNextProcessingRange()); + } + return TDuration::Seconds(1); +} + +void TShadowDiskActor::OnMigrationProgress( + const NActors::TActorContext& ctx, + ui64 migrationIndex) +{ + using EReason = TEvVolumePrivate::TUpdateShadowDiskStateRequest::EReason; + + ProcessedBlockCount = migrationIndex; + + auto request = + std::make_unique( + CheckpointId, + EReason::FillProgressUpdate, + migrationIndex, + SrcConfig->GetBlockCount()); + + NCloud::Send(ctx, VolumeActorId, std::move(request)); +} + +void TShadowDiskActor::OnMigrationFinished(const NActors::TActorContext& ctx) +{ + using EReason = TEvVolumePrivate::TUpdateShadowDiskStateRequest::EReason; + + ProcessedBlockCount = SrcConfig->GetBlockCount(); + + auto request = + std::make_unique( + CheckpointId, + EReason::FillCompleted, + ProcessedBlockCount, + SrcConfig->GetBlockCount()); + + NCloud::Send(ctx, VolumeActorId, std::move(request)); +} + +void TShadowDiskActor::OnMigrationError(const NActors::TActorContext& ctx) +{ + using EReason = TEvVolumePrivate::TUpdateShadowDiskStateRequest::EReason; + + auto request = + std::make_unique( + CheckpointId, + EReason::FillError, + ProcessedBlockCount, + SrcConfig->GetBlockCount()); + + NCloud::Send(ctx, VolumeActorId, std::move(request)); +} + +void TShadowDiskActor::AcquireShadowDisk( + const NActors::TActorContext& ctx, + EAcquireReason acquireReason) +{ + switch (acquireReason) { + case EAcquireReason::FirstAcquire: { + STORAGE_CHECK_PRECONDITION(WaitingForAcquire()); + STORAGE_CHECK_PRECONDITION(DstActorId == TActorId()); + STORAGE_CHECK_PRECONDITION(AcquireActorId == TActorId()); + } break; + case EAcquireReason::PeriodicalReAcquire: { + STORAGE_CHECK_PRECONDITION(!WaitingForAcquire()); + STORAGE_CHECK_PRECONDITION(DstActorId != TActorId()); + + if (AcquireActorId != TActorId()) { + return; + } + } break; + case EAcquireReason::ForcedReAcquire: { + STORAGE_CHECK_PRECONDITION(!WaitingForAcquire()); + STORAGE_CHECK_PRECONDITION(DstActorId != TActorId()); + + if (ForcedReAcquireInProgress) { + return; + } + ForcedReAcquireInProgress = true; + } break; + } + + AcquireActorId = NCloud::Register( + ctx, + std::make_unique( + Config, + ShadowDiskId, + ShadowDiskDevices, + acquireReason, + ReadOnlyMount(), + AreWritesToSrcDiskImpossible(), + MountSeqNumber, + Generation, + SelfId())); + + PoisonPillHelper.TakeOwnership(ctx, AcquireActorId); +} + +void TShadowDiskActor::HandleAcquireDiskResponse( + const TEvDiskRegistry::TEvAcquireDiskResponse::TPtr& ev, + const NActors::TActorContext& ctx) +{ + PoisonPillHelper.ReleaseOwnership(ctx, ev->Sender); + if (AcquireActorId == ev->Sender) { + AcquireActorId = {}; + } + + auto* msg = ev->Get(); + auto& record = msg->Record; + auto acquireReason = static_cast(ev->Cookie); + + if (acquireReason == EAcquireReason::ForcedReAcquire) { + ForcedReAcquireInProgress = false; + } + + if (HasError(record.GetError())) { + if (acquireReason != EAcquireReason::PeriodicalReAcquire) { + SetErrorState(ctx); + } + return; + } + + if (acquireReason == EAcquireReason::FirstAcquire) { + CreateShadowDiskPartitionActor(ctx, record.GetDevices()); + } +} + +void TShadowDiskActor::CreateShadowDiskConfig() +{ + STORAGE_CHECK_PRECONDITION(!ShadowDiskDevices.empty()); + + TNonreplicatedPartitionConfig::TVolumeInfo volumeInfo{ + TInstant(), + GetCheckpointShadowDiskType(SrcConfig->GetVolumeInfo().MediaKind)}; + + DstConfig = std::make_shared( + ShadowDiskDevices, + ReadOnlyMount() ? NProto::VOLUME_IO_ERROR_READ_ONLY + : NProto::VOLUME_IO_OK, + ShadowDiskId, + SrcConfig->GetBlockSize(), + volumeInfo, + SelfId(), // need to handle TEvRdmaUnavailable, TEvReacquireDisk + true, // muteIOErrors + false, // markBlocksUsed + THashSet(), // freshDeviceIds + TDuration(), // maxTimedOutDeviceStateDuration + false, // maxTimedOutDeviceStateDurationOverridden + true // useSimpleMigrationBandwidthLimiter + ); + + TimeoutCalculator.emplace( + Config->GetMaxShadowDiskFillBandwidth(), + Config->GetExpectedDiskAgentSize(), + DstConfig); +} + +void TShadowDiskActor::CreateShadowDiskPartitionActor( + const NActors::TActorContext& ctx, + const TDevices& acquiredShadowDiskDevices) +{ + using EReason = TEvVolumePrivate::TUpdateShadowDiskStateRequest::EReason; + + STORAGE_CHECK_PRECONDITION(WaitingForAcquire()); + STORAGE_CHECK_PRECONDITION(DstActorId == TActorId()); + + ShadowDiskDevices = acquiredShadowDiskDevices; + + CreateShadowDiskConfig(); + + DstActorId = NCloud::Register( + ctx, + CreateNonreplicatedPartition( + Config, + DstConfig, + SelfId(), + RdmaClient)); + PoisonPillHelper.TakeOwnership(ctx, DstActorId); + + if (State == EActorState::WaitAcquireForRead) { + // Ready to serve checkpoint reads. + State = EActorState::CheckpointReady; + } else { + STORAGE_CHECK_PRECONDITION( + State == EActorState::WaitAcquireForPrepareStart || + State == EActorState::WaitAcquireForPrepareContinue); + + // Ready to fill shadow disk with data. + State = EActorState::Preparing; + + TNonreplicatedPartitionMigrationCommonActor::InitWork( + ctx, + SrcActorId, + DstActorId); + TNonreplicatedPartitionMigrationCommonActor::StartWork(ctx); + + // Persist state. + NCloud::Send( + ctx, + VolumeActorId, + std::make_unique( + CheckpointId, + EReason::FillProgressUpdate, + GetNextProcessingRange().Start, + SrcConfig->GetBlockCount())); + } + + STORAGE_CHECK_PRECONDITION(Acquired()); + SchedulePeriodicalReAcquire(ctx); +} + +void TShadowDiskActor::SchedulePeriodicalReAcquire(const TActorContext& ctx) +{ + ctx.Schedule(Config->GetClientRemountPeriod(), new TEvents::TEvWakeup()); +} + +void TShadowDiskActor::SetErrorState(const NActors::TActorContext& ctx) +{ + using EReason = TEvVolumePrivate::TUpdateShadowDiskStateRequest::EReason; + + State = EActorState::Error; + + // Persist state. + NCloud::Send( + ctx, + VolumeActorId, + std::make_unique( + CheckpointId, + EReason::FillError, + ProcessedBlockCount, + SrcConfig->GetBlockCount())); +} + +bool TShadowDiskActor::CanJustForwardWritesToSrcDisk() const +{ + return State == EActorState::CheckpointReady || + State == EActorState::Error || + State == EActorState::WaitAcquireForPrepareStart; +} + +bool TShadowDiskActor::AreWritesToSrcDiskForbidden() const +{ + return State == EActorState::WaitAcquireForPrepareContinue; +} + +bool TShadowDiskActor::AreWritesToSrcDiskImpossible() const +{ + return AreWritesToSrcDiskForbidden() || ForcedReAcquireInProgress; +} + +bool TShadowDiskActor::WaitingForAcquire() const +{ + return State == EActorState::WaitAcquireForPrepareStart || + State == EActorState::WaitAcquireForPrepareContinue || + State == EActorState::WaitAcquireForRead; +} + +bool TShadowDiskActor::Acquired() const +{ + return State == EActorState::Preparing || + State == EActorState::CheckpointReady; +} + +bool TShadowDiskActor::ReadOnlyMount() const +{ + STORAGE_CHECK_PRECONDITION(State != EActorState::Error); + + return State == EActorState::WaitAcquireForRead || + State == EActorState::CheckpointReady; +} + +template +void TShadowDiskActor::HandleReadBlocks( + const typename TMethod::TRequest::TPtr& ev, + const NActors::TActorContext& ctx) +{ + auto* msg = ev->Get(); + auto& record = msg->Record; + const auto& checkpointId = record.GetCheckpointId(); + + if (checkpointId.empty() || checkpointId != CheckpointId) { + // Forward read request to Source partition. + ForwardRequestToSrcPartition(ev, ctx); + return; + } + + if (State != EActorState::CheckpointReady) { + NCloud::Reply( + ctx, + *ev, + std::make_unique(MakeError( + E_REJECTED, + TStringBuilder() + << "Can't read from checkpoint " << CheckpointId.Quote() + << " while the data is being filled in."))); + return; + } + + // Remove checkpointId and read checkpoint data from shadow disk. + record.SetCheckpointId(TString()); + ForwardRequestToShadowPartition(ev, ctx); +} + +template +bool TShadowDiskActor::HandleWriteZeroBlocks( + const typename TMethod::TRequest::TPtr& ev, + const NActors::TActorContext& ctx) +{ + if (CanJustForwardWritesToSrcDisk()) { + ForwardRequestToSrcPartition(ev, ctx); + return true; + } + + if (AreWritesToSrcDiskForbidden()) { + NCloud::Reply( + ctx, + *ev, + std::make_unique(MakeError( + E_REJECTED, + TStringBuilder() + << "Can't write to source disk while shadow disk " + << ShadowDiskId.Quote() << " not ready yet."))); + return true; + } + + // Migration is currently in progress. It is necessary to forward write/zero + // requests to the source and shadow disk with the base class + // TNonreplicatedPartitionMigrationCommonActor + return false; +} + +template +void TShadowDiskActor::ForwardRequestToSrcPartition( + const typename TMethod::TRequest::TPtr& ev, + const NActors::TActorContext& ctx) +{ + ForwardMessageToActor(ev, ctx, SrcActorId); +} + +template +void TShadowDiskActor::ForwardRequestToShadowPartition( + const typename TMethod::TRequest::TPtr& ev, + const NActors::TActorContext& ctx) +{ + if (DstActorId == NActors::TActorId()) { + NCloud::Reply( + ctx, + *ev, + std::make_unique( + MakeError(E_REJECTED, "shadow disk partition not ready yet"))); + return; + } + + if (State != EActorState::CheckpointReady) { + NCloud::Reply( + ctx, + *ev, + std::make_unique( + MakeError(E_REJECTED, "shadow disk fill in progress"))); + return; + } + + ForwardMessageToActor(ev, ctx, DstActorId); +} + +void TShadowDiskActor::HandleUpdateShadowDiskStateResponse( + const TEvVolumePrivate::TEvUpdateShadowDiskStateResponse::TPtr& ev, + const NActors::TActorContext& ctx) +{ + auto* msg = ev->Get(); + + switch (msg->NewState) { + case EShadowDiskState::None: + case EShadowDiskState::New: { + STORAGE_CHECK_PRECONDITION( + msg->NewState != EShadowDiskState::New && + msg->NewState != EShadowDiskState::None); + LOG_ERROR_S( + ctx, + TBlockStoreComponents::VOLUME, + "State of shadow disk " << ShadowDiskId.Quote() + << " unexpectedly changed to " + << ToString(msg->NewState)); + State = EActorState::Error; + } break; + case EShadowDiskState::Preparing: { + LOG_INFO_S( + ctx, + TBlockStoreComponents::VOLUME, + "State of shadow disk " + << ShadowDiskId.Quote() << " changed to " + << ToString(msg->NewState) + << ", processed block count: " << msg->ProcessedBlockCount); + State = EActorState::Preparing; + STORAGE_CHECK_PRECONDITION(Acquired()); + } break; + case EShadowDiskState::Ready: { + LOG_INFO_S( + ctx, + TBlockStoreComponents::VOLUME, + "State of shadow disk " << ShadowDiskId.Quote() + << " changed to " + << ToString(msg->NewState)); + State = EActorState::CheckpointReady; + STORAGE_CHECK_PRECONDITION(Acquired()); + } break; + case EShadowDiskState::Error: { + LOG_WARN_S( + ctx, + TBlockStoreComponents::VOLUME, + "State of shadow disk " << ShadowDiskId.Quote() + << " changed to " + << ToString(msg->NewState)); + State = EActorState::Error; + } break; + } +} + +void TShadowDiskActor::HandleUpdateCounters( + const TEvNonreplPartitionPrivate::TEvUpdateCounters::TPtr& ev, + const TActorContext& ctx) +{ + Y_UNUSED(ev); + Y_UNUSED(ctx); + + // Block sending statistics counters from the base class by processing the + // TEvUpdateCounters message ourselves. +} + +void TShadowDiskActor::HandleShadowDiskCounters( + const TEvVolume::TEvDiskRegistryBasedPartitionCounters::TPtr& ev, + const NActors::TActorContext& ctx) +{ + // Forward stat from shadow disk to volume. + ForwardMessageToActor(ev, ctx, VolumeActorId); +} + +void TShadowDiskActor::HandleWakeup( + const NActors::TEvents::TEvWakeup::TPtr& ev, + const NActors::TActorContext& ctx) +{ + Y_UNUSED(ev); + + AcquireShadowDisk(ctx, EAcquireReason::PeriodicalReAcquire); + SchedulePeriodicalReAcquire(ctx); +} + +void TShadowDiskActor::HandleRdmaUnavailable( + const TEvVolume::TEvRdmaUnavailable::TPtr& ev, + const TActorContext& ctx) +{ + ForwardMessageToActor(ev, ctx, VolumeActorId); +} + +void TShadowDiskActor::HandleReacquireDisk( + const TEvVolume::TEvReacquireDisk::TPtr& ev, + const NActors::TActorContext& ctx) +{ + Y_UNUSED(ev); + + // If an E_BS_INVALID_SESSION error occurred while working with the shadow + // disk, the shadow disk partition sends a TEvReacquireDisk message. In this + // case, we try to acquire shadow disk again. + + // If we are in prepare state, this means that writing errors to the + // shadow disk will lead to blocking of the source disk. In that case, we + // are trying to reacquire during short period MaxBlockingTotalTimeout. + + // If we are in ready state, this means that writing errors to the + // shadow disk will not lead to blocking of the source disk. In that case, + // we are trying to reacquire during long period MaxNonBlockingTotalTimeout. + + LOG_WARN_S( + ctx, + TBlockStoreComponents::VOLUME, + "Got TEvReacquireDisk for shadow disk " << ShadowDiskId.Quote()); + + switch (State) { + case EActorState::WaitAcquireForPrepareStart: + case EActorState::WaitAcquireForPrepareContinue: + case EActorState::WaitAcquireForRead: + case EActorState::Error: { + // If we are waiting for disk acquire, or in error state, then we + // should not receive TEvReacquireDisk message. + STORAGE_CHECK_PRECONDITION(false); + } break; + case EActorState::Preparing: + case EActorState::CheckpointReady: { + AcquireShadowDisk(ctx, EAcquireReason::ForcedReAcquire); + } break; + } +} + +} // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/volume/actors/shadow_disk_actor.h b/cloud/blockstore/libs/storage/volume/actors/shadow_disk_actor.h new file mode 100644 index 00000000000..2030300c006 --- /dev/null +++ b/cloud/blockstore/libs/storage/volume/actors/shadow_disk_actor.h @@ -0,0 +1,217 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace NCloud::NBlockStore::NStorage { + +/////////////////////////////////////////////////////////////////////////////// + +// The actor for shadow disk. Can migrate data from the source disk into the +// shadow disk and serves read requests from the checkpoint. +class TShadowDiskActor final + : public TNonreplicatedPartitionMigrationCommonActor + , public IMigrationOwner +{ +public: + enum class EAcquireReason + { + FirstAcquire, + PeriodicalReAcquire, + ForcedReAcquire, + }; + +private: + + // We always start from the WaitAcquireFor* or Error states. + // + // WaitAcquireForPrepareStart -> Preparing -> CheckpointReady + // | | + // v v + // Error Error + // + // WaitAcquireForPrepareContinue -> Preparing -> CheckpointReady + // | | + // v v + // Error Error + // + // WaitAcquireForRead -> CheckpointReady + // | + // v + // Error + + enum class EActorState + { + // We are waiting for the acquire of the shadow disk devices. The shadow + // disk has not filled up yet and we are starting from the beginning of + // the disk. + // We do not block writes to the source disk. + WaitAcquireForPrepareStart, + + // Waiting for the acquire of the shadow disk devices. The + // shadow disk has already been partially filled, and we are not + // starting from the beginning of the disk. + // We block writes to the source disk. + WaitAcquireForPrepareContinue, + + // Waiting for the acquire of the shadow disk devices. The disk is + // already completely filled and we will only read checkpoint data from it. + WaitAcquireForRead, + + // The devices of the shadow disk have been successfully acquired and we + // are currently filling it. + Preparing, + + // The devices of the shadow disk have been successfully acquired and we + // are ready to read the checkpoint data. + CheckpointReady, + + // Something went wrong and we stopped filling the shadow disk. + // At the same time, we do not interfere with the operation of the + // source disk. + Error, + }; + + const TStorageConfigPtr Config; + const NRdma::IClientPtr RdmaClient; + const TNonreplicatedPartitionConfigPtr SrcConfig; + const TString CheckpointId; + const TString SourceDiskId; + const TString ShadowDiskId; + const ui64 MountSeqNumber = 0; + const ui32 Generation = 0; + const NActors::TActorId VolumeActorId; + const NActors::TActorId SrcActorId; + + TNonreplicatedPartitionConfigPtr DstConfig; + NActors::TActorId DstActorId; + ui64 ProcessedBlockCount = 0; + + EActorState State = EActorState::Error; + std::optional TimeoutCalculator; + + NActors::TActorId AcquireActorId; + // The list of devices received on first acquire. + TDevices ShadowDiskDevices; + + bool ForcedReAcquireInProgress = false; + +public: + TShadowDiskActor( + TStorageConfigPtr config, + NRdma::IClientPtr rdmaClient, + IProfileLogPtr profileLog, + IBlockDigestGeneratorPtr digestGenerator, + TString rwClientId, + ui64 mountSeqNumber, + ui32 generation, + TNonreplicatedPartitionConfigPtr srcConfig, + NActors::TActorId volumeActorId, + NActors::TActorId srcActorId, + const TActiveCheckpointInfo& checkpointInfo); + + ~TShadowDiskActor() override; + + // IMigrationOwner implementation + void OnBootstrap(const NActors::TActorContext& ctx) override; + bool OnMessage( + const NActors::TActorContext& ctx, + TAutoPtr& ev) override; + TDuration CalculateMigrationTimeout() override; + void OnMigrationProgress( + const NActors::TActorContext& ctx, + ui64 migrationIndex) override; + void OnMigrationFinished(const NActors::TActorContext& ctx) override; + void OnMigrationError(const NActors::TActorContext& ctx) override; + +private: + void AcquireShadowDisk( + const NActors::TActorContext& ctx, + EAcquireReason acquireReason); + void HandleAcquireDiskResponse( + const TEvDiskRegistry::TEvAcquireDiskResponse::TPtr& ev, + const NActors::TActorContext& ctx); + + void CreateShadowDiskConfig(); + void CreateShadowDiskPartitionActor( + const NActors::TActorContext& ctx, + const TDevices& acquiredShadowDiskDevices); + void SetErrorState(const NActors::TActorContext& ctx); + void SchedulePeriodicalReAcquire(const NActors::TActorContext& ctx); + + // If we haven't started migrating to the shadow disk yet, we can send + // write and zero requests directly to the source disk. + bool CanJustForwardWritesToSrcDisk() const; + + // If the shadow disk is only partially filled, and it is not ready to + // write (because it is not acquired), we reject writes to the source disk. + bool AreWritesToSrcDiskForbidden() const; + + // If the shadow disk is not acquired, or has lost acquiring, then user + // writes to the source disk will not be considered completed, the client + // will repeat write attempts. Since we don't want to slow down the + // client's writing for a long time, we need to keep track of the time + // during which the writing did not occur in order to stop attempts to + // fill a broken shadow disk. + bool AreWritesToSrcDiskImpossible() const; + + bool WaitingForAcquire() const; + bool Acquired() const; + bool ReadOnlyMount() const; + + template + void ForwardRequestToSrcPartition( + const typename TMethod::TRequest::TPtr& ev, + const NActors::TActorContext& ctx); + + template + void ForwardRequestToShadowPartition( + const typename TMethod::TRequest::TPtr& ev, + const NActors::TActorContext& ctx); + + template + void HandleReadBlocks( + const typename TMethod::TRequest::TPtr& ev, + const NActors::TActorContext& ctx); + + template + bool HandleWriteZeroBlocks( + const typename TMethod::TRequest::TPtr& ev, + const NActors::TActorContext& ctx); + + void HandleUpdateShadowDiskStateResponse( + const TEvVolumePrivate::TEvUpdateShadowDiskStateResponse::TPtr& ev, + const NActors::TActorContext& ctx); + + void HandleUpdateCounters( + const TEvNonreplPartitionPrivate::TEvUpdateCounters::TPtr& ev, + const NActors::TActorContext& ctx); + + void HandleShadowDiskCounters( + const TEvVolume::TEvDiskRegistryBasedPartitionCounters::TPtr& ev, + const NActors::TActorContext& ctx); + + void HandleWakeup( + const NActors::TEvents::TEvWakeup::TPtr& ev, + const NActors::TActorContext& ctx); + + void HandleRdmaUnavailable( + const TEvVolume::TEvRdmaUnavailable::TPtr& ev, + const NActors::TActorContext& ctx); + + void HandleReacquireDisk( + const TEvVolume::TEvReacquireDisk::TPtr& ev, + const NActors::TActorContext& ctx); +}; + +} // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/volume/actors/ya.make b/cloud/blockstore/libs/storage/volume/actors/ya.make index dcdbfad1f92..37aeac95772 100644 --- a/cloud/blockstore/libs/storage/volume/actors/ya.make +++ b/cloud/blockstore/libs/storage/volume/actors/ya.make @@ -4,6 +4,7 @@ SRCS( forward_read_marked.cpp forward_write_and_mark_used.cpp read_disk_registry_based_overlay.cpp + shadow_disk_actor.cpp ) PEERDIR( diff --git a/cloud/blockstore/libs/storage/volume/model/checkpoint.cpp b/cloud/blockstore/libs/storage/volume/model/checkpoint.cpp index 9c6813b3551..b2d65981d45 100644 --- a/cloud/blockstore/libs/storage/volume/model/checkpoint.cpp +++ b/cloud/blockstore/libs/storage/volume/model/checkpoint.cpp @@ -91,9 +91,20 @@ void TCheckpointStore::SetShadowDiskState( ui64 totalBlockCount) { if (auto* checkpointData = ActiveCheckpoints.FindPtr(checkpointId)) { + if (shadowDiskState == EShadowDiskState::Ready) { + checkpointData->Data = ECheckpointData::DataPresent; + } checkpointData->ShadowDiskState = shadowDiskState; checkpointData->ProcessedBlockCount = processedBlockCount; checkpointData->TotalBlockCount = totalBlockCount; + + if (auto* checkpointRequest = + CheckpointRequests.FindPtr(checkpointData->RequestId)) + { + checkpointRequest->ShadowDiskState = shadowDiskState; + checkpointRequest->ShadowDiskProcessedBlockCount = + processedBlockCount; + } } } diff --git a/cloud/blockstore/libs/storage/volume/model/merge.cpp b/cloud/blockstore/libs/storage/volume/model/merge.cpp index f6a3beed449..68d9242424e 100644 --- a/cloud/blockstore/libs/storage/volume/model/merge.cpp +++ b/cloud/blockstore/libs/storage/volume/model/merge.cpp @@ -44,4 +44,120 @@ void MergeStripedBitMask( } } +void MergeDescribeBlocksResponse( + NProto::TDescribeBlocksResponse& src, + NProto::TDescribeBlocksResponse& dst, + const ui32 blocksPerStripe, + const ui32 blockSize, + const ui32 partitionsCount, + const ui32 partitionId) +{ + for (const auto& freshBlockRange: src.GetFreshBlockRanges()) { + SplitFreshBlockRangeFromRelativeToGlobalIndices( + freshBlockRange, + blocksPerStripe, + blockSize, + partitionsCount, + partitionId, + &dst); + } + + const auto& srcBlobPieces = src.GetBlobPieces(); + + for (const auto& blobPiece: srcBlobPieces) { + NProto::TBlobPiece dstBlobPiece; + dstBlobPiece.MutableBlobId()->CopyFrom(blobPiece.GetBlobId()); + dstBlobPiece.SetBSGroupId(blobPiece.GetBSGroupId()); + + for (const auto& srcRange: blobPiece.GetRanges()) { + SplitBlobPieceRangeFromRelativeToGlobalIndices( + srcRange, + blocksPerStripe, + partitionsCount, + partitionId, + &dstBlobPiece); + } + dst.MutableBlobPieces()->Add(std::move(dstBlobPiece)); + } +} + +void SplitFreshBlockRangeFromRelativeToGlobalIndices( + const NProto::TFreshBlockRange& srcRange, + const ui32 blocksPerStripe, + const ui32 blockSize, + const ui32 partitionsCount, + const ui32 partitionId, + NProto::TDescribeBlocksResponse* dst) +{ + const ui32 startIndex = srcRange.GetStartIndex(); + ui32 blocksCount = 0; + + const char* srcRangePtr = srcRange.GetBlocksContent().Data(); + while (blocksCount < srcRange.GetBlocksCount()) { + const auto index = RelativeToGlobalIndex( + blocksPerStripe, + startIndex + blocksCount, + partitionsCount, + partitionId); + + const auto stripeRange = CalculateStripeRange(blocksPerStripe, index); + + const auto rangeBlocksCount = std::min( + static_cast(stripeRange.End) - index + 1, + static_cast(srcRange.GetBlocksCount()) - blocksCount); + + NProto::TFreshBlockRange dstRange; + dstRange.SetStartIndex(index); + dstRange.SetBlocksCount(rangeBlocksCount); + + const ui64 bytesCount = rangeBlocksCount * blockSize; + dstRange.MutableBlocksContent()->resize(bytesCount); + char* dstRangePtr = dstRange.MutableBlocksContent()->begin(); + std::memcpy( + dstRangePtr, + srcRangePtr, + bytesCount); + + srcRangePtr += bytesCount; + blocksCount += rangeBlocksCount; + + dst->MutableFreshBlockRanges()->Add(std::move(dstRange)); + } +} + +void SplitBlobPieceRangeFromRelativeToGlobalIndices( + const NProto::TRangeInBlob& srcRange, + const ui32 blocksPerStripe, + const ui32 partitionsCount, + const ui32 partitionId, + NProto::TBlobPiece* dstBlobPiece) +{ + const ui32 blobOffset = srcRange.GetBlobOffset(); + const ui32 blockIndex = srcRange.GetBlockIndex(); + ui32 blocksCount = 0; + + while (blocksCount < srcRange.GetBlocksCount()) { + const auto index = RelativeToGlobalIndex( + blocksPerStripe, + blockIndex + blocksCount, + partitionsCount, + partitionId); + + const auto stripeRange = CalculateStripeRange(blocksPerStripe, index); + + const auto rangeBlocksCount = std::min( + static_cast(stripeRange.End) - index + 1, + static_cast(srcRange.GetBlocksCount()) - blocksCount); + + NProto::TRangeInBlob dstRange; + dstRange.SetBlobOffset(blobOffset + blocksCount); + dstRange.SetBlockIndex(index); + dstRange.SetBlocksCount(rangeBlocksCount); + + blocksCount += rangeBlocksCount; + + dstBlobPiece->MutableRanges()->Add(std::move(dstRange)); + } +} + } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/volume/model/merge.h b/cloud/blockstore/libs/storage/volume/model/merge.h index b00cb681de5..ecd833ab376 100644 --- a/cloud/blockstore/libs/storage/volume/model/merge.h +++ b/cloud/blockstore/libs/storage/volume/model/merge.h @@ -1,5 +1,7 @@ #pragma once +#include + #include namespace NCloud::NBlockStore::NStorage { @@ -15,4 +17,27 @@ void MergeStripedBitMask( const TString& srcMask, TString& dstMask); +void MergeDescribeBlocksResponse( + NProto::TDescribeBlocksResponse& src, + NProto::TDescribeBlocksResponse& dst, + const ui32 blocksPerStripe, + const ui32 blockSize, + const ui32 partitionsCount, + const ui32 partitionId); + +void SplitFreshBlockRangeFromRelativeToGlobalIndices( + const NProto::TFreshBlockRange& srcRange, + const ui32 blocksPerStripe, + const ui32 blockSize, + const ui32 partitionsCount, + const ui32 partitionId, + NProto::TDescribeBlocksResponse* dst); + +void SplitBlobPieceRangeFromRelativeToGlobalIndices( + const NProto::TRangeInBlob& srcRange, + const ui32 blocksPerStripe, + const ui32 partitionsCount, + const ui32 partitionId, + NProto::TBlobPiece* dstBlobPiece); + } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/volume/model/merge_ut.cpp b/cloud/blockstore/libs/storage/volume/model/merge_ut.cpp index 70d3d9b1829..eee5716f4fd 100644 --- a/cloud/blockstore/libs/storage/volume/model/merge_ut.cpp +++ b/cloud/blockstore/libs/storage/volume/model/merge_ut.cpp @@ -130,6 +130,71 @@ Y_UNIT_TEST_SUITE(TMergeTest) } } + Y_UNIT_TEST(ShouldSplitFreshBlockRangeFromRelativeToGlobalIndices) + { + NProto::TFreshBlockRange freshData; + freshData.SetStartIndex(5); + freshData.SetBlocksCount(8); + freshData.MutableBlocksContent()->append(TString(1024, 'X')); + + NProto::TDescribeBlocksResponse dst; + SplitFreshBlockRangeFromRelativeToGlobalIndices( + freshData, + 4, // blocksPerStripe + 128, // blockSize + 2, // partitionsCount + 0, // partitionId + &dst); + + UNIT_ASSERT_VALUES_EQUAL(3, dst.FreshBlockRangesSize()); + + const auto& range1 = dst.GetFreshBlockRanges(0); + UNIT_ASSERT_VALUES_EQUAL(9, range1.GetStartIndex()); + UNIT_ASSERT_VALUES_EQUAL(3, range1.GetBlocksCount()); + + const auto& range2 = dst.GetFreshBlockRanges(1); + UNIT_ASSERT_VALUES_EQUAL(16, range2.GetStartIndex()); + UNIT_ASSERT_VALUES_EQUAL(4, range2.GetBlocksCount()); + + TString actualContent; + for (size_t i = 0; i < dst.FreshBlockRangesSize(); ++i) { + const auto& freshRange = dst.GetFreshBlockRanges(i); + actualContent += freshRange.GetBlocksContent(); + } + + UNIT_ASSERT_VALUES_EQUAL(1024, actualContent.size()); + for (size_t i = 0; i < actualContent.size(); i++) { + UNIT_ASSERT_VALUES_EQUAL('X', actualContent[i]); + } + } + + Y_UNIT_TEST(ShouldSplitBlobPieceRangeFromRelativeToGlobalIndices) + { + NProto::TRangeInBlob rangeInBlob; + rangeInBlob.SetBlobOffset(10); + rangeInBlob.SetBlockIndex(13); + rangeInBlob.SetBlocksCount(1024); + + NProto::TBlobPiece dst; + SplitBlobPieceRangeFromRelativeToGlobalIndices( + rangeInBlob, + 4, // blocksPerStripe + 2, // partitionsCount + 0, // partitionId + &dst); + + UNIT_ASSERT_VALUES_EQUAL(257, dst.RangesSize()); + + const auto& range1 = dst.GetRanges(0); + UNIT_ASSERT_VALUES_EQUAL(10, range1.GetBlobOffset()); + UNIT_ASSERT_VALUES_EQUAL(25, range1.GetBlockIndex()); + UNIT_ASSERT_VALUES_EQUAL(3, range1.GetBlocksCount()); + + const auto& range2 = dst.GetRanges(1); + UNIT_ASSERT_VALUES_EQUAL(13, range2.GetBlobOffset()); + UNIT_ASSERT_VALUES_EQUAL(32, range2.GetBlockIndex()); + UNIT_ASSERT_VALUES_EQUAL(4, range2.GetBlocksCount()); + } } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/volume/model/stripe.cpp b/cloud/blockstore/libs/storage/volume/model/stripe.cpp index f2b514b590f..3b7b3b1cc52 100644 --- a/cloud/blockstore/libs/storage/volume/model/stripe.cpp +++ b/cloud/blockstore/libs/storage/volume/model/stripe.cpp @@ -49,6 +49,16 @@ ui64 RelativeToGlobalIndex( return stripe * blocksPerStripe + offsetInStripe; } +TBlockRange64 CalculateStripeRange( + const ui32 blocksPerStripe, + const ui64 globalIndex) +{ + const auto stripeInd = globalIndex / blocksPerStripe; + return TBlockRange64::WithLength( + stripeInd * blocksPerStripe, + blocksPerStripe); +} + ui32 CalculateRequestCount( const ui32 blocksPerStripe, const TBlockRange64& original, diff --git a/cloud/blockstore/libs/storage/volume/model/stripe.h b/cloud/blockstore/libs/storage/volume/model/stripe.h index 3cc7c6dfedf..2856c2c8b47 100644 --- a/cloud/blockstore/libs/storage/volume/model/stripe.h +++ b/cloud/blockstore/libs/storage/volume/model/stripe.h @@ -31,6 +31,10 @@ ui64 RelativeToGlobalIndex( const ui32 partitionCount, const ui32 partitionId); +TBlockRange64 CalculateStripeRange( + const ui32 blocksPerStripe, + const ui64 globalIndex); + ui32 CalculateRequestCount( const ui32 blocksPerStripe, const TBlockRange64& original, diff --git a/cloud/blockstore/libs/storage/volume/model/stripe_ut.cpp b/cloud/blockstore/libs/storage/volume/model/stripe_ut.cpp index 8fa779b74b1..276c14e4493 100644 --- a/cloud/blockstore/libs/storage/volume/model/stripe_ut.cpp +++ b/cloud/blockstore/libs/storage/volume/model/stripe_ut.cpp @@ -152,6 +152,25 @@ Y_UNIT_TEST_SUITE(TStripeTest) UNIT_ASSERT_VALUES_EQUAL(75, RelativeToGlobalIndex(10, 25, 3, 1)); } + Y_UNIT_TEST(ShouldCalculateStripeRange) + { + UNIT_ASSERT_VALUES_EQUAL( + DescribeRange(TBlockRange64::WithLength(0, 10)), + DescribeRange(CalculateStripeRange(10, 0))); + + UNIT_ASSERT_VALUES_EQUAL( + DescribeRange(TBlockRange64::WithLength(0, 10)), + DescribeRange(CalculateStripeRange(10, 5))); + + UNIT_ASSERT_VALUES_EQUAL( + DescribeRange(TBlockRange64::WithLength(10, 10)), + DescribeRange(CalculateStripeRange(10, 15))); + + UNIT_ASSERT_VALUES_EQUAL( + DescribeRange(TBlockRange64::WithLength(10, 10)), + DescribeRange(CalculateStripeRange(10, 19))); + } + Y_UNIT_TEST(ShouldCalculateRequestCount) { UNIT_ASSERT_VALUES_EQUAL( diff --git a/cloud/blockstore/libs/storage/volume/partition_requests.h b/cloud/blockstore/libs/storage/volume/partition_requests.h index a68d30c6088..dc097133eae 100644 --- a/cloud/blockstore/libs/storage/volume/partition_requests.h +++ b/cloud/blockstore/libs/storage/volume/partition_requests.h @@ -14,6 +14,8 @@ #include #include #include + +#include #include #include @@ -50,14 +52,20 @@ bool ToPartitionRequests( TVector>* requests, TBlockRange64* blockRange) { - Y_UNUSED(partitions); Y_UNUSED(blockSize); Y_UNUSED(blocksPerStripe); Y_UNUSED(ev); Y_UNUSED(requests); Y_UNUSED(blockRange); - Y_ABORT_UNLESS(0); + Y_ABORT_UNLESS(!partitions.empty()); + STORAGE_VERIFY_C( + false, + NCloud::TWellKnownEntityTypes::DISK, + partitions[0].PartitionConfig.GetDiskId(), + TStringBuilder() << "ToPartitionRequests not implemented for " + << TMethod::Name); + return false; } @@ -678,6 +686,7 @@ class TPartitionRequestActor final const ui64 VolumeRequestId; const TBlockRange64 OriginalRange; const ui32 BlocksPerStripe; + const ui32 BlockSize; const ui32 PartitionsCount; TVector> PartitionRequests; const TRequestTraceInfo TraceInfo; @@ -695,6 +704,7 @@ class TPartitionRequestActor final ui64 volumeRequestId, TBlockRange64 originalRange, ui32 blocksPerStripe, + ui32 blockSize, ui32 partitionsCount, TVector> partitionRequests, TRequestTraceInfo traceInfo); @@ -865,6 +875,25 @@ class TPartitionRequestActor final } } + void Merge( + NProto::TDescribeBlocksResponse& src, + ui32 requestNo, + NProto::TDescribeBlocksResponse& dst) + { + if (FAILED(src.GetError().GetCode())) { + *dst.MutableError() = std::move(*src.MutableError()); + return; + } + + MergeDescribeBlocksResponse( + src, + dst, + BlocksPerStripe, + BlockSize, + PartitionsCount, + PartitionRequests[requestNo].PartitionId); + } + template void Merge( T& src, @@ -908,6 +937,7 @@ TPartitionRequestActor::TPartitionRequestActor( ui64 volumeRequestId, TBlockRange64 originalRange, ui32 blocksPerStripe, + ui32 blockSize, ui32 partitionsCount, TVector> partitionRequests, TRequestTraceInfo traceInfo) @@ -916,6 +946,7 @@ TPartitionRequestActor::TPartitionRequestActor( , VolumeRequestId(volumeRequestId) , OriginalRange(originalRange) , BlocksPerStripe(blocksPerStripe) + , BlockSize(blockSize) , PartitionsCount(partitionsCount) , PartitionRequests(std::move(partitionRequests)) , TraceInfo(std::move(traceInfo)) diff --git a/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp b/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp index c2a563fb7c4..8f943a99309 100644 --- a/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp +++ b/cloud/blockstore/libs/storage/volume/testlib/test_env.cpp @@ -369,11 +369,11 @@ std::unique_ptr TVolumeClient::CreateDescri std::unique_ptr TVolumeClient::CreateCreateCheckpointRequest( const TString& checkpointId, - bool isLight) + NProto::ECheckpointType checkpointType) { auto request = std::make_unique(); request->Record.SetCheckpointId(checkpointId); - request->Record.SetIsLight(isLight); + request->Record.SetCheckpointType(checkpointType); return request; } @@ -669,6 +669,7 @@ std::unique_ptr PrepareTestActorRuntime( 0 ) ); + runtime->EnableScheduleForActor(MakeDiskRegistryProxyServiceId()); runtime->AddLocalService( MakeDiskAgentServiceId(runtime->GetNodeId()), diff --git a/cloud/blockstore/libs/storage/volume/testlib/test_env.h b/cloud/blockstore/libs/storage/volume/testlib/test_env.h index d6e0b57f2cd..1e838cd4231 100644 --- a/cloud/blockstore/libs/storage/volume/testlib/test_env.h +++ b/cloud/blockstore/libs/storage/volume/testlib/test_env.h @@ -393,7 +393,7 @@ class TVolumeClient std::unique_ptr CreateCreateCheckpointRequest( const TString& checkpointId, - bool isLight = false); + NProto::ECheckpointType checkpointType = NProto::ECheckpointType::NORMAL); std::unique_ptr CreateDeleteCheckpointRequest( const TString& checkpointId); diff --git a/cloud/blockstore/libs/storage/volume/volume_actor.cpp b/cloud/blockstore/libs/storage/volume/volume_actor.cpp index 00ec36b68fd..10c3aaeba88 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor.cpp @@ -39,6 +39,12 @@ using namespace NCloud::NBlockStore::NStorage::NPartition; using namespace NCloud::NStorage; +namespace { + +constexpr TInstant DRTabletIdRequestRetryInterval = TInstant::Seconds(3); + +} // namespace + //////////////////////////////////////////////////////////////////////////////// const TVolumeActor::TStateInfo TVolumeActor::States[STATE_MAX] = { @@ -147,6 +153,8 @@ void TVolumeActor::ReportTabletState(const TActorContext& ctx) void TVolumeActor::RegisterCounters(const TActorContext& ctx) { + Y_DEBUG_ABORT_UNLESS(State); + Y_UNUSED(ctx); if (!Counters) { @@ -162,7 +170,7 @@ void TVolumeActor::RegisterCounters(const TActorContext& ctx) } if (!VolumeSelfCounters) { - VolumeSelfCounters = CreateVolumeSelfCounters(CountersPolicy); + VolumeSelfCounters = CreateVolumeSelfCounters(State->CountersPolicy()); } } @@ -232,7 +240,6 @@ void TVolumeActor::OnActivateExecutor(const TActorContext& ctx) "[%lu] Activated executor", TabletID()); - RegisterCounters(ctx); ScheduleRegularUpdates(ctx); if (!Executor()->GetStats().IsFollower) { @@ -289,7 +296,7 @@ void TVolumeActor::BeforeDie(const TActorContext& ctx) for (auto& [part, handler]: WaitForPartitions) { if (handler) { - std::invoke(handler, ctx, MakeError(E_REJECTED, "Tablet is dead")); + std::invoke(handler, ctx, MakeError(E_REJECTED, "tablet is shutting down")); } } StoppedPartitions.clear(); @@ -393,12 +400,26 @@ void TVolumeActor::OnServicePipeDisconnect( void TVolumeActor::RegisterVolume(const TActorContext& ctx) { + DoRegisterVolume(ctx, State->GetDiskId()); +} + +void TVolumeActor::DoRegisterVolume( + const TActorContext& ctx, + const TString& diskId) +{ + Y_DEBUG_ABORT_UNLESS(diskId); + NProto::TVolume volume; VolumeConfigToVolume(State->GetMeta().GetVolumeConfig(), volume); + if (State->GetDiskId() != diskId) { + // Handle shadow disk. + volume.SetDiskId(diskId); + } + { auto request = std::make_unique( - State->GetDiskId(), + volume.GetDiskId(), TabletID(), volume); NCloud::Send(ctx, MakeStorageServiceId(), std::move(request)); @@ -406,7 +427,7 @@ void TVolumeActor::RegisterVolume(const TActorContext& ctx) { auto request = std::make_unique( - State->GetDiskId(), + volume.GetDiskId(), TabletID(), std::move(volume)); NCloud::Send(ctx, MakeStorageStatsServiceId(), std::move(request)); @@ -415,21 +436,39 @@ void TVolumeActor::RegisterVolume(const TActorContext& ctx) void TVolumeActor::UnregisterVolume(const TActorContext& ctx) { - if (State) { - { - auto request = std::make_unique( - State->GetDiskId()); - NCloud::Send(ctx, MakeStorageServiceId(), std::move(request)); - } + if (!State) { + return; + } - { - auto request = std::make_unique( - State->GetDiskId()); - NCloud::Send(ctx, MakeStorageStatsServiceId(), std::move(request)); + DoUnregisterVolume(ctx, State->GetDiskId()); + + // Unregister shadow disks volumes. + for (const auto& [checkpointId, checkpointInfo]: + State->GetCheckpointStore().GetActiveCheckpoints()) + { + if (checkpointInfo.ShadowDiskId) { + DoUnregisterVolume(ctx, checkpointInfo.ShadowDiskId); } } } +void TVolumeActor::DoUnregisterVolume( + const TActorContext& ctx, + const TString& diskId) +{ + Y_DEBUG_ABORT_UNLESS(diskId); + + NCloud::Send( + ctx, + MakeStorageServiceId(), + std::make_unique(diskId)); + + NCloud::Send( + ctx, + MakeStorageStatsServiceId(), + std::make_unique(diskId)); +} + void TVolumeActor::SendVolumeConfigUpdated(const TActorContext& ctx) { NProto::TVolume volume; @@ -670,6 +709,28 @@ void TVolumeActor::HandleServerDestroyed( OnServicePipeDisconnect(ctx, msg->ServerId, now); } +void TVolumeActor::HandleGetDrTabletInfoResponse( + const TEvDiskRegistryProxy::TEvGetDrTabletInfoResponse::TPtr& ev, + const NActors::TActorContext& ctx) +{ + Y_UNUSED(ctx); + const auto* msg = ev->Get(); + + if (HasError(msg->GetError())) { + auto request = + std::make_unique(); + TActivationContext::Schedule( + DRTabletIdRequestRetryInterval, + new IEventHandle( + MakeDiskRegistryProxyServiceId(), + SelfId(), + request.release())); + return; + } + + DiskRegistryTabletId = msg->TabletId; +} + void TVolumeActor::ResetServicePipes(const TActorContext& ctx) { for (const auto& actor: State->ClearPipeServerIds(ctx.Now())) { @@ -859,6 +920,10 @@ STFUNC(TVolumeActor::StateInit) TEvVolumePrivate::TEvUpdateShadowDiskStateRequest, HandleUpdateShadowDiskState); + HFunc( + TEvDiskRegistryProxy::TEvGetDrTabletInfoResponse, + HandleGetDrTabletInfoResponse); + BLOCKSTORE_HANDLE_REQUEST(WaitReady, TEvVolume) default: @@ -952,6 +1017,10 @@ STFUNC(TVolumeActor::StateWork) TEvVolumePrivate::TEvUpdateShadowDiskStateRequest, HandleUpdateShadowDiskState); + HFunc( + TEvDiskRegistryProxy::TEvGetDrTabletInfoResponse, + HandleGetDrTabletInfoResponse); + default: if (!HandleRequests(ev) && !HandleDefaultEvents(ev, SelfId())) { HandleUnexpectedEvent(ev, TBlockStoreComponents::VOLUME); @@ -989,6 +1058,8 @@ STFUNC(TVolumeActor::StateZombie) IgnoreFunc(TEvVolumePrivate::TEvUpdateShadowDiskStateRequest); + IgnoreFunc(TEvDiskRegistryProxy::TEvGetDrTabletInfoResponse); + default: if (!RejectRequests(ev)) { HandleUnexpectedEvent(ev, TBlockStoreComponents::VOLUME); diff --git a/cloud/blockstore/libs/storage/volume/volume_actor.h b/cloud/blockstore/libs/storage/volume/volume_actor.h index eb373039e63..d96c2ad29c3 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor.h +++ b/cloud/blockstore/libs/storage/volume/volume_actor.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -247,7 +248,6 @@ class TVolumeActor final bool ShuttingDown = false; - EPublishingPolicy CountersPolicy = EPublishingPolicy::All; TVolumeSelfCountersPtr VolumeSelfCounters; struct TAcquireReleaseDiskRequest @@ -323,6 +323,8 @@ class TVolumeActor final ui32 MultipartitionWriteAndZeroRequestsInProgress = 0; + ui64 DiskRegistryTabletId = 0; + // requests in progress THashSet Actors; TIntrusiveList ActiveTransactions; @@ -382,6 +384,9 @@ class TVolumeActor final } void SendPartStatsToService(const NActors::TActorContext& ctx); + void DoSendPartStatsToService( + const NActors::TActorContext& ctx, + const TString& diskId); void SendSelfStatsToService(const NActors::TActorContext& ctx); void OnActivateExecutor(const NActors::TActorContext& ctx) override; @@ -470,7 +475,13 @@ class TVolumeActor final void ResetServicePipes(const NActors::TActorContext& ctx); void RegisterVolume(const NActors::TActorContext& ctx); + void DoRegisterVolume( + const NActors::TActorContext& ctx, + const TString& diskId); void UnregisterVolume(const NActors::TActorContext& ctx); + void DoUnregisterVolume( + const NActors::TActorContext& ctx, + const TString& diskId); void SendVolumeConfigUpdated(const NActors::TActorContext& ctx); void SendVolumeSelfCounters(const NActors::TActorContext& ctx); @@ -642,6 +653,14 @@ class TVolumeActor final void ProcessNextCheckpointRequest(const NActors::TActorContext& ctx); + void HandleUpdateShadowDiskStateRequest( + const TEvVolumePrivate::TEvUpdateShadowDiskStateRequest::TPtr& ev, + const NActors::TActorContext& ctx); + + void HandleGetDrTabletInfoResponse( + const TEvDiskRegistryProxy::TEvGetDrTabletInfoResponse::TPtr& ev, + const NActors::TActorContext& ctx); + void HandleTabletStatus( const TEvBootstrapper::TEvStatus::TPtr& ev, const NActors::TActorContext& ctx); @@ -916,6 +935,12 @@ class TVolumeActor final const TEvPartitionCommonPrivate::TEvLongRunningOperation::TPtr& ev, const NActors::TActorContext& ctx); + NActors::TActorId WrapNonreplActorIfNeeded( + const NActors::TActorContext& ctx, + NActors::TActorId nonreplicatedActorId, + std::shared_ptr srcConfig); + + void RestartDiskRegistryBasedPartition(const NActors::TActorContext& ctx); void StartPartitionsImpl(const NActors::TActorContext& ctx); BLOCKSTORE_VOLUME_REQUESTS(BLOCKSTORE_IMPLEMENT_REQUEST, TEvVolume) diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_allocatedisk.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_allocatedisk.cpp index 15c4d6cfba1..c6ff140082e 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_allocatedisk.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_allocatedisk.cpp @@ -219,8 +219,7 @@ void TVolumeActor::HandleAllocateDiskIfNeeded( void TVolumeActor::ScheduleAllocateDiskIfNeeded(const TActorContext& ctx) { if (State && !DiskAllocationScheduled) { - const auto mediaKind = State->GetConfig().GetStorageMediaKind(); - if (!IsDiskRegistryMediaKind(mediaKind)) { + if (!State->IsDiskRegistryMediaKind()) { return; } diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_checkpoint.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_checkpoint.cpp index 04d9e870a1c..bc6b02ea8de 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_checkpoint.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_checkpoint.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -138,6 +139,7 @@ class TCheckpointActor final const TActorId VolumeActorId; const TPartitionDescrs PartitionDescrs; const TRequestTraceInfo TraceInfo; + const ECheckpointRequestType RequestType; const bool CreateCheckpointShadowDisk; ui32 DrainResponses = 0; @@ -156,6 +158,7 @@ class TCheckpointActor final TActorId volumeActorId, TPartitionDescrs partitionDescrs, TRequestTraceInfo traceInfo, + ECheckpointRequestType requestType, bool createCheckpointShadowDisk); void Bootstrap(const TActorContext& ctx); @@ -228,6 +231,7 @@ TCheckpointActor::TCheckpointActor( TActorId volumeActorId, TPartitionDescrs partitionDescrs, TRequestTraceInfo traceInfo, + ECheckpointRequestType requestType, bool createCheckpointShadowDisk) : RequestInfo(std::move(requestInfo)) , RequestId(requestId) @@ -237,6 +241,7 @@ TCheckpointActor::TCheckpointActor( , VolumeActorId(volumeActorId) , PartitionDescrs(std::move(partitionDescrs)) , TraceInfo(std::move(traceInfo)) + , RequestType(requestType) , CreateCheckpointShadowDisk(createCheckpointShadowDisk) , ChildCallContexts(Reserve(PartitionDescrs.size())) {} @@ -541,7 +546,7 @@ void TCheckpointActor::HandlePoisonPill( Error = MakeError( E_REJECTED, - "tablet is dead" + "tablet is shutting down" ); ReplyAndDie(ctx); @@ -687,6 +692,13 @@ void TCheckpointActor::DoActionForBlobStorageBasedPartition( const auto selfId = TBase::SelfId(); auto request = std::make_unique(); request->Record.SetCheckpointId(CheckpointId); + if constexpr (std::is_same_v) { + request->Record.SetCheckpointType( + RequestType == ECheckpointRequestType::CreateWithoutData + ? NProto::ECheckpointType::WITHOUT_DATA + : NProto::ECheckpointType::NORMAL + ); + } ForkTraces(request->CallContext); @@ -1155,6 +1167,7 @@ void TVolumeActor::ProcessNextCheckpointRequest(const TActorContext& ctx) requestInfo->IsTraced, requestInfo->TraceTs, TraceSerializer), + request.ReqType, Config->GetUseShadowDisksForNonreplDiskCheckpoints()); break; } @@ -1179,6 +1192,7 @@ void TVolumeActor::ProcessNextCheckpointRequest(const TActorContext& ctx) requestInfo->IsTraced, requestInfo->TraceTs, TraceSerializer), + request.ReqType, Config->GetUseShadowDisksForNonreplDiskCheckpoints()); break; } @@ -1197,6 +1211,7 @@ void TVolumeActor::ProcessNextCheckpointRequest(const TActorContext& ctx) requestInfo->IsTraced, requestInfo->TraceTs, TraceSerializer), + request.ReqType, Config->GetUseShadowDisksForNonreplDiskCheckpoints()); break; } @@ -1331,6 +1346,19 @@ void TVolumeActor::CompleteUpdateCheckpointRequest( args.Completed ? "completed" : "rejected", args.ShadowDiskId.Quote().c_str()); + const bool needToDestroyShadowActor = + (request.ReqType == ECheckpointRequestType::Delete || + request.ReqType == ECheckpointRequestType::DeleteData) && + State->GetCheckpointStore().HasShadowActor(request.CheckpointId); + + if (needToDestroyShadowActor) { + auto checkpointInfo = + State->GetCheckpointStore().GetCheckpoint(request.CheckpointId); + if (checkpointInfo && checkpointInfo->ShadowDiskId) { + DoUnregisterVolume(ctx, checkpointInfo->ShadowDiskId); + } + } + State->SetCheckpointRequestFinished( request, args.Completed, @@ -1338,6 +1366,15 @@ void TVolumeActor::CompleteUpdateCheckpointRequest( args.ShadowDiskState); CheckpointRequests.erase(request.RequestId); + const bool needToCreateShadowActor = + request.ReqType == ECheckpointRequestType::Create && + !request.ShadowDiskId.Empty() && + !State->GetCheckpointStore().HasShadowActor(request.CheckpointId); + + if (needToDestroyShadowActor || needToCreateShadowActor) { + RestartDiskRegistryBasedPartition(ctx); + } + if (request.Type == ECheckpointType::Light) { const bool needReply = args.RequestInfo && SelfId() != args.RequestInfo->Sender; diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_forward.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_forward.cpp index 62c3ac734f9..e191c59425b 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_forward.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_forward.cpp @@ -27,6 +27,12 @@ namespace { //////////////////////////////////////////////////////////////////////////////// +template +constexpr bool IsDescribeBlocksMethod = + std::is_same_v; + +//////////////////////////////////////////////////////////////////////////////// + template bool CanForwardToPartition(ui32 partitionCount) { @@ -127,7 +133,11 @@ bool TVolumeActor::HandleRequest( return false; } - if (partitionRequests.size() == 1) { + // Should always forward request via TPartitionRequestActor for + // DesribeBlocks method and multi-partitioned volume. + if (State->GetPartitions().size() == 1 || + (partitionRequests.size() == 1 && !IsDescribeBlocksMethod)) + { ev->Get()->Record = std::move(partitionRequests.front().Event->Record); SendRequestToPartition( ctx, @@ -166,6 +176,7 @@ bool TVolumeActor::HandleRequest( volumeRequestId, blockRange, blocksPerStripe, + State->GetBlockSize(), State->GetPartitions().size(), std::move(partitionRequests), TRequestTraceInfo(isTraced, traceTs, TraceSerializer)); @@ -315,7 +326,7 @@ template NProto::TError TVolumeActor::ProcessAndValidateReadFromCheckpoint( typename TMethod::TRequest::ProtoRecordType& record) const { - if (!IsDiskRegistryMediaKind(State->GetConfig().GetStorageMediaKind())) { + if (!State->IsDiskRegistryMediaKind()) { return {}; } @@ -574,7 +585,7 @@ void TVolumeActor::ForwardRequest( return; } - if (IsDiskRegistryMediaKind(State->GetConfig().GetStorageMediaKind())) { + if (State->IsDiskRegistryMediaKind()) { if (State->GetMeta().GetDevices().empty()) { replyError(MakeError(E_REJECTED, TStringBuilder() << "Storage not allocated for volume: " diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_forward_trackused.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_forward_trackused.cpp index bb8c8004739..248f485ecfe 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_forward_trackused.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_forward_trackused.cpp @@ -33,10 +33,10 @@ bool TVolumeActor::SendRequestToPartitionWithUsedBlockTracking( const auto& volumeConfig = State->GetMeta().GetVolumeConfig(); const bool encryptedDiskRegistryBasedDisk = - IsDiskRegistryMediaKind(State->GetConfig().GetStorageMediaKind()) && + State->IsDiskRegistryMediaKind() && volumeConfig.GetEncryptionDesc().GetMode() != NProto::NO_ENCRYPTION; const bool overlayDiskRegistryBasedDisk = - IsDiskRegistryMediaKind(State->GetConfig().GetStorageMediaKind()) && + State->IsDiskRegistryMediaKind() && !State->GetBaseDiskId().Empty(); if constexpr (IsWriteMethod) { @@ -146,7 +146,7 @@ bool TVolumeActor::SendRequestToPartitionWithUsedBlockTracking< // TODO: NBS-3228: remove checks for disk registry based disks after normal // checkpoints for disk registry based disks are implemented completely. - if (!IsDiskRegistryMediaKind(State->GetConfig().GetStorageMediaKind())) { + if (!State->IsDiskRegistryMediaKind()) { return false; } diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_loadstate.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_loadstate.cpp index b5beace57c8..a9a75354aca 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_loadstate.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_loadstate.cpp @@ -137,12 +137,12 @@ void TVolumeActor::CompleteLoadState( ResetThrottlingPolicy(); for (const auto& partStats: args.PartStats) { - const auto& stats = partStats.Stats; - // info doesn't have to be always present // see NBS-1668#603e955e319cc33b747904fb - if (auto* info = State->GetPartitionStatInfoById(partStats.Id)) { - CopyCachedStatsToPartCounters(stats, *info); + if (auto* info = + State->GetPartitionStatInfoByTabletId(partStats.TabletId)) + { + CopyCachedStatsToPartCounters(partStats.Stats, *info); } } @@ -153,17 +153,10 @@ void TVolumeActor::CompleteLoadState( "[%lu] State initialization finished", TabletID()); + RegisterCounters(ctx); RegisterVolume(ctx); - const bool isDiskRegistryBased = - IsDiskRegistryMediaKind(State->GetConfig().GetStorageMediaKind()); - if (isDiskRegistryBased) { - CountersPolicy = EPublishingPolicy::DiskRegistryBased; - } else { - CountersPolicy = EPublishingPolicy::Repl; - } - - if (isDiskRegistryBased || PendingRequests.size()) { + if (State->IsDiskRegistryMediaKind() || PendingRequests.size()) { StartPartitionsForUse(ctx); } else if (State->GetShouldStartPartitionsForGc(ctx.Now()) && !Config->GetDisableStartPartitionsForGc()) @@ -186,6 +179,10 @@ void TVolumeActor::CompleteLoadState( TabletID(), GetLoadTime().MicroSeconds()); + ctx.Send( + MakeDiskRegistryProxyServiceId(), + new TEvDiskRegistryProxy::TEvGetDrTabletInfoRequest()); + SignalTabletActive(ctx); ScheduleProcessUpdateVolumeConfig(ctx); ScheduleAllocateDiskIfNeeded(ctx); diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_monitoring.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_monitoring.cpp index 477aba45593..4b0537cfcc7 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_monitoring.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_monitoring.cpp @@ -655,12 +655,12 @@ void TVolumeActor::RenderCheckpoints(IOutputStream& out) const TABLEH() { out << "Delete"; } } } - for (const auto& [r, checkpointType]: + for (const auto& [r, checkpointInfo]: State->GetCheckpointStore().GetActiveCheckpoints()) { TABLER() { TABLED() { out << r; } - TABLED() { out << checkpointType; } + TABLED() { out << checkpointInfo; } TABLED() { BuildDeleteCheckpointButton( out, @@ -672,8 +672,8 @@ void TVolumeActor::RenderCheckpoints(IOutputStream& out) const } }; DIV_CLASS("row") { - const bool isDiskRegistryMediaKind = IsDiskRegistryMediaKind( - State->GetConfig().GetStorageMediaKind()); + const bool isDiskRegistryMediaKind = + State->IsDiskRegistryMediaKind(); TAG(TH3) { out << "CheckpointRequests"; } TABLE_SORTABLE_CLASS("table table-bordered") { TABLEHEAD() { @@ -971,7 +971,20 @@ void TVolumeActor::RenderConfig(IOutputStream& out) const TABLEHEAD() { TABLER() { TABLED() { out << "Disk Id"; } - TABLED() { out << volumeConfig.GetDiskId(); } + TABLED() { + const auto mediaKind = + State->GetConfig().GetStorageMediaKind(); + if (DiskRegistryTabletId && + IsDiskRegistryMediaKind(mediaKind)) + { + out << "" << volumeConfig.GetDiskId() << ""; + } else { + out << volumeConfig.GetDiskId(); + } + } } TABLER() { TABLED() { out << "Base Disk Id"; } @@ -1593,8 +1606,8 @@ void TVolumeActor::HandleHttpInfo_RenderNonreplPartitionInfo( } } - auto& infos = State->GetPartitionStatInfos(); - if (infos && infos.front().LastCounters) { + auto infos = State->GetPartitionStatInfos(); + if (!infos.empty() && infos.front().LastCounters) { const auto& counters = infos.front().LastCounters; TABLER() { TABLED() { out << "HasBrokenDevice"; } diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_monitoring_checkpoint.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_monitoring_checkpoint.cpp index 60cc64e2dde..6991a843416 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_monitoring_checkpoint.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_monitoring_checkpoint.cpp @@ -173,7 +173,7 @@ void THttpCheckpointActor::HandleCreateCheckpointRequest( { Y_UNUSED(ev); - ReplyAndDie(ctx, "create", MakeError(E_REJECTED, "Tablet is dead")); + ReplyAndDie(ctx, "create", MakeError(E_REJECTED, "tablet is shutting down")); } void THttpCheckpointActor::HandleDeleteCheckpointRequest( @@ -182,7 +182,7 @@ void THttpCheckpointActor::HandleDeleteCheckpointRequest( { Y_UNUSED(ev); - ReplyAndDie(ctx, "delete", MakeError(E_REJECTED, "Tablet is dead")); + ReplyAndDie(ctx, "delete", MakeError(E_REJECTED, "tablet is shutting down")); } void THttpCheckpointActor::HandlePoisonPill( @@ -191,7 +191,7 @@ void THttpCheckpointActor::HandlePoisonPill( { Y_UNUSED(ev); - ReplyAndDie(ctx, Action == CreateCheckpoint ? "create" : "delete", MakeError(E_REJECTED, "Tablet is dead")); + ReplyAndDie(ctx, Action == CreateCheckpoint ? "create" : "delete", MakeError(E_REJECTED, "tablet is shutting down")); } STFUNC(THttpCheckpointActor::StateWork) diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_reset_seqnumber.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_reset_seqnumber.cpp index 1a47d841783..9af2fba9460 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_reset_seqnumber.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_reset_seqnumber.cpp @@ -122,7 +122,7 @@ void THttpResetMountSeqNumberActor::HandleResetMountSeqNumber( Y_UNUSED(ev); Y_UNUSED(ctx); - Notify(ctx, "tablet is dead", EAlertLevel::DANGER); + Notify(ctx, "tablet is shutting down", EAlertLevel::DANGER); Die(ctx); } @@ -134,7 +134,7 @@ void THttpResetMountSeqNumberActor::HandlePoisonPill( Y_UNUSED(ev); Y_UNUSED(ctx); - Notify(ctx, "tablet is dead", EAlertLevel::DANGER); + Notify(ctx, "tablet is shutting down", EAlertLevel::DANGER); Die(ctx); } diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_resync.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_resync.cpp index 96c98ccec03..3e40f958ae1 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_resync.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_resync.cpp @@ -167,9 +167,7 @@ void TVolumeActor::CompleteToggleResync( Y_UNUSED(args); if (args.ResyncWasNeeded != State->IsMirrorResyncNeeded()) { - StopPartitions(ctx); - StartPartitionsForUse(ctx); - ResetServicePipes(ctx); + RestartDiskRegistryBasedPartition(ctx); } else { State->SetReadWriteError({}); } diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp index 580b74ac1bf..6380b31f12a 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp @@ -11,14 +11,14 @@ #include #include #include - +#include #include +#include + #include #include -#include - namespace NCloud::NBlockStore::NStorage { using namespace NActors; @@ -252,8 +252,54 @@ void TVolumeActor::SetupDiskRegistryBasedPartitions(const TActorContext& ctx) } State->SetDiskRegistryBasedPartitionActor( - nonreplicatedActorId, - std::move(nonreplicatedConfig)); + WrapNonreplActorIfNeeded(ctx, nonreplicatedActorId, nonreplicatedConfig), + nonreplicatedConfig); +} + +NActors::TActorId TVolumeActor::WrapNonreplActorIfNeeded( + const TActorContext& ctx, + NActors::TActorId nonreplicatedActorId, + std::shared_ptr srcConfig) +{ + for (const auto& [checkpointId, checkpointInfo]: + State->GetCheckpointStore().GetActiveCheckpoints()) + { + if (checkpointInfo.Data == ECheckpointData::DataDeleted || + checkpointInfo.ShadowDiskId.Empty() || + State->GetCheckpointStore().HasShadowActor(checkpointId)) + { + continue; + } + + nonreplicatedActorId = NCloud::Register( + ctx, + Config, + GetRdmaClient(), + ProfileLog, + BlockDigestGenerator, + State->GetReadWriteAccessClientId(), + State->GetMountSeqNumber(), + Executor()->Generation(), + srcConfig, + SelfId(), + nonreplicatedActorId, + checkpointInfo); + + State->GetCheckpointStore().ShadowActorCreated(checkpointId); + DoRegisterVolume(ctx, checkpointInfo.ShadowDiskId); + } + return nonreplicatedActorId; +} + +void TVolumeActor::RestartDiskRegistryBasedPartition(const TActorContext& ctx) +{ + if (!State->IsDiskRegistryMediaKind()) { + return; + } + + StopPartitions(ctx); + StartPartitionsForUse(ctx); + ResetServicePipes(ctx); } void TVolumeActor::StartPartitionsImpl(const TActorContext& ctx) @@ -269,7 +315,7 @@ void TVolumeActor::StartPartitionsImpl(const TActorContext& ctx) SendBootExternalRequest(ctx, partition); } - if (IsDiskRegistryMediaKind(State->GetConfig().GetStorageMediaKind())) { + if (State->IsDiskRegistryMediaKind()) { SetupDiskRegistryBasedPartitions(ctx); if (State->Ready()) { @@ -296,6 +342,12 @@ void TVolumeActor::StopPartitions(const TActorContext& ctx) return; } + for (const auto& [checkpointId, _]: + State->GetCheckpointStore().GetActiveCheckpoints()) + { + State->GetCheckpointStore().ShadowActorDestroyed(checkpointId); + } + for (auto& part : State->GetPartitions()) { // Reset previous boot attempts part.RetryCookie.Detach(); @@ -523,7 +575,6 @@ void TVolumeActor::HandleTabletStatus( switch (msg->Status) { case TEvBootstrapper::STARTED: partition->SetStarted(msg->TabletUser); - Y_ABORT_UNLESS(State->SetPartitionStatActor(msg->TabletId, msg->TabletUser)); NCloud::Send( ctx, msg->TabletUser, diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_stats.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_stats.cpp index 57ebe11dc7a..512741d2a6d 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_stats.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_stats.cpp @@ -31,6 +31,8 @@ LWTRACE_USING(BLOCKSTORE_STORAGE_PROVIDER); xxx(CleanupQueueBytes, __VA_ARGS__)\ xxx(GarbageQueueBytes, __VA_ARGS__)\ xxx(ChannelHistorySize, __VA_ARGS__)\ + xxx(UnconfirmedBlobCount, __VA_ARGS__)\ + xxx(ConfirmedBlobCount, __VA_ARGS__)\ // BLOCKSTORE_CACHED_COUNTERS //////////////////////////////////////////////////////////////////////////////// @@ -101,6 +103,8 @@ void TVolumeActor::HandleDiskRegistryBasedPartCounters( const TEvVolume::TEvDiskRegistryBasedPartitionCounters::TPtr& ev, const TActorContext& ctx) { + Y_DEBUG_ABORT_UNLESS(State->IsDiskRegistryMediaKind()); + auto* msg = ev->Get(); if (auto* resourceMetrics = GetResourceMetrics(); resourceMetrics) { @@ -127,10 +131,9 @@ void TVolumeActor::HandleDiskRegistryBasedPartCounters( msg->CallContext ); - const bool doesPartitionBelongToDisk = - State->GetDiskRegistryBasedPartitionActor() == ev->Sender || - State->GetDiskId() == msg->DiskId; - if (!doesPartitionBelongToDisk) { + auto* statInfo = State->GetPartitionStatByDiskId(msg->DiskId); + + if (!statInfo) { LOG_INFO( ctx, TBlockStoreComponents::VOLUME, @@ -141,34 +144,33 @@ void TVolumeActor::HandleDiskRegistryBasedPartCounters( return; } - auto& info = State->GetPartitionStatInfos().front(); - if (!info.LastCounters) { - info.LastCounters = CreatePartitionDiskCounters(CountersPolicy); + if (!statInfo->LastCounters) { + statInfo->LastCounters = + CreatePartitionDiskCounters(State->CountersPolicy()); } - info.LastCounters->Add(*msg->DiskCounters); + statInfo->LastCounters->Add(*msg->DiskCounters); - UpdateCachedStats(*msg->DiskCounters, info.CachedCounters); - CopyPartCountersToCachedStats(*msg->DiskCounters, info.CachedCountersProto); + UpdateCachedStats(*msg->DiskCounters, statInfo->CachedCounters); + CopyPartCountersToCachedStats( + *msg->DiskCounters, + statInfo->CachedCountersProto); TVolumeDatabase::TPartStats partStats; - partStats.Stats = info.CachedCountersProto; - - auto kind = State->GetConfig().GetStorageMediaKind(); - Y_DEBUG_ABORT_UNLESS(IsDiskRegistryMediaKind(kind)); + partStats.Stats = statInfo->CachedCountersProto; ExecuteTx( ctx, std::move(requestInfo), - std::move(partStats), - false - ); + std::move(partStats)); } void TVolumeActor::HandlePartCounters( const TEvStatsService::TEvVolumePartCounters::TPtr& ev, const TActorContext& ctx) { + Y_DEBUG_ABORT_UNLESS(!State->IsDiskRegistryMediaKind()); + auto* msg = ev->Get(); auto requestInfo = CreateRequestInfo( @@ -177,42 +179,44 @@ void TVolumeActor::HandlePartCounters( msg->CallContext ); - ui32 index = 0; - if (!State->FindPartitionStatInfoByOwner(ev->Sender, index)) { - LOG_INFO(ctx, TBlockStoreComponents::VOLUME, + auto tabletId = State->FindPartitionTabletId(ev->Sender); + auto* statInfo = + tabletId ? State->GetPartitionStatInfoByTabletId(*tabletId) : nullptr; + if (!statInfo) { + LOG_INFO( + ctx, + TBlockStoreComponents::VOLUME, "Partition %s for disk %s counters not found", ToString(ev->Sender).c_str(), State->GetDiskId().Quote().c_str()); return; } - auto& info = State->GetPartitionStatInfos()[index]; - if (!info.LastCounters) { - info.LastCounters = CreatePartitionDiskCounters(CountersPolicy); - info.LastMetrics = std::move(msg->BlobLoadMetrics); + if (!statInfo->LastCounters) { + statInfo->LastCounters = + CreatePartitionDiskCounters(State->CountersPolicy()); + statInfo->LastMetrics = std::move(msg->BlobLoadMetrics); } - info.LastSystemCpu += msg->VolumeSystemCpu; - info.LastUserCpu += msg->VolumeUserCpu; + statInfo->LastSystemCpu += msg->VolumeSystemCpu; + statInfo->LastUserCpu += msg->VolumeUserCpu; - info.LastCounters->Add(*msg->DiskCounters); + statInfo->LastCounters->Add(*msg->DiskCounters); - UpdateCachedStats(*msg->DiskCounters, info.CachedCounters); - CopyPartCountersToCachedStats(*msg->DiskCounters, info.CachedCountersProto); + UpdateCachedStats(*msg->DiskCounters, statInfo->CachedCounters); + CopyPartCountersToCachedStats( + *msg->DiskCounters, + statInfo->CachedCountersProto); TVolumeDatabase::TPartStats partStats; - auto kind = State->GetConfig().GetStorageMediaKind(); - Y_DEBUG_ABORT_UNLESS(!IsDiskRegistryMediaKind(kind)); - partStats.Id = State->GetPartitions()[index].TabletId; - partStats.Stats = info.CachedCountersProto; + partStats.TabletId = statInfo->TabletId; + partStats.Stats = statInfo->CachedCountersProto; ExecuteTx( ctx, std::move(requestInfo), - std::move(partStats), - true - ); + std::move(partStats)); } //////////////////////////////////////////////////////////////////////////////// @@ -237,10 +241,11 @@ void TVolumeActor::ExecuteSavePartStats( Y_UNUSED(ctx); TVolumeDatabase db(tx.DB); - if (args.IsReplicatedVolume) { - db.WritePartStats(args.PartStats.Id, args.PartStats.Stats); + if (State->IsDiskRegistryMediaKind()) { + db.WriteNonReplPartStats(args.PartStats.TabletId, args.PartStats.Stats); } else { - db.WriteNonReplPartStats(args.PartStats.Id, args.PartStats.Stats); + Y_DEBUG_ABORT_UNLESS(args.PartStats.TabletId); + db.WritePartStats(args.PartStats.TabletId, args.PartStats.Stats); } } @@ -251,7 +256,7 @@ void TVolumeActor::CompleteSavePartStats( LOG_DEBUG(ctx, TBlockStoreComponents::VOLUME, "[%lu] Part %lu stats saved", TabletID(), - args.PartStats.Id); + args.PartStats.TabletId); NCloud::Send( ctx, @@ -264,7 +269,22 @@ void TVolumeActor::CompleteSavePartStats( void TVolumeActor::SendPartStatsToService(const TActorContext& ctx) { - auto stats = CreatePartitionDiskCounters(CountersPolicy); + DoSendPartStatsToService(ctx, State->GetConfig().GetDiskId()); + + for (const auto& [checkpointId, checkpointInfo]: + State->GetCheckpointStore().GetActiveCheckpoints()) + { + if (checkpointInfo.ShadowDiskId) { + DoSendPartStatsToService(ctx, checkpointInfo.ShadowDiskId); + } + } +} + +void TVolumeActor::DoSendPartStatsToService( + const NActors::TActorContext& ctx, + const TString& diskId) +{ + auto stats = CreatePartitionDiskCounters(State->CountersPolicy()); ui64 systemCpu = 0; ui64 userCpu = 0; // XXX - we need to "manually" calculate total channel history @@ -276,8 +296,12 @@ void TVolumeActor::SendPartStatsToService(const TActorContext& ctx) NBlobMetrics::TBlobLoadMetrics offsetPartitionMetrics; + bool partStatFound = false; for (auto& info: State->GetPartitionStatInfos()) { + if (info.DiskId != diskId) { + continue; + } if (!info.LastCounters) { stats->AggregateWith(info.CachedCounters); channelsHistorySize += @@ -294,8 +318,12 @@ void TVolumeActor::SendPartStatsToService(const TActorContext& ctx) info.LastCounters = nullptr; info.LastSystemCpu = 0; info.LastUserCpu = 0; + partStatFound = true; } + if (!partStatFound) { + return; + } stats->Simple.ChannelHistorySize.Set(channelsHistorySize); auto blobLoadMetrics = NBlobMetrics::MakeBlobLoadMetrics( @@ -307,7 +335,7 @@ void TVolumeActor::SendPartStatsToService(const TActorContext& ctx) auto request = std::make_unique( MakeIntrusive(), - State->GetConfig().GetDiskId(), + diskId, std::move(stats), systemCpu, userCpu, @@ -382,7 +410,7 @@ void TVolumeActor::SendSelfStatsToService(const TActorContext& ctx) State->GetMeta().GetMigrations().size() == 0); SendVolumeSelfCounters(ctx); - VolumeSelfCounters = CreateVolumeSelfCounters(CountersPolicy); + VolumeSelfCounters = CreateVolumeSelfCounters(State->CountersPolicy()); } void TVolumeActor::HandleGetVolumeLoadInfo( diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_statvolume.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_statvolume.cpp index 4324371338d..f5a3739595d 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_statvolume.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_statvolume.cpp @@ -98,6 +98,9 @@ void Merge(const NProto::TVolumeStats& source, NProto::TVolumeStats& target) MERGE_FIELD_MAX(CompactionDelay); MERGE_FIELD(CleanupQueueBytes); + + MERGE_FIELD(UnconfirmedBlobCount); + MERGE_FIELD(ConfirmedBlobCount); } //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_throttling.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_throttling.cpp index 5c20a8668ac..e56b5795c8c 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_throttling.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_throttling.cpp @@ -133,16 +133,20 @@ void TVolumeActor::HandleBackpressureReport( const NPartition::TEvPartition::TEvBackpressureReport::TPtr& ev, const TActorContext& ctx) { - ui32 index = State->GetPartitions().size(); - if (!State->FindPartitionStatInfoByOwner(ev->Sender, index)) { - LOG_WARN(ctx, TBlockStoreComponents::VOLUME, + auto index = State->FindPartitionIndex(ev->Sender); + if (!index) { + LOG_WARN( + ctx, + TBlockStoreComponents::VOLUME, "Partition %s for disk %s backpressure report not found", ToString(ev->Sender).c_str(), State->GetDiskId().Quote().c_str()); + + index = State->GetPartitions().size(); } auto& policy = State->AccessThrottlingPolicy(); - policy.OnBackpressureReport(ctx.Now(), *ev->Get(), index); + policy.OnBackpressureReport(ctx.Now(), *ev->Get(), *index); } void TVolumeActor::HandleWakeup( diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_updateconfig.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_updateconfig.cpp index 547edc6dd42..2a9242d13b4 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_updateconfig.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_updateconfig.cpp @@ -58,6 +58,7 @@ auto BuildNewMeta( partitionConfig.SetMaxBlocksInBlob(volumeConfig.GetMaxBlocksInBlob()); partitionConfig.SetZoneBlockCount(volumeConfig.GetZoneBlockCount()); partitionConfig.SetStorageMediaKind(mediaKind); + partitionConfig.SetIsSystem(volumeConfig.GetIsSystem()); while (partitionConfig.ExplicitChannelProfilesSize() < volumeConfig.ExplicitChannelProfilesSize()) { @@ -342,6 +343,7 @@ void TVolumeActor::CompleteUpdateConfig( )); ResetThrottlingPolicy(); + RegisterCounters(ctx); RegisterVolume(ctx); } diff --git a/cloud/blockstore/libs/storage/volume/volume_database.h b/cloud/blockstore/libs/storage/volume/volume_database.h index c38b4cd3b24..8c4a0bedd2f 100644 --- a/cloud/blockstore/libs/storage/volume/volume_database.h +++ b/cloud/blockstore/libs/storage/volume/volume_database.h @@ -87,7 +87,9 @@ class TVolumeDatabase struct TPartStats { - ui64 Id = 0; + // For DiskRegistry-based always 0. + // For BlobStorage-based TabletId used. + ui64 TabletId = 0; NProto::TCachedPartStats Stats; }; diff --git a/cloud/blockstore/libs/storage/volume/volume_database_ut.cpp b/cloud/blockstore/libs/storage/volume/volume_database_ut.cpp index 06a1879d136..4f713a461c2 100644 --- a/cloud/blockstore/libs/storage/volume/volume_database_ut.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_database_ut.cpp @@ -503,7 +503,7 @@ Y_UNIT_TEST_SUITE(TVolumeDatabaseTest) UNIT_ASSERT_VALUES_EQUAL(2, stats.size()); for (ui32 i = 0; i < 2; ++i) { - UNIT_ASSERT_VALUES_EQUAL(i, stats[i].Id); + UNIT_ASSERT_VALUES_EQUAL(i, stats[i].TabletId); const auto& stat = stats[i].Stats; UNIT_ASSERT_VALUES_EQUAL(1 * i, stat.GetMixedBytesCount()); UNIT_ASSERT_VALUES_EQUAL(2 * i, stat.GetMergedBytesCount()); @@ -552,7 +552,7 @@ Y_UNIT_TEST_SUITE(TVolumeDatabaseTest) UNIT_ASSERT_VALUES_EQUAL(2, stats.size()); for (ui32 i = 0; i < 2; ++i) { - UNIT_ASSERT_VALUES_EQUAL(i, stats[i].Id); + UNIT_ASSERT_VALUES_EQUAL(i, stats[i].TabletId); const auto& stat = stats[i].Stats; UNIT_ASSERT_VALUES_EQUAL(1 * i, stat.GetMixedBytesCount()); UNIT_ASSERT_VALUES_EQUAL(2 * i, stat.GetMergedBytesCount()); diff --git a/cloud/blockstore/libs/storage/volume/volume_state.cpp b/cloud/blockstore/libs/storage/volume/volume_state.cpp index 92d79fb680e..f5e85ffa8a2 100644 --- a/cloud/blockstore/libs/storage/volume/volume_state.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_state.cpp @@ -189,10 +189,9 @@ void TVolumeState::Reset() UseRdmaForThisVolume = false; AcceptInvalidDiskAllocationResponse = false; - if (IsDiskRegistryMediaKind(Config->GetStorageMediaKind())) { + if (IsDiskRegistryMediaKind()) { if (Meta.GetDevices().size()) { - PartitionStatInfos.emplace_back( - EPublishingPolicy::DiskRegistryBased); + CreatePartitionStatInfo(GetDiskId(), 0); } const auto& encryptionDesc = Meta.GetVolumeConfig().GetEncryptionDesc(); if (encryptionDesc.GetMode() != NProto::NO_ENCRYPTION) { @@ -205,7 +204,7 @@ void TVolumeState::Reset() Meta.GetConfig(), StorageConfig->GetTabletRebootCoolDownIncrement(), StorageConfig->GetTabletRebootCoolDownMax()); - PartitionStatInfos.emplace_back(EPublishingPolicy::Repl); + CreatePartitionStatInfo(GetDiskId(), tabletId); } } @@ -295,6 +294,11 @@ void TVolumeState::FillDeviceInfo(NProto::TVolume& volume) const volume); } +bool TVolumeState::IsDiskRegistryMediaKind() const +{ + return NCloud::IsDiskRegistryMediaKind(Config->GetStorageMediaKind()); +} + //////////////////////////////////////////////////////////////////////////////// TPartitionInfo* TVolumeState::GetPartition(ui64 tabletId) @@ -307,15 +311,26 @@ TPartitionInfo* TVolumeState::GetPartition(ui64 tabletId) return nullptr; } -bool TVolumeState::FindPartitionIndex(ui64 tabletId, ui32& index) const +std::optional +TVolumeState::FindPartitionIndex(NActors::TActorId owner) const { for (ui32 i = 0; i < Partitions.size(); ++i) { - if (Partitions[i].TabletId == tabletId) { - index = i; - return true; + if (Partitions[i].Owner == owner) { + return i; } } - return false; + return std::nullopt; +} + +std::optional +TVolumeState::FindPartitionTabletId(NActors::TActorId owner) const +{ + for (const auto& partition: Partitions) { + if (partition.Owner == owner) { + return partition.TabletId; + } + } + return std::nullopt; } void TVolumeState::SetPartitionsState(TPartitionInfo::EState state) @@ -327,7 +342,7 @@ void TVolumeState::SetPartitionsState(TPartitionInfo::EState state) TPartitionInfo::EState TVolumeState::UpdatePartitionsState() { - if (IsDiskRegistryMediaKind(Config->GetStorageMediaKind())) { + if (IsDiskRegistryMediaKind()) { ui64 bytes = 0; for (const auto& device: Meta.GetDevices()) { bytes += device.GetBlocksCount() * device.GetBlockSize(); @@ -397,6 +412,14 @@ TString TVolumeState::GetPartitionsError() const return out.Str(); } +void TVolumeState::SetDiskRegistryBasedPartitionActor( + const NActors::TActorId& actor, + TNonreplicatedPartitionConfigPtr config) +{ + DiskRegistryBasedPartitionActor = actor; + NonreplicatedPartitionConfig = std::move(config); +} + //////////////////////////////////////////////////////////////////////////////// TVolumeState::TAddClientResult TVolumeState::AddClient( @@ -809,43 +832,54 @@ void TVolumeState::CleanupHistoryIfNeeded(TInstant oldest) //////////////////////////////////////////////////////////////////////////////// -bool TVolumeState::FindPartitionStatInfoByOwner( - const TActorId& actorId, - ui32& index) const +EPublishingPolicy TVolumeState::CountersPolicy() const { - for (ui32 i = 0; i < PartitionStatInfos.size(); ++i) { - if (PartitionStatInfos[i].Owner == actorId) { - index = i; - return true; - } - } - return false; + return IsDiskRegistryMediaKind() ? EPublishingPolicy::DiskRegistryBased + : EPublishingPolicy::Repl; } -TPartitionStatInfo* TVolumeState::GetPartitionStatInfoById(ui64 id) +TPartitionStatInfo& TVolumeState::CreatePartitionStatInfo( + const TString& diskId, + ui64 tabletId) { - if (IsDiskRegistryMediaKind(Config->GetStorageMediaKind())) { - if (id >= PartitionStatInfos.size()) { - return nullptr; + PartitionStatInfos.push_back( + TPartitionStatInfo(diskId, tabletId, CountersPolicy())); + return PartitionStatInfos.back(); +} + +TPartitionStatInfo* TVolumeState::GetPartitionStatInfoByTabletId(ui64 tabletId) +{ + if (IsDiskRegistryMediaKind()) { + Y_DEBUG_ABORT_UNLESS(tabletId == 0); + return &PartitionStatInfos.front(); + } + + for (auto& statInfo: PartitionStatInfos) { + if (statInfo.TabletId == tabletId) { + return &statInfo; } - return &PartitionStatInfos[id]; - } else { - ui32 index = 0; - if (FindPartitionIndex(id, index)) { - return &PartitionStatInfos[index]; - }; - return nullptr; } + return nullptr; } -bool TVolumeState::SetPartitionStatActor(ui64 id, const TActorId& actor) +TPartitionStatInfo* +TVolumeState::GetPartitionStatByDiskId(const TString& diskId) { - ui32 index = 0; - if (FindPartitionIndex(id, index)) { - PartitionStatInfos[index].Owner = actor; - return true; - }; - return false; + for (auto& statInfo: PartitionStatInfos) { + if (statInfo.DiskId == diskId) { + return &statInfo; + } + } + + for (const auto& [checkpointId, checkpointInfo]: + GetCheckpointStore().GetActiveCheckpoints()) + { + if (checkpointInfo.ShadowDiskId == diskId) { + return &CreatePartitionStatInfo(diskId, 0); + } + } + + return nullptr; } bool TVolumeState::GetMuteIOErrors() const diff --git a/cloud/blockstore/libs/storage/volume/volume_state.h b/cloud/blockstore/libs/storage/volume/volume_state.h index 13b5d326ecc..a4d9ed11839 100644 --- a/cloud/blockstore/libs/storage/volume/volume_state.h +++ b/cloud/blockstore/libs/storage/volume/volume_state.h @@ -79,7 +79,9 @@ struct THistoryLogItem struct TPartitionStatInfo { - NActors::TActorId Owner = {}; + const TString DiskId; + const ui64 TabletId; + TPartitionDiskCountersPtr LastCounters = {}; TPartitionDiskCounters CachedCounters; NProto::TCachedPartStats CachedCountersProto; @@ -87,8 +89,13 @@ struct TPartitionStatInfo ui64 LastUserCpu = 0; NBlobMetrics::TBlobLoadMetrics LastMetrics; - TPartitionStatInfo(EPublishingPolicy countersPolicy) - : CachedCounters(countersPolicy) + TPartitionStatInfo( + const TString& diskId, + const ui64 tabletId, + EPublishingPolicy countersPolicy) + : DiskId(diskId) + , TabletId(tabletId) + , CachedCounters(countersPolicy) {} }; @@ -257,6 +264,8 @@ class TVolumeState void FillDeviceInfo(NProto::TVolume& volume) const; + bool IsDiskRegistryMediaKind() const; + // // Partitions // @@ -267,7 +276,8 @@ class TVolumeState } TPartitionInfo* GetPartition(ui64 tabletId); - bool FindPartitionIndex(ui64 tabletId, ui32& index) const; + std::optional FindPartitionIndex(NActors::TActorId owner) const; + std::optional FindPartitionTabletId(NActors::TActorId owner) const; // // State @@ -300,12 +310,7 @@ class TVolumeState void SetDiskRegistryBasedPartitionActor( const NActors::TActorId& actor, - TNonreplicatedPartitionConfigPtr config) - { - PartitionStatInfos[0].Owner = actor; - DiskRegistryBasedPartitionActor = actor; - NonreplicatedPartitionConfig = std::move(config); - } + TNonreplicatedPartitionConfigPtr config); const NActors::TActorId& GetDiskRegistryBasedPartitionActor() const { @@ -317,18 +322,25 @@ class TVolumeState return NonreplicatedPartitionConfig; } - bool FindPartitionStatInfoByOwner(const NActors::TActorId& actorId, ui32& index) const; + // + // PartitionStat + // + + EPublishingPolicy CountersPolicy() const; + + TPartitionStatInfo& + CreatePartitionStatInfo(const TString& diskId, ui64 tabletId); - TPartitionStatInfo* GetPartitionStatInfoById(ui64 id); + TPartitionStatInfo* GetPartitionStatInfoByTabletId(ui64 tabletId); - bool SetPartitionStatActor(ui64 id, const NActors::TActorId& actor); + TPartitionStatInfo* GetPartitionStatByDiskId(const TString& diskId); - const TVector& GetPartitionStatInfos() const + std::span GetPartitionStatInfos() const { return PartitionStatInfos; } - TVector& GetPartitionStatInfos() + std::span GetPartitionStatInfos() { return PartitionStatInfos; } diff --git a/cloud/blockstore/libs/storage/volume/volume_tx.h b/cloud/blockstore/libs/storage/volume/volume_tx.h index e4a64409217..5f7a9f7b15c 100644 --- a/cloud/blockstore/libs/storage/volume/volume_tx.h +++ b/cloud/blockstore/libs/storage/volume/volume_tx.h @@ -373,15 +373,12 @@ struct TTxVolume { const TRequestInfoPtr RequestInfo; const TVolumeDatabase::TPartStats PartStats; - const bool IsReplicatedVolume; TSavePartStats( TRequestInfoPtr requestInfo, - TVolumeDatabase::TPartStats partStats, - bool isReplicatedVolume) + TVolumeDatabase::TPartStats partStats) : RequestInfo(std::move(requestInfo)) , PartStats(std::move(partStats)) - , IsReplicatedVolume(isReplicatedVolume) {} void Clear() diff --git a/cloud/blockstore/libs/storage/volume/volume_ut.cpp b/cloud/blockstore/libs/storage/volume/volume_ut.cpp index 92c1395384f..b127d22514d 100644 --- a/cloud/blockstore/libs/storage/volume/volume_ut.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_ut.cpp @@ -3632,6 +3632,115 @@ Y_UNIT_TEST_SUITE(TVolumeTest) } } + Y_UNIT_TEST(ShouldHandleDescribeBlocksRequestForMultipartitionVolume) + { + auto runtime = PrepareTestActorRuntime(); + + TVolumeClient volume(*runtime); + volume.UpdateVolumeConfig( + 0, // maxBandwidth + 0, // maxIops + 0, // burstPercentage + 0, // maxPostponedWeight + false, // throttlingEnabled + 1, // version + NCloud::NProto::EStorageMediaKind::STORAGE_MEDIA_HDD, + 8192, // block count per partition + "vol0", // diskId + "cloud", // cloudId + "folder", // folderId + 2, // partitions count + 2 // blocksPerStripe + ); + volume.WaitReady(); + + auto clientInfo = CreateVolumeClientInfo( + NProto::VOLUME_ACCESS_READ_WRITE, + NProto::VOLUME_MOUNT_LOCAL, + 0); + + volume.AddClient(clientInfo); + + auto range = TBlockRange64::WithLength(1, 8192); + volume.WriteBlocks(range, clientInfo.GetClientId(), 'X'); + + auto request = volume.CreateDescribeBlocksRequest( + range, + clientInfo.GetClientId() + ); + + volume.SendToPipe(std::move(request)); + const auto response1 = volume.RecvResponse(); + const auto& message1 = response1->Record; + + UNIT_ASSERT(SUCCEEDED(response1->GetStatus())); + UNIT_ASSERT_VALUES_EQUAL(0, message1.FreshBlockRangesSize()); + UNIT_ASSERT_VALUES_EQUAL(8, message1.BlobPiecesSize()); + + const auto& blobPiece1 = message1.GetBlobPieces(0); + UNIT_ASSERT_VALUES_EQUAL(513, blobPiece1.RangesSize()); + const auto& range1 = blobPiece1.GetRanges(0); + UNIT_ASSERT_VALUES_EQUAL(0, range1.GetBlobOffset()); + UNIT_ASSERT_VALUES_EQUAL(1, range1.GetBlockIndex()); + UNIT_ASSERT_VALUES_EQUAL(1, range1.GetBlocksCount()); + + const auto& range2 = blobPiece1.GetRanges(1); + UNIT_ASSERT_VALUES_EQUAL(1, range2.GetBlobOffset()); + UNIT_ASSERT_VALUES_EQUAL(4, range2.GetBlockIndex()); + UNIT_ASSERT_VALUES_EQUAL(2, range2.GetBlocksCount()); + + range = TBlockRange64::WithLength(9000, 256); + volume.WriteBlocks(range, clientInfo.GetClientId(), 'Y'); + + request = volume.CreateDescribeBlocksRequest( + range, + clientInfo.GetClientId() + ); + + volume.SendToPipe(std::move(request)); + const auto response2 = volume.RecvResponse(); + const auto& message2 = response2->Record; + + UNIT_ASSERT(SUCCEEDED(response2->GetStatus())); + UNIT_ASSERT_VALUES_EQUAL(256, message2.FreshBlockRangesSize()); + UNIT_ASSERT_VALUES_EQUAL(0, message2.BlobPiecesSize()); + + const auto& freshBlockRange1 = message2.GetFreshBlockRanges(0); + UNIT_ASSERT_VALUES_EQUAL(9000, freshBlockRange1.GetStartIndex()); + UNIT_ASSERT_VALUES_EQUAL(1, freshBlockRange1.GetBlocksCount()); + + TString actualContent; + for (size_t i = 0; i < message2.FreshBlockRangesSize(); ++i) { + const auto& freshRange = message2.GetFreshBlockRanges(i); + actualContent += freshRange.GetBlocksContent(); + } + + UNIT_ASSERT_VALUES_EQUAL(range.Size() * DefaultBlockSize, actualContent.size()); + for (size_t i = 0; i < actualContent.size(); i++) { + UNIT_ASSERT_VALUES_EQUAL('Y', actualContent[i]); + } + + range = TBlockRange64::WithLength(10000, 1); + volume.WriteBlocks(range, clientInfo.GetClientId(), 'Z'); + + request = volume.CreateDescribeBlocksRequest( + range, + clientInfo.GetClientId() + ); + + volume.SendToPipe(std::move(request)); + const auto response3 = volume.RecvResponse(); + const auto& message3 = response3->Record; + + UNIT_ASSERT(SUCCEEDED(response3->GetStatus())); + UNIT_ASSERT_VALUES_EQUAL(1, message3.FreshBlockRangesSize()); + UNIT_ASSERT_VALUES_EQUAL(0, message3.BlobPiecesSize()); + + const auto& freshBlockRange2 = message3.GetFreshBlockRanges(0); + UNIT_ASSERT_VALUES_EQUAL(10000, freshBlockRange2.GetStartIndex()); + UNIT_ASSERT_VALUES_EQUAL(1, freshBlockRange2.GetBlocksCount()); + } + Y_UNIT_TEST(ShouldHandleGetUsedBlocksRequest) { auto runtime = PrepareTestActorRuntime(); diff --git a/cloud/blockstore/libs/storage/volume/volume_ut_checkpoint.cpp b/cloud/blockstore/libs/storage/volume/volume_ut_checkpoint.cpp index 71fc7f205e5..c83973e8be4 100644 --- a/cloud/blockstore/libs/storage/volume/volume_ut_checkpoint.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_ut_checkpoint.cpp @@ -1650,7 +1650,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) {{"action","createCheckpoint"}, {"checkpointid","1"}}), HTTP_METHOD::HTTP_METHOD_POST); - UNIT_ASSERT_C(httpResponse->Html.Contains("Tablet is dead"), true); + UNIT_ASSERT_C(httpResponse->Html.Contains("tablet is shutting down"), true); } Y_UNIT_TEST(ShouldFailHttpGetCreateCheckpoint) @@ -1884,6 +1884,74 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) } } + Y_UNIT_TEST(ShouldNotReadFromCheckpointWithoutData) + { + NProto::TStorageServiceConfig config; + auto runtime = PrepareTestActorRuntime(config); + + TVolumeClient volume(*runtime); + volume.UpdateVolumeConfig(); + + volume.WaitReady(); + + auto clientInfo = CreateVolumeClientInfo( + NProto::VOLUME_ACCESS_READ_WRITE, + NProto::VOLUME_MOUNT_LOCAL, + 0); + volume.AddClient(clientInfo); + + volume.WriteBlocks( + TBlockRange64::WithLength(0, 1024), + clientInfo.GetClientId(), + 1); + volume.CreateCheckpoint("c1", NProto::ECheckpointType::WITHOUT_DATA); + volume.WriteBlocks( + TBlockRange64::WithLength(0, 1024), + clientInfo.GetClientId(), + 2); + + { + // Read from checkpoint without data failed + volume.SendReadBlocksRequest( + GetBlockRangeById(0), + clientInfo.GetClientId(), + "c1"); + + auto response = volume.RecvReadBlocksResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_NOT_FOUND, response->GetStatus()); + UNIT_ASSERT_VALUES_EQUAL( + "checkpoint not found: \"c1\"", + response->GetError().GetMessage()); + UNIT_ASSERT(HasProtoFlag( + response->GetError().GetFlags(), + NProto::EF_SILENT)); + } + + volume.CreateCheckpoint("c2"); + volume.DeleteCheckpointData("c2"); + volume.WriteBlocks( + TBlockRange64::WithLength(0, 1024), + clientInfo.GetClientId(), + 2); + + { + // Read from checkpoint without data failed + volume.SendReadBlocksRequest( + GetBlockRangeById(0), + clientInfo.GetClientId(), + "c2"); + + auto response = volume.RecvReadBlocksResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_NOT_FOUND, response->GetStatus()); + UNIT_ASSERT_VALUES_EQUAL( + "checkpoint not found: \"c2\"", + response->GetError().GetMessage()); + UNIT_ASSERT(HasProtoFlag( + response->GetError().GetFlags(), + NProto::EF_SILENT)); + } + } + Y_UNIT_TEST(ShouldNotRejectRZRequestsDuringSinglePartionCheckpointCreation) { NProto::TStorageServiceConfig config; @@ -2031,7 +2099,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) 0); volume.AddClient(clientInfo); - volume.CreateCheckpoint("c1", true); + volume.CreateCheckpoint("c1", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2063,7 +2131,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) TBlockRange64::MakeClosedInterval(1022, 1023), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c2", true); + volume.CreateCheckpoint("c2", NProto::ECheckpointType::LIGHT); auto popCountStr = [](const TString& s) { ui64 count = 0; @@ -2146,7 +2214,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) TBlockRange64::MakeOneBlock(0), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c1", true); + volume.CreateCheckpoint("c1", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2171,7 +2239,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) } // checkpoint creation should be idempotent - volume.CreateCheckpoint("c1", true); + volume.CreateCheckpoint("c1", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2186,7 +2254,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) volume.WriteBlocks(TBlockRange64::MakeOneBlock(1), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c2", true); + volume.CreateCheckpoint("c2", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2200,7 +2268,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) } // checkpoint creation should be idempotent - volume.CreateCheckpoint("c2", true); + volume.CreateCheckpoint("c2", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2213,7 +2281,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) UNIT_ASSERT_VALUES_EQUAL(0b00000010, ui8(mask[0])); } - volume.CreateCheckpoint("c3", true); + volume.CreateCheckpoint("c3", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2243,7 +2311,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) UNIT_ASSERT_VALUES_EQUAL(0b00000000, ui8(mask[0])); } - volume.CreateCheckpoint("c4", true); + volume.CreateCheckpoint("c4", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2300,7 +2368,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) 0); volume.AddClient(clientInfo); - volume.CreateCheckpoint("c1", true); + volume.CreateCheckpoint("c1", NProto::ECheckpointType::LIGHT); volume.WriteBlocks(TBlockRange64::MakeOneBlock(0), clientInfo.GetClientId(), 2); @@ -2317,7 +2385,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) UNIT_ASSERT_VALUES_EQUAL(0b11111111, ui8(mask[0])); } - volume.CreateCheckpoint("c2", true); + volume.CreateCheckpoint("c2", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2331,7 +2399,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) } volume.WriteBlocks(TBlockRange64::MakeOneBlock(1), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c3", true); + volume.CreateCheckpoint("c3", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2385,20 +2453,20 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) 0); volume.AddClient(clientInfo); - volume.CreateCheckpoint("c1", true); + volume.CreateCheckpoint("c1", NProto::ECheckpointType::LIGHT); volume.WriteBlocks( TBlockRange64::MakeOneBlock(0), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c2", true); + volume.CreateCheckpoint("c2", NProto::ECheckpointType::LIGHT); volume.WriteBlocks(TBlockRange64::MakeOneBlock(1), clientInfo.GetClientId(), 2); volume.RebootTablet(); volume.WriteBlocks(TBlockRange64::MakeOneBlock(2), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c3", true); + volume.CreateCheckpoint("c3", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2411,7 +2479,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) UNIT_ASSERT_VALUES_EQUAL(0b11111111, ui8(mask[0])); } - volume.CreateCheckpoint("c4", true); + volume.CreateCheckpoint("c4", NProto::ECheckpointType::LIGHT); // should not see changes earlier than checkpoint c3 { @@ -2469,12 +2537,12 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) 0); volume.AddClient(clientInfo); - volume.CreateCheckpoint("c1", true); + volume.CreateCheckpoint("c1", NProto::ECheckpointType::LIGHT); volume.WriteBlocks(TBlockRange64::MakeOneBlock(0), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c2", true); + volume.CreateCheckpoint("c2", NProto::ECheckpointType::LIGHT); - volume.CreateCheckpoint("c3", true); + volume.CreateCheckpoint("c3", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2556,12 +2624,12 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) 0); volume.AddClient(clientInfo); - volume.CreateCheckpoint("c1", true); + volume.CreateCheckpoint("c1", NProto::ECheckpointType::LIGHT); volume.WriteBlocks(TBlockRange64::MakeOneBlock(0), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c2", true); + volume.CreateCheckpoint("c2", NProto::ECheckpointType::LIGHT); volume.WriteBlocks(TBlockRange64::MakeOneBlock(1), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c1", true); + volume.CreateCheckpoint("c1", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2574,7 +2642,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) UNIT_ASSERT_VALUES_EQUAL(0b00000001, ui8(mask[0])); } - volume.CreateCheckpoint("c3", true); + volume.CreateCheckpoint("c3", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2599,8 +2667,8 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) volume.RebootTablet(); - volume.CreateCheckpoint("c1", true); - volume.CreateCheckpoint("c4", true); + volume.CreateCheckpoint("c1", NProto::ECheckpointType::LIGHT); + volume.CreateCheckpoint("c4", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2614,9 +2682,9 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) } volume.WriteBlocks(TBlockRange64::MakeOneBlock(2), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c3", true); - volume.CreateCheckpoint("c5", true); - volume.CreateCheckpoint("c2", true); + volume.CreateCheckpoint("c3", NProto::ECheckpointType::LIGHT); + volume.CreateCheckpoint("c5", NProto::ECheckpointType::LIGHT); + volume.CreateCheckpoint("c2", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2666,11 +2734,11 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) 0); volume.AddClient(clientInfo); - volume.CreateCheckpoint("c1", true); + volume.CreateCheckpoint("c1", NProto::ECheckpointType::LIGHT); volume.WriteBlocks(TBlockRange64::MakeOneBlock(0), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c2", true); + volume.CreateCheckpoint("c2", NProto::ECheckpointType::LIGHT); volume.WriteBlocks(TBlockRange64::MakeOneBlock(1), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c3", true); + volume.CreateCheckpoint("c3", NProto::ECheckpointType::LIGHT); UNIT_ASSERT_VALUES_EQUAL( S_OK, @@ -2739,7 +2807,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) } volume.WriteBlocks(TBlockRange64::MakeOneBlock(2), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c4", true); + volume.CreateCheckpoint("c4", NProto::ECheckpointType::LIGHT); { volume.SendGetChangedBlocksRequest( TBlockRange64::MakeClosedInterval(0, 1023), @@ -2750,7 +2818,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) } volume.WriteBlocks(TBlockRange64::MakeOneBlock(3), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c5", true); + volume.CreateCheckpoint("c5", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2802,7 +2870,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) } volume.WriteBlocks(TBlockRange64::MakeOneBlock(4), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c6", true); + volume.CreateCheckpoint("c6", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2816,7 +2884,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) } volume.WriteBlocks(TBlockRange64::MakeOneBlock(5), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c7", true); + volume.CreateCheckpoint("c7", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -2866,10 +2934,10 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) 0); volume.AddClient(clientInfo); - volume.CreateCheckpoint("c1", true); + volume.CreateCheckpoint("c1", NProto::ECheckpointType::LIGHT); volume.WriteBlocks(TBlockRange64::MakeOneBlock(0), clientInfo.GetClientId(), 2); - volume.CreateCheckpoint("c2", false); - volume.CreateCheckpoint("c3", true); + volume.CreateCheckpoint("c2", NProto::ECheckpointType::NORMAL); + volume.CreateCheckpoint("c3", NProto::ECheckpointType::LIGHT); { volume.SendGetChangedBlocksRequest( @@ -2953,7 +3021,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) UNIT_ASSERT_VALUES_UNEQUAL(S_OK, volume.RecvGetChangedBlocksResponse()->GetStatus()); } - volume.CreateCheckpoint("c1", true); + volume.CreateCheckpoint("c1", NProto::ECheckpointType::LIGHT); volume.WriteBlocks(TBlockRange64::MakeClosedInterval(0, 1), clientInfo.GetClientId(), 1); { @@ -2977,7 +3045,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) UNIT_ASSERT_VALUES_EQUAL(0b11111111, ui8(mask[0])); } - volume.CreateCheckpoint("c2", true); + volume.CreateCheckpoint("c2", NProto::ECheckpointType::LIGHT); { auto response = volume.GetChangedBlocks( @@ -3024,7 +3092,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) UNIT_ASSERT_VALUES_EQUAL(0b11111111, ui8(mask[0])); } - volume.CreateCheckpoint("c3", true); + volume.CreateCheckpoint("c3", NProto::ECheckpointType::LIGHT); volume.WriteBlocks(TBlockRange64::MakeClosedInterval(0, 1), clientInfo.GetClientId(), 1); { @@ -3106,7 +3174,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) volume.AddClient(clientInfo); const TString checkpointId = "c1"; - volume.CreateCheckpoint(checkpointId, true); + volume.CreateCheckpoint(checkpointId, NProto::ECheckpointType::LIGHT); { // Validate checkpoint state (ready). auto status = volume.GetCheckpointStatus("c1"); @@ -3138,7 +3206,8 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) GetBlockContent(42)); } - Y_UNIT_TEST(ShouldCreateCheckpointWithShadowDisk) + void DoShouldCreateCheckpointWithShadowDisk( + NProto::EStorageMediaKind mediaKind) { NProto::TStorageServiceConfig config; config.SetUseShadowDisksForNonreplDiskCheckpoints(true); @@ -3163,8 +3232,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) }; runtime->SetObserverFunc(countAllocateDeallocateDiskRequest); - const auto expectedBlockCount = - DefaultDeviceBlockSize * DefaultDeviceBlockCount / DefaultBlockSize; + const ui64 expectedBlockCount = 32768; TVolumeClient volume(*runtime); volume.UpdateVolumeConfig( @@ -3174,7 +3242,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) 0, false, 1, - NCloud::NProto::STORAGE_MEDIA_SSD_NONREPLICATED, + mediaKind, expectedBlockCount); volume.WaitReady(); @@ -3185,9 +3253,9 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) 0); volume.AddClient(clientInfo); - // Write some data. + // Write all '1' to block 1. volume.WriteBlocks( - TBlockRange64::MakeOneBlock(0), + TBlockRange64::MakeOneBlock(1), clientInfo.GetClientId(), GetBlockContent(1)); @@ -3199,11 +3267,8 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) UNIT_ASSERT_VALUES_EQUAL(1, allocateRequestCount); UNIT_ASSERT_VALUES_EQUAL(0, deallocateRequestCount); - // Writes to the disk are not blocked. - volume.WriteBlocks( - TBlockRange64::MakeOneBlock(0), - clientInfo.GetClientId(), - GetBlockContent(2)); + // Reconnect pipe since partition has restarted. + volume.ReconnectPipe(); { // Validate checkpoint state (not ready). auto status = volume.GetCheckpointStatus("c1"); @@ -3212,53 +3277,133 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) status->Record.GetCheckpointStatus()); } - using EReason = - TEvVolumePrivate::TUpdateShadowDiskStateRequest::EReason; - // Set Checkpoint fill in progress. - volume.UpdateShadowDiskState( - "c1", - EReason::FillProgressUpdate, - 10, - expectedBlockCount); + auto tryWriteBlock = [&](ui64 blockIndx, ui8 content) -> bool + { + auto request = volume.CreateWriteBlocksRequest( + TBlockRange64::MakeOneBlock(blockIndx), + clientInfo.GetClientId(), + GetBlockContent(content)); + volume.SendToPipe(std::move(request)); + auto response = + volume.RecvResponse(); + if (response->GetStatus() != S_OK) { + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + UNIT_ASSERT_VALUES_EQUAL( + "Request WriteBlocks intersects with currently migrated " + "range", + response->GetError().GetMessage()); + return false; + } + return true; + }; - { // Validate checkpoint state (not ready). - auto status = volume.GetCheckpointStatus("c1"); - UNIT_ASSERT_EQUAL( - NProto::ECheckpointStatus::NOT_READY, - status->Record.GetCheckpointStatus()); - } + auto tryReadBlock = + [&](ui64 blockIndx, ui8 content, const TString& checkpoint) -> bool + { + auto request = volume.CreateReadBlocksRequest( + TBlockRange64::MakeOneBlock(blockIndx), + clientInfo.GetClientId(), + checkpoint); + volume.SendToPipe(std::move(request)); + auto response = + volume.RecvResponse(); + if (response->GetStatus() != S_OK) { + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + UNIT_ASSERT_VALUES_EQUAL( + "Can't read from checkpoint \"c1\" while the data is being " + "filled in.", + response->GetError().GetMessage()); + return false; + } + const auto& bufs = response->Record.GetBlocks().GetBuffers(); + UNIT_ASSERT_VALUES_EQUAL(1, bufs.size()); + UNIT_ASSERT_VALUES_EQUAL(GetBlockContent(content), bufs[0]); + return true; + }; - volume.UpdateShadowDiskState( - "c1", - EReason::FillCompleted, - expectedBlockCount, - expectedBlockCount); + // The index of the block that is recorded during the filling of the + // shadow disk. We use it later to check the contents of the disk and + // the checkpoint. + ui64 blockIndxToVerify = 0; + + // Writes to the disk are not blocked. But may overlap with migrating + // blocks. + for (ui64 i = 0;; ++i) { + ui64 blockIndx = i % expectedBlockCount; + ui8 content = i % 256; + + // We are trying to write the block to the disk. It may fail if it + // intersects with the current migrated range. + if (tryWriteBlock(blockIndx, content)) { + // We save the index of the successfully written block with + // non-zero data to check the contents of the disk and the + // checkpoint later. + if (content != 0 && blockIndxToVerify == 0) { + blockIndxToVerify = blockIndx; + } - { // Validate checkpoint state (ready). - auto status = volume.GetCheckpointStatus("c1"); - UNIT_ASSERT_EQUAL( - NProto::ECheckpointStatus::READY, - status->Record.GetCheckpointStatus()); + // Reading from disk should always be successful. + UNIT_ASSERT(tryReadBlock(blockIndx, content, "")); + + // Reading from the checkpoint will be successful when the + // checkpoint preparation is completed. + if (tryReadBlock(0, 0, "c1")) { + // Validate checkpoint state (ready). + auto status = volume.GetCheckpointStatus("c1"); + UNIT_ASSERT_EQUAL( + NProto::ECheckpointStatus::READY, + status->Record.GetCheckpointStatus()); + break; + } + + // Validate checkpoint state (not ready). + auto status = volume.GetCheckpointStatus("c1"); + UNIT_ASSERT_EQUAL( + NProto::ECheckpointStatus::NOT_READY, + status->Record.GetCheckpointStatus()); + } + + // Advance migration. + runtime->DispatchEvents({}, TDuration::MilliSeconds(250)); } - // Writes to the disk are not blocked. - volume.WriteBlocks( - TBlockRange64::MakeOneBlock(0), - clientInfo.GetClientId(), - GetBlockContent(2)); + // Check that the recording to the disk has happened. + UNIT_ASSERT_UNEQUAL(0, blockIndxToVerify); + + // Read block blockIndxToVerify from disk. It should contain valid data. + UNIT_ASSERT( + tryReadBlock(blockIndxToVerify, blockIndxToVerify % 256, "")); - // TODO(drbasic) read from checkpoint (success). + // Write all '0' to block blockIndxToVerify. + UNIT_ASSERT(tryWriteBlock(blockIndxToVerify, 0)); + + // Read block blockIndxToVerify from the disk. It should contain all + // '0'. + UNIT_ASSERT(tryReadBlock(blockIndxToVerify, 0, "")); + + // Read block blockIndxToVerify from checkpoint. It should contain valid + // data. + UNIT_ASSERT( + tryReadBlock(blockIndxToVerify, blockIndxToVerify % 256, "c1")); // Delete checkpoint data. volume.DeleteCheckpointData("c1"); UNIT_ASSERT_VALUES_EQUAL(1, allocateRequestCount); UNIT_ASSERT_VALUES_EQUAL(1, deallocateRequestCount); - { // Validate checkpoint state (error). - auto status = volume.GetCheckpointStatus("c1"); - UNIT_ASSERT_EQUAL( - NProto::ECheckpointStatus::ERROR, - status->Record.GetCheckpointStatus()); + // Reconnect pipe since partition has restarted. + volume.ReconnectPipe(); + + // Read from checkpoint without data should fail. + { + auto request = volume.CreateReadBlocksRequest( + TBlockRange64::MakeOneBlock(0), + clientInfo.GetClientId(), + "c1"); + volume.SendToPipe(std::move(request)); + auto response = + volume.RecvResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_NOT_FOUND, response->GetStatus()); } // Delete checkpoint. @@ -3266,12 +3411,458 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) UNIT_ASSERT_VALUES_EQUAL(1, allocateRequestCount); UNIT_ASSERT_VALUES_EQUAL(2, deallocateRequestCount); - { // Checkpoint state not found for deleted checkpoint. - volume.SendGetCheckpointStatusRequest("c1"); - auto response = volume.RecvGetCheckpointStatusResponse(); - UNIT_ASSERT_VALUES_EQUAL(E_NOT_FOUND, response->GetStatus()); + // Reconnect pipe since partition has restarted. + volume.ReconnectPipe(); + + // Write OK. + UNIT_ASSERT(tryWriteBlock(0, 4)); + + // Read block 0 from the disk. It should contain all '4'. + UNIT_ASSERT(tryReadBlock(0, 4, "")); + } + + Y_UNIT_TEST(ShouldCreateCheckpointWithShadowDiskNonrepl) + { + DoShouldCreateCheckpointWithShadowDisk( + NCloud::NProto::STORAGE_MEDIA_SSD_NONREPLICATED); + } + + Y_UNIT_TEST(ShouldCreateCheckpointWithShadowDiskMirror2) + { + DoShouldCreateCheckpointWithShadowDisk( + NCloud::NProto::STORAGE_MEDIA_SSD_MIRROR2); + } + + Y_UNIT_TEST(ShouldCreateCheckpointWithShadowDiskMirror3) + { + DoShouldCreateCheckpointWithShadowDisk( + NCloud::NProto::STORAGE_MEDIA_SSD_MIRROR3); + } + + Y_UNIT_TEST(ShouldCreateCheckpointWithShadowDiskHddNonrepl) + { + DoShouldCreateCheckpointWithShadowDisk( + NCloud::NProto::STORAGE_MEDIA_HDD_NONREPLICATED); + } + + void ShouldRetryWhenAcquiringShadowDisk(ui32 requestMessage, ui32 responseMessage) + { + NProto::TStorageServiceConfig config; + config.SetUseShadowDisksForNonreplDiskCheckpoints(true); + auto runtime = PrepareTestActorRuntime(config); + + bool simulateNondelivery = true; + bool simulateErrorResponse = true; + auto describeDiskRequestsFilter = [&](TAutoPtr& event) + { + if (event->GetTypeRewrite() == requestMessage && + simulateNondelivery) + { // Simulate non-delivery describe message to DiskRegistry. + auto sendTo = event->Sender; + runtime->Send( + new IEventHandle( + sendTo, + sendTo, + event->ReleaseBase().Release(), + 0, + event->Cookie, + nullptr), + 0); + simulateNondelivery = false; + return TTestActorRuntime::EEventAction::DROP; + } + + if (event->GetTypeRewrite() == responseMessage && + simulateErrorResponse) + { // Simulate response with error from DiskRegistry. + if (event->GetTypeRewrite() == + TEvDiskRegistry::EvDescribeDiskResponse) + { + auto* msg = + event->Get(); + msg->Record.MutableError()->SetCode(E_REJECTED); + } + if (event->GetTypeRewrite() == + TEvDiskRegistry::EvAcquireDiskResponse) + { + auto* msg = + event->Get(); + msg->Record.MutableError()->SetCode(E_REJECTED); + } + simulateErrorResponse = false; + } + return TTestActorRuntime::DefaultObserverFunc(event); + }; + runtime->SetObserverFunc(describeDiskRequestsFilter); + + // Create volume. + TVolumeClient volume(*runtime); + volume.UpdateVolumeConfig( + 0, + 0, + 0, + 0, + false, + 1, + NCloud::NProto::STORAGE_MEDIA_SSD_NONREPLICATED, + 32768); + + volume.WaitReady(); + + auto clientInfo = CreateVolumeClientInfo( + NProto::VOLUME_ACCESS_READ_WRITE, + NProto::VOLUME_MOUNT_LOCAL, + 0); + volume.AddClient(clientInfo); + + // Create checkpoint. + volume.CreateCheckpoint("c1"); + + // Reconnect pipe since partition has restarted. + volume.ReconnectPipe(); + + // Wait for shadow disk become ready. + for (;;) { + // Validate checkpoint state (ready). + auto status = + volume.GetCheckpointStatus("c1")->Record.GetCheckpointStatus(); + + if (status == NProto::ECheckpointStatus::READY) { + break; + } + + if (status == NProto::ECheckpointStatus::ERROR) { + UNIT_ASSERT_C(false, "Got error status for checkpoint"); + } + + // Advance shadow disk fill. + runtime->DispatchEvents({}, TDuration::MilliSeconds(250)); + } + + // Check that interceptions of messages have occurred. + UNIT_ASSERT(!simulateNondelivery); + UNIT_ASSERT(!simulateErrorResponse); + } + + Y_UNIT_TEST(ShouldRetryDescribeShadowDisk) + { + ShouldRetryWhenAcquiringShadowDisk( + TEvDiskRegistry::EvDescribeDiskRequest, + TEvDiskRegistry::EvDescribeDiskResponse); + } + + Y_UNIT_TEST(ShouldRetryAcquireShadowDisk) + { + ShouldRetryWhenAcquiringShadowDisk( + TEvDiskRegistry::EvAcquireDiskRequest, + TEvDiskRegistry::EvAcquireDiskResponse); + } + + Y_UNIT_TEST(ShouldStopAcquiringAfterTimeout) + { + NProto::TStorageServiceConfig config; + config.SetUseShadowDisksForNonreplDiskCheckpoints(true); + config.SetMaxAcquireShadowDiskTotalTimeoutWhenNonBlocked(2000); + + auto runtime = PrepareTestActorRuntime(config); + + auto describeDiskRequestsFilter = [&](TAutoPtr& event) + { + if (event->GetTypeRewrite() == + TEvDiskRegistry::EvDescribeDiskResponse) + { // Simulate response with error from DiskRegistry. + auto* msg = + event->Get(); + msg->Record.MutableError()->SetCode(E_REJECTED); + } + return TTestActorRuntime::DefaultObserverFunc(event); + }; + runtime->SetObserverFunc(describeDiskRequestsFilter); + + // Create volume. + TVolumeClient volume(*runtime); + volume.UpdateVolumeConfig( + 0, + 0, + 0, + 0, + false, + 1, + NCloud::NProto::STORAGE_MEDIA_SSD_NONREPLICATED, + 32768); + + volume.WaitReady(); + + auto clientInfo = CreateVolumeClientInfo( + NProto::VOLUME_ACCESS_READ_WRITE, + NProto::VOLUME_MOUNT_LOCAL, + 0); + volume.AddClient(clientInfo); + + // Create checkpoint. + volume.CreateCheckpoint("c1"); + + // Reconnect pipe since partition has restarted. + volume.ReconnectPipe(); + + // Wait for shadow disk enter to the error state. + for (;;) { + auto status = + volume.GetCheckpointStatus("c1")->Record.GetCheckpointStatus(); + + if (status == NProto::ECheckpointStatus::READY) { + UNIT_ASSERT_C(false, "Got ready status for checkpoint"); + } + + if (status == NProto::ECheckpointStatus::ERROR) { + break; + } + + // Advance time. + runtime->DispatchEvents({}, TDuration::MilliSeconds(250)); + } + } + + Y_UNIT_TEST(ShouldBlockWritesWhenReAcquire) + { + NProto::TStorageServiceConfig config; + config.SetUseShadowDisksForNonreplDiskCheckpoints(true); + config.SetMaxShadowDiskFillBandwidth(1); + auto runtime = PrepareTestActorRuntime(config); + + const ui64 expectedBlockCount = 32768; + + TVolumeClient volume(*runtime); + volume.UpdateVolumeConfig( + 0, + 0, + 0, + 0, + false, + 1, + NCloud::NProto::STORAGE_MEDIA_SSD_NONREPLICATED, + expectedBlockCount); + + volume.WaitReady(); + + auto clientInfo = CreateVolumeClientInfo( + NProto::VOLUME_ACCESS_READ_WRITE, + NProto::VOLUME_MOUNT_LOCAL, + 0); + volume.AddClient(clientInfo); + + auto tryWriteBlock = [&](ui64 blockIndx, ui8 content) -> bool + { + auto request = volume.CreateWriteBlocksRequest( + TBlockRange64::MakeOneBlock(blockIndx), + clientInfo.GetClientId(), + GetBlockContent(content)); + volume.SendToPipe(std::move(request)); + auto response = + volume.RecvResponse(); + if (response->GetStatus() != S_OK) { + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + UNIT_ASSERT_VALUES_EQUAL( + "Can't write to source disk while shadow disk \"vol0c1\" " + "not ready yet.", + response->GetError().GetMessage()); + return false; + } + return true; + }; + + auto tryReadBlock = [&](ui64 blockIndx) -> bool + { + auto request = volume.CreateReadBlocksRequest( + TBlockRange64::MakeOneBlock(blockIndx), + clientInfo.GetClientId()); + volume.SendToPipe(std::move(request)); + auto response = + volume.RecvResponse(); + if (response->GetStatus() != S_OK) { + return false; + } + return true; + }; + + // Create checkpoint. + volume.CreateCheckpoint("c1"); + + // Reconnect pipe since partition has restarted. + volume.ReconnectPipe(); + + // We check that attempts to write to the disk are not blocked. + UNIT_ASSERT(tryWriteBlock(expectedBlockCount - 1, 0)); + + { // Emulate partial disk fill + using EReason = + TEvVolumePrivate::TUpdateShadowDiskStateRequest::EReason; + + auto request = std::make_unique< + TEvVolumePrivate::TEvUpdateShadowDiskStateRequest>( + "c1", + EReason::FillProgressUpdate, + 128, + expectedBlockCount); + + volume.SendToPipe(std::move(request)); + runtime->DispatchEvents({}, TDuration::MilliSeconds(100)); + } + + // Steal the acquire response. + // We will return it later to complete the shadow disk acquiring. + std::unique_ptr stolenAcquireResponse; + auto stealAcquireResponse = [&](TAutoPtr& event) + { + if (event->GetTypeRewrite() == + TEvDiskRegistry::EvAcquireDiskResponse) + { + stolenAcquireResponse.reset(event.Release()); + return TTestActorRuntime::EEventAction::DROP; + } + return TTestActorRuntime::DefaultObserverFunc(event); + }; + auto oldFilter = runtime->SetObserverFunc(stealAcquireResponse); + + // Reboot volume tablet. + NKikimr::RebootTablet(*runtime, TestTabletId, volume.GetSender()); + volume.ReconnectPipe(); + + // Check that the acquire response was stolen. + UNIT_ASSERT(stolenAcquireResponse); + runtime->SetObserverFunc(oldFilter); + + // We check that attempts to write to the disk are blocked. Since the + // shadow disk is partially filled, and the shadow disk acquiring has + // not completed yet. + UNIT_ASSERT(!tryWriteBlock(expectedBlockCount - 1, 0)); + UNIT_ASSERT(tryReadBlock(expectedBlockCount - 1)); + + // Return stolen acquire response. The acquiring should be completed. + runtime->Send(stolenAcquireResponse.release()); + runtime->DispatchEvents({}, TDuration::MilliSeconds(250)); + + // We check that attempts to write to the disk are not blocked. + UNIT_ASSERT(tryWriteBlock(expectedBlockCount - 1, 0)); + UNIT_ASSERT(tryReadBlock(expectedBlockCount - 1)); + } + + Y_UNIT_TEST(ShouldBlockWritesWhenReAcquireForShortTime) + { + NProto::TStorageServiceConfig config; + config.SetUseShadowDisksForNonreplDiskCheckpoints(true); + config.SetMaxShadowDiskFillBandwidth(1); + config.SetMaxAcquireShadowDiskTotalTimeoutWhenBlocked(5000); + auto runtime = PrepareTestActorRuntime(config); + + const ui64 expectedBlockCount = 32768; + + TVolumeClient volume(*runtime); + volume.UpdateVolumeConfig( + 0, + 0, + 0, + 0, + false, + 1, + NCloud::NProto::STORAGE_MEDIA_SSD_NONREPLICATED, + expectedBlockCount); + + volume.WaitReady(); + + auto clientInfo = CreateVolumeClientInfo( + NProto::VOLUME_ACCESS_READ_WRITE, + NProto::VOLUME_MOUNT_LOCAL, + 0); + volume.AddClient(clientInfo); + + auto tryWriteBlock = [&](ui64 blockIndx, ui8 content) -> bool + { + auto request = volume.CreateWriteBlocksRequest( + TBlockRange64::MakeOneBlock(blockIndx), + clientInfo.GetClientId(), + GetBlockContent(content)); + volume.SendToPipe(std::move(request)); + auto response = + volume.RecvResponse(); + if (response->GetStatus() != S_OK) { + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + UNIT_ASSERT_VALUES_EQUAL( + "Can't write to source disk while shadow disk \"vol0c1\" " + "not ready yet.", + response->GetError().GetMessage()); + return false; + } + return true; + }; + + // Create checkpoint. + volume.CreateCheckpoint("c1"); + + // Reconnect pipe since partition has restarted. + volume.ReconnectPipe(); + + // We check that attempts to write to the disk are not blocked. + UNIT_ASSERT(tryWriteBlock(expectedBlockCount - 1, 0)); + + { // Emulate partial disk fill + using EReason = + TEvVolumePrivate::TUpdateShadowDiskStateRequest::EReason; + + auto request = std::make_unique< + TEvVolumePrivate::TEvUpdateShadowDiskStateRequest>( + "c1", + EReason::FillProgressUpdate, + 128, + expectedBlockCount); + + volume.SendToPipe(std::move(request)); + runtime->DispatchEvents({}, TDuration::MilliSeconds(100)); } + + // Steal the acquire response. + std::unique_ptr stolenAcquireResponse; + auto stealAcquireResponse = [&](TAutoPtr& event) + { + if (event->GetTypeRewrite() == + TEvDiskRegistry::EvAcquireDiskResponse) + { + stolenAcquireResponse.reset(event.Release()); + return TTestActorRuntime::EEventAction::DROP; + } + return TTestActorRuntime::DefaultObserverFunc(event); + }; + auto oldFilter = runtime->SetObserverFunc(stealAcquireResponse); + + // Reboot volume tablet. + NKikimr::RebootTablet(*runtime, TestTabletId, volume.GetSender()); + volume.ReconnectPipe(); + + // Check that the acquire response was stolen. + UNIT_ASSERT(stolenAcquireResponse); + runtime->SetObserverFunc(oldFilter); + + // We check that attempts to write to the disk are blocked. Since the + // shadow disk is partially filled, and the shadow disk acquiring has + // not completed yet. + UNIT_ASSERT(!tryWriteBlock(expectedBlockCount - 1, 0)); + + // Wait MaxAcquireShadowDiskTotalTimeoutWhenBlocked timeout. + // After that shadow disk enter to the error state and writes to the + // source disk will be allowed. + runtime->DispatchEvents( + {}, + TDuration::MilliSeconds( + config.GetMaxAcquireShadowDiskTotalTimeoutWhenBlocked())); + + UNIT_ASSERT_EQUAL( + NProto::ECheckpointStatus::ERROR, + volume.GetCheckpointStatus("c1")->Record.GetCheckpointStatus()); + + // We check that attempts to write to the disk are not blocked. + UNIT_ASSERT(tryWriteBlock(expectedBlockCount - 1, 0)); } + } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/volume/volume_ut_stats.cpp b/cloud/blockstore/libs/storage/volume/volume_ut_stats.cpp index bb7a14c6b4b..18202bb8388 100644 --- a/cloud/blockstore/libs/storage/volume/volume_ut_stats.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_ut_stats.cpp @@ -430,6 +430,123 @@ Y_UNIT_TEST_SUITE(TVolumeStatsTest) return nonEmptyReports == 3; }}, TDuration::Seconds(1)); } + + Y_UNIT_TEST(ShouldSendPartitionStatsForShadowDisk) + { + constexpr ui64 DiskBlockCount = 32768; + constexpr ui64 DiskBlockSize = 4096; + constexpr ui64 DiskByteCount = DiskBlockCount * DiskBlockSize; + constexpr ui32 WriteBlockCount = 2; + constexpr ui32 ReadFromSourceBlockCount = 5; + constexpr ui32 ReadFromCheckpointBlockCount = 10; + + NProto::TStorageServiceConfig config; + config.SetUseShadowDisksForNonreplDiskCheckpoints(true); + auto runtime = PrepareTestActorRuntime(config); + + struct TReadAndWriteByteCount + { + ui64 ReadByteCount = 0; + ui64 WriteByteCount = 0; + }; + TMap statsForDisks; + auto statEventInterceptor = [&](TAutoPtr& event) + { + if (event->Recipient == MakeStorageStatsServiceId() && + event->GetTypeRewrite() == + TEvStatsService::EvVolumePartCounters) + { + auto* msg = + event->Get(); + statsForDisks[msg->DiskId].ReadByteCount += + msg->DiskCounters->RequestCounters.ReadBlocks.RequestBytes; + statsForDisks[msg->DiskId].WriteByteCount += + msg->DiskCounters->RequestCounters.WriteBlocks.RequestBytes; + } + return TTestActorRuntime::DefaultObserverFunc(event); + }; + runtime->SetObserverFunc(statEventInterceptor); + + TVolumeClient volume(*runtime); + volume.UpdateVolumeConfig( + 0, + 0, + 0, + 0, + false, + 1, + NCloud::NProto::STORAGE_MEDIA_SSD_NONREPLICATED, + DiskBlockCount, + "vol0"); + + volume.WaitReady(); + + auto clientInfo = CreateVolumeClientInfo( + NProto::VOLUME_ACCESS_READ_WRITE, + NProto::VOLUME_MOUNT_LOCAL, + 0); + volume.AddClient(clientInfo); + + // Create checkpoint. + volume.CreateCheckpoint("c1"); + + // Reconnect pipe since partition has restarted. + volume.ReconnectPipe(); + runtime->DispatchEvents({}, TDuration::MilliSeconds(10)); + + // Write to the source disk. These writes will be mirrored to the shadow + // disk too. + volume.WriteBlocks( + TBlockRange64::WithLength( + DiskBlockCount - WriteBlockCount - 1, + WriteBlockCount), + clientInfo.GetClientId(), + GetBlockContent(2)); + + // Read from the source disk. + volume.ReadBlocks( + TBlockRange64::WithLength(0, ReadFromSourceBlockCount), + clientInfo.GetClientId()); + + // Wait for checkpoint get ready. + for (;;) { + auto status = volume.GetCheckpointStatus("c1"); + if (status->Record.GetCheckpointStatus() == + NProto::ECheckpointStatus::READY) + { + break; + } + runtime->DispatchEvents({}, TDuration::MilliSeconds(10)); + } + + // Read from checkpoint. + volume.ReadBlocks( + TBlockRange64::WithLength(0, ReadFromCheckpointBlockCount), + clientInfo.GetClientId(), + "c1"); + + // Wait for stats send to StorageStatsService. + runtime->AdvanceCurrentTime(UpdateCountersInterval); + runtime->DispatchEvents({}, TDuration::Seconds(1)); + runtime->AdvanceCurrentTime(UpdateCountersInterval); + runtime->DispatchEvents({}, TDuration::Seconds(1)); + + // Validate bytes count for source and shadow disks. + UNIT_ASSERT_VALUES_EQUAL(2, statsForDisks.size()); + UNIT_ASSERT_VALUES_EQUAL( + DiskByteCount + ReadFromSourceBlockCount * DiskBlockSize, + statsForDisks["vol0"].ReadByteCount); + UNIT_ASSERT_VALUES_EQUAL( + WriteBlockCount * DiskBlockSize, + statsForDisks["vol0"].WriteByteCount); + + UNIT_ASSERT_VALUES_EQUAL( + ReadFromCheckpointBlockCount * DiskBlockSize, + statsForDisks["vol0c1"].ReadByteCount); + UNIT_ASSERT_VALUES_EQUAL( + DiskByteCount + WriteBlockCount * DiskBlockSize, + statsForDisks["vol0c1"].WriteByteCount); + } } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/volume_proxy/ya.make b/cloud/blockstore/libs/storage/volume_proxy/ya.make index ed8990a5300..9dc7a5caa99 100644 --- a/cloud/blockstore/libs/storage/volume_proxy/ya.make +++ b/cloud/blockstore/libs/storage/volume_proxy/ya.make @@ -10,7 +10,6 @@ PEERDIR( cloud/blockstore/libs/storage/core contrib/ydb/library/actors/core contrib/ydb/core/tablet - contrib/ydb/core/testlib ) END() diff --git a/cloud/blockstore/libs/vhost/server.cpp b/cloud/blockstore/libs/vhost/server.cpp index badd3278c7f..51394986186 100644 --- a/cloud/blockstore/libs/vhost/server.cpp +++ b/cloud/blockstore/libs/vhost/server.cpp @@ -663,7 +663,7 @@ TFuture TServer::StartEndpoint( auto it = EndpointMap.find(socketPath); if (it != EndpointMap.end()) { NProto::TError error; - error.SetCode(E_FAIL); + error.SetCode(S_ALREADY); error.SetMessage(TStringBuilder() << "endpoint " << socketPath.Quote() << " has already been started"); diff --git a/cloud/blockstore/private/api/protos/disk.proto b/cloud/blockstore/private/api/protos/disk.proto index 4c2a0bfd15f..c10006d6012 100644 --- a/cloud/blockstore/private/api/protos/disk.proto +++ b/cloud/blockstore/private/api/protos/disk.proto @@ -69,3 +69,15 @@ message TGetDiskRegistryTabletInfoResponse { uint64 TabletId = 1; } + +//////////////////////////////////////////////////////////////////////////////// +// DiskRegistrySetWritableState request/response. + +message TDiskRegistrySetWritableStateRequest +{ + bool State = 1; +} + +message TDiskRegistrySetWritableStateResponse +{ +} diff --git a/cloud/blockstore/public/api/protos/endpoints.proto b/cloud/blockstore/public/api/protos/endpoints.proto index 495f99f3740..27e4bb8ff44 100644 --- a/cloud/blockstore/public/api/protos/endpoints.proto +++ b/cloud/blockstore/public/api/protos/endpoints.proto @@ -87,6 +87,14 @@ message TStartEndpointRequest // Restore endpoint after restart bool Persistent = 24; + + oneof NbdDevice { + // The device file (e.g. "/dev/nbd0") which nbd-client will connect to. + string NbdDeviceFile = 25; + + // Use any free nbd device file which nbd-client will connect to. + bool UseFreeNbdDeviceFile = 26; + } } message TStartEndpointResponse @@ -96,6 +104,9 @@ message TStartEndpointResponse // Volume information. TVolume Volume = 2; + + // The device file (e.g. "/dev/nbd0") which nbd-client connected to. + string NbdDeviceFile = 3; } //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/blockstore/public/api/protos/volume.proto b/cloud/blockstore/public/api/protos/volume.proto index c0f799b6a89..9f91e1edfcb 100644 --- a/cloud/blockstore/public/api/protos/volume.proto +++ b/cloud/blockstore/public/api/protos/volume.proto @@ -389,6 +389,9 @@ message TVolumeStats uint64 CleanupQueueBytes = 26; // TODO: current max device timeout + + uint32 UnconfirmedBlobCount = 27; + uint32 ConfirmedBlobCount = 28; } //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/blockstore/public/sdk/go/client/discovery.go b/cloud/blockstore/public/sdk/go/client/discovery.go index e60296ad54e..0a4fe062e42 100644 --- a/cloud/blockstore/public/sdk/go/client/discovery.go +++ b/cloud/blockstore/public/sdk/go/client/discovery.go @@ -755,6 +755,24 @@ func (client *discoveryClient) CreateCheckpoint( return resp.(*protos.TCreateCheckpointResponse), err } +func (client *discoveryClient) GetCheckpointStatus( + ctx context.Context, + req *protos.TGetCheckpointStatusRequest, +) (*protos.TGetCheckpointStatusResponse, error) { + + resp, err := client.executeRequest( + ctx, + func(ctx context.Context, impl ClientIface) (response, error) { + return impl.GetCheckpointStatus(ctx, req) + }) + + if err != nil { + return nil, err + } + + return resp.(*protos.TGetCheckpointStatusResponse), err +} + func (client *discoveryClient) DeleteCheckpoint( ctx context.Context, req *protos.TDeleteCheckpointRequest, diff --git a/cloud/blockstore/public/sdk/go/client/durable.go b/cloud/blockstore/public/sdk/go/client/durable.go index 7d81c7eca58..5361952fc0f 100644 --- a/cloud/blockstore/public/sdk/go/client/durable.go +++ b/cloud/blockstore/public/sdk/go/client/durable.go @@ -464,6 +464,22 @@ func (client *durableClient) CreateCheckpoint( return resp.(*protos.TCreateCheckpointResponse), err } +func (client *durableClient) GetCheckpointStatus( + ctx context.Context, + req *protos.TGetCheckpointStatusRequest, +) (*protos.TGetCheckpointStatusResponse, error) { + + resp, err := client.executeRequest( + ctx, + req, + func(ctx context.Context) (response, error) { + return client.impl.GetCheckpointStatus(ctx, req) + }, + ) + + return resp.(*protos.TGetCheckpointStatusResponse), err +} + func (client *durableClient) DeleteCheckpoint( ctx context.Context, req *protos.TDeleteCheckpointRequest, diff --git a/cloud/blockstore/public/sdk/go/client/grpc.go b/cloud/blockstore/public/sdk/go/client/grpc.go index f1dcc769233..89f8e558ee9 100644 --- a/cloud/blockstore/public/sdk/go/client/grpc.go +++ b/cloud/blockstore/public/sdk/go/client/grpc.go @@ -577,6 +577,24 @@ func (client *grpcClient) CreateCheckpoint( return resp.(*protos.TCreateCheckpointResponse), err } +func (client *grpcClient) GetCheckpointStatus( + ctx context.Context, + req *protos.TGetCheckpointStatusRequest, +) (*protos.TGetCheckpointStatusResponse, error) { + + if req.Headers == nil { + req.Headers = &protos.THeaders{} + } + resp, err := client.executeRequest( + ctx, + req, + func(ctx context.Context) (response, error) { + return client.impl.GetCheckpointStatus(ctx, req) + }) + + return resp.(*protos.TGetCheckpointStatusResponse), err +} + func (client *grpcClient) DeleteCheckpoint( ctx context.Context, req *protos.TDeleteCheckpointRequest, diff --git a/cloud/blockstore/public/sdk/go/client/iface.go b/cloud/blockstore/public/sdk/go/client/iface.go index 8ffc40013ee..a446849c2d6 100644 --- a/cloud/blockstore/public/sdk/go/client/iface.go +++ b/cloud/blockstore/public/sdk/go/client/iface.go @@ -157,6 +157,11 @@ type ClientIface interface { req *protos.TCreateCheckpointRequest, ) (*protos.TCreateCheckpointResponse, error) + GetCheckpointStatus( + ctx context.Context, + req *protos.TGetCheckpointStatusRequest, + ) (*protos.TGetCheckpointStatusResponse, error) + DeleteCheckpoint( ctx context.Context, req *protos.TDeleteCheckpointRequest, diff --git a/cloud/blockstore/public/sdk/go/client/request.go b/cloud/blockstore/public/sdk/go/client/request.go index f353f57732e..ccb33092841 100644 --- a/cloud/blockstore/public/sdk/go/client/request.go +++ b/cloud/blockstore/public/sdk/go/client/request.go @@ -67,21 +67,21 @@ func requestLogLevel(req request) LogLevel { func requestDetails(req request) string { if readReq, ok := req.(*protos.TReadBlocksRequest); ok { return fmt.Sprintf( - " (offset: %d, count: %d)", + " (offset: %d, blocks count: %d)", readReq.StartIndex, readReq.BlocksCount) } if writeReq, ok := req.(*protos.TWriteBlocksRequest); ok { return fmt.Sprintf( - " (offset: %d, count: %d)", + " (offset: %d, buffers count: %d)", writeReq.StartIndex, len(writeReq.Blocks.Buffers)) } if zeroReq, ok := req.(*protos.TZeroBlocksRequest); ok { return fmt.Sprintf( - " (offset: %d, count: %d)", + " (offset: %d, blocks count: %d)", zeroReq.StartIndex, zeroReq.BlocksCount) } diff --git a/cloud/blockstore/public/sdk/go/client/safe_client.go b/cloud/blockstore/public/sdk/go/client/safe_client.go index de5666c037c..f26e17b6dba 100644 --- a/cloud/blockstore/public/sdk/go/client/safe_client.go +++ b/cloud/blockstore/public/sdk/go/client/safe_client.go @@ -285,6 +285,21 @@ func (client *safeClient) CreateCheckpoint( return err } +func (client *safeClient) GetCheckpointStatus( + ctx context.Context, + diskId string, + checkpointId string, +) (protos.ECheckpointStatus, error) { + + req := &protos.TGetCheckpointStatusRequest{ + DiskId: diskId, + CheckpointId: checkpointId, + } + + resp, err := client.Impl.GetCheckpointStatus(ctx, req) + return resp.CheckpointStatus, err +} + func (client *safeClient) DeleteCheckpoint( ctx context.Context, diskId string, diff --git a/cloud/blockstore/public/sdk/go/client/test_client.go b/cloud/blockstore/public/sdk/go/client/test_client.go index 7def48a17d9..2912535938f 100644 --- a/cloud/blockstore/public/sdk/go/client/test_client.go +++ b/cloud/blockstore/public/sdk/go/client/test_client.go @@ -27,6 +27,7 @@ type listKeyringsHandlerFunc func(ctx context.Context, req *protos.TListKeyrings type describeEndpointHandlerFunc func(ctx context.Context, req *protos.TDescribeEndpointRequest) (*protos.TDescribeEndpointResponse, error) type refreshEndpointHandlerFunc func(ctx context.Context, req *protos.TRefreshEndpointRequest) (*protos.TRefreshEndpointResponse, error) type createCheckpointHandlerFunc func(ctx context.Context, req *protos.TCreateCheckpointRequest) (*protos.TCreateCheckpointResponse, error) +type getCheckpointStatusHandlerFunc func(ctx context.Context, req *protos.TGetCheckpointStatusRequest) (*protos.TGetCheckpointStatusResponse, error) type deleteCheckpointHandlerFunc func(ctx context.Context, req *protos.TDeleteCheckpointRequest) (*protos.TDeleteCheckpointResponse, error) type getChangedBlocksHandlerFunc func(ctx context.Context, req *protos.TGetChangedBlocksRequest) (*protos.TGetChangedBlocksResponse, error) type describeVolumeHandlerFunc func(ctx context.Context, req *protos.TDescribeVolumeRequest) (*protos.TDescribeVolumeResponse, error) @@ -68,6 +69,7 @@ type testClient struct { DescribeEndpointHandler describeEndpointHandlerFunc RefreshEndpointHandler refreshEndpointHandlerFunc CreateCheckpointHandler createCheckpointHandlerFunc + GetCheckpointStatusHandler getCheckpointStatusHandlerFunc DeleteCheckpointHandler deleteCheckpointHandlerFunc GetChangedBlocksHandler getChangedBlocksHandlerFunc DescribeVolumeHandler describeVolumeHandlerFunc @@ -371,6 +373,18 @@ func (client *testClient) CreateCheckpoint( return &protos.TCreateCheckpointResponse{}, nil } +func (client *testClient) GetCheckpointStatus( + ctx context.Context, + req *protos.TGetCheckpointStatusRequest, +) (*protos.TGetCheckpointStatusResponse, error) { + + if client.GetCheckpointStatusHandler != nil { + return client.GetCheckpointStatusHandler(ctx, req) + } + + return &protos.TGetCheckpointStatusResponse{}, nil +} + func (client *testClient) DeleteCheckpoint( ctx context.Context, req *protos.TDeleteCheckpointRequest, diff --git a/cloud/blockstore/pylibs/common/ssh_client.py b/cloud/blockstore/pylibs/common/ssh_client.py index c694a09b5c2..d2057ed9cc7 100644 --- a/cloud/blockstore/pylibs/common/ssh_client.py +++ b/cloud/blockstore/pylibs/common/ssh_client.py @@ -131,7 +131,6 @@ def configure_ssh_client( self._key_path_cmd_argument = [] else: self._key_path_cmd_argument = ['-i', str(self._ssh_key_path)] - self._key_path_cmd_argument = ['-i', str(self._ssh_key_path)] self._authorization_string = f'{self._username}@{self._host}' self._scp_authorization_string = self._authorization_string try: @@ -212,7 +211,7 @@ def truncate(self, remote_path: str, size: int) -> None: self._exec_command( ['truncate', '-s', f'{size}', remote_path], check=True) - @retry(tries=4, delay=20) + @retry(tries=10, delay=20) def upload_file(self, local_path: str, remote_path: str) -> None: command_line = [ 'scp', diff --git a/cloud/blockstore/tests/client/test_with_client.py b/cloud/blockstore/tests/client/test_with_client.py index 9f483a1c250..5e2f250c74c 100644 --- a/cloud/blockstore/tests/client/test_with_client.py +++ b/cloud/blockstore/tests/client/test_with_client.py @@ -20,6 +20,12 @@ TDescribeVolumeResponse, TListVolumesResponse, ) +from cloud.blockstore.public.api.protos.checkpoints_pb2 import ( + TGetCheckpointStatusRequest, + TGetCheckpointStatusResponse, + ECheckpointStatus, +) +from cloud.blockstore.public.sdk.python.client.error_codes import EResult from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest from cloud.blockstore.tests.python.lib.nonreplicated_setup import ( enable_writable_state, @@ -126,6 +132,20 @@ def list_volumes(env, run): return file_parse(env.results_path, TListVolumesResponse()) +def get_checkpoint_status(env, run, disk_id, checkpoint_id): + clear_file(env.results_file) + req = TGetCheckpointStatusRequest() + req.DiskId = disk_id + req.CheckpointId = checkpoint_id + tmp_file = tempfile.NamedTemporaryFile(suffix=".tmp") + tmp_file.write(text_format.MessageToString(req).encode("utf8")) + tmp_file.flush() + run("getcheckpointstatus", + "--input", tmp_file.name, + "--proto") + return file_parse(env.results_path, TGetCheckpointStatusResponse()) + + def describe_placement_group(env, run, group_id): clear_file(env.results_file) req = TDescribePlacementGroupRequest() @@ -788,6 +808,33 @@ def test_create_destroy_placementgroup(): "--group-id", "group-0") +def test_get_checkpoint_status(): + env, run = setup(with_nrd=True, nrd_device_count=2, rack='') + + run("createvolume", + "--disk-id", "vol1", + "--blocks-count", str(NRD_BLOCKS_COUNT), + "--storage-media-kind", "nonreplicated") + assert file_equal(env.results_path, 'OK\n') + clear_file(env.results_file) + + run("createcheckpoint", + "--disk-id", "vol1", + "--checkpoint-id", "cp1") + assert file_equal(env.results_path, 'OK\n') + clear_file(env.results_file) + + response = get_checkpoint_status(env, run, "vol1", "cp1") + assert response.CheckpointStatus == ECheckpointStatus.READY + + response = get_checkpoint_status(env, run, "non-exists-volume", "cp1") + assert response.Error.Message == "Path not found" + + response = get_checkpoint_status(env, run, "vol1", "non-exists-checkpoint") + assert response.Error.Code == EResult.E_NOT_FOUND.value + assert response.Error.Message == "Checkpoint not found" + + def test_disabled_configs_dispatcher(): storage = TStorageServiceConfig() storage.ConfigsDispatcherServiceEnabled = False diff --git a/cloud/blockstore/tests/functional/ya.make b/cloud/blockstore/tests/functional/ya.make index 5ce86342838..e75e96391ca 100644 --- a/cloud/blockstore/tests/functional/ya.make +++ b/cloud/blockstore/tests/functional/ya.make @@ -12,7 +12,7 @@ IF(NOT SANITIZER_TYPE) ) ENDIF() -# https://st.yandex-team.ru/DEVTOOLSSUPPORT-18977#6285fbd36101de4de4e29f48 +# DEVTOOLSSUPPORT-18977 IF (SANITIZER_TYPE != "undefined" AND SANITIZER_TYPE != "memory") RECURSE_ROOT_RELATIVE( cloud/blockstore/tests/plugin diff --git a/cloud/blockstore/tests/loadtest/local-checkpoint/test.py b/cloud/blockstore/tests/loadtest/local-checkpoint/test.py index 304e7a919f8..ee09ccc9386 100644 --- a/cloud/blockstore/tests/loadtest/local-checkpoint/test.py +++ b/cloud/blockstore/tests/loadtest/local-checkpoint/test.py @@ -3,44 +3,13 @@ import yatest.common as common from cloud.blockstore.config.server_pb2 import TServerAppConfig, TServerConfig, TKikimrServiceConfig -from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig +from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test -import google.protobuf.json_format as protojson - - -def parse_storage_config(param): - if param is None: - return None - - return protojson.Parse(param, TStorageServiceConfig()) - def default_storage_config_patch(): - bw = 1 << 7 # 128 MB/s - iops = 1 << 16 - - storage = TStorageServiceConfig() - storage.ThrottlingEnabled = True - - storage.SSDUnitReadBandwidth = bw - storage.SSDUnitWriteBandwidth = bw - storage.SSDMaxReadBandwidth = bw - storage.SSDMaxWriteBandwidth = bw - storage.SSDUnitReadIops = iops - storage.SSDUnitWriteIops = iops - storage.SSDMaxReadIops = iops - storage.SSDMaxWriteIops = iops - - storage.HDDUnitReadBandwidth = bw - storage.HDDUnitWriteBandwidth = bw - storage.HDDMaxReadBandwidth = bw - storage.HDDMaxWriteBandwidth = bw - storage.HDDUnitReadIops = iops - storage.HDDUnitWriteIops = iops - storage.HDDMaxReadIops = iops - storage.HDDMaxWriteIops = iops + storage = storage_config_with_default_limits() storage.InactiveClientsTimeout = 10000 diff --git a/cloud/blockstore/tests/loadtest/local-edgecase/test.py b/cloud/blockstore/tests/loadtest/local-edgecase/test.py index 5c3fb12906d..a6cd2fae646 100644 --- a/cloud/blockstore/tests/loadtest/local-edgecase/test.py +++ b/cloud/blockstore/tests/loadtest/local-edgecase/test.py @@ -3,35 +3,14 @@ import yatest.common as common from cloud.blockstore.config.server_pb2 import TServerAppConfig, TServerConfig, TKikimrServiceConfig -from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig, CT_LOAD +from cloud.blockstore.config.storage_pb2 import CT_LOAD +from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test def default_storage_config_patch(tablet_version=1): - bw = 1 << 7 # 128 MB/s - iops = 1 << 16 - - storage = TStorageServiceConfig() - storage.ThrottlingEnabled = True - - storage.SSDUnitReadBandwidth = bw - storage.SSDUnitWriteBandwidth = bw - storage.SSDMaxReadBandwidth = bw - storage.SSDMaxWriteBandwidth = bw - storage.SSDUnitReadIops = iops - storage.SSDUnitWriteIops = iops - storage.SSDMaxReadIops = iops - storage.SSDMaxWriteIops = iops - - storage.HDDUnitReadBandwidth = bw - storage.HDDUnitWriteBandwidth = bw - storage.HDDMaxReadBandwidth = bw - storage.HDDMaxWriteBandwidth = bw - storage.HDDUnitReadIops = iops - storage.HDDUnitWriteIops = iops - storage.HDDMaxReadIops = iops - storage.HDDMaxWriteIops = iops + storage = storage_config_with_default_limits() if tablet_version == 2: storage.BlockDigestsEnabled = True @@ -44,7 +23,7 @@ def default_storage_config_patch(tablet_version=1): def storage_config_with_batching(tablet_version=1): - storage = TStorageServiceConfig() + storage = storage_config_with_default_limits() storage.WriteRequestBatchingEnabled = True storage.ThrottlingEnabled = False @@ -57,7 +36,7 @@ def storage_config_with_batching(tablet_version=1): def storage_config_with_new_compaction(): - storage = TStorageServiceConfig() + storage = storage_config_with_default_limits() storage.SSDCompactionType = CT_LOAD storage.HDDCompactionType = CT_LOAD storage.V1GarbageCompactionEnabled = True diff --git a/cloud/blockstore/tests/loadtest/local-emergency/test.py b/cloud/blockstore/tests/loadtest/local-emergency/test.py index 79f89eda6de..302a731e611 100644 --- a/cloud/blockstore/tests/loadtest/local-emergency/test.py +++ b/cloud/blockstore/tests/loadtest/local-emergency/test.py @@ -3,8 +3,8 @@ import yatest.common as common -from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig from cloud.blockstore.public.sdk.python.client import CreateClient, Session +from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest from cloud.blockstore.tests.python.lib.test_base import run_test @@ -12,8 +12,7 @@ def default_storage_config(cache_folder): - storage = TStorageServiceConfig() - storage.ThrottlingEnabled = True + storage = storage_config_with_default_limits() storage.HDDSystemChannelPoolKind = "rot" storage.SSDSystemChannelPoolKind = "rot" storage.HybridSystemChannelPoolKind = "rot" diff --git a/cloud/blockstore/tests/loadtest/local-endpoints/test.py b/cloud/blockstore/tests/loadtest/local-endpoints/test.py index 6a31afb7e41..98dcfcc7a5f 100644 --- a/cloud/blockstore/tests/loadtest/local-endpoints/test.py +++ b/cloud/blockstore/tests/loadtest/local-endpoints/test.py @@ -5,8 +5,8 @@ from cloud.blockstore.config.client_pb2 import TClientConfig from cloud.blockstore.config.server_pb2 import \ TServerAppConfig, TServerConfig, TKikimrServiceConfig -from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig +from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test @@ -30,30 +30,7 @@ def __init__( def default_storage_config(): - bw = 1 << 5 # 32 MB/s - iops = 1 << 12 # 4096 iops - - storage = TStorageServiceConfig() - storage.ThrottlingEnabled = True - - storage.SSDUnitReadBandwidth = bw - storage.SSDUnitWriteBandwidth = bw - storage.SSDMaxReadBandwidth = bw - storage.SSDMaxWriteBandwidth = bw - storage.SSDUnitReadIops = iops - storage.SSDUnitWriteIops = iops - storage.SSDMaxReadIops = iops - storage.SSDMaxWriteIops = iops - - storage.HDDUnitReadBandwidth = bw - storage.HDDUnitWriteBandwidth = bw - storage.HDDMaxReadBandwidth = bw - storage.HDDMaxWriteBandwidth = bw - storage.HDDUnitReadIops = iops - storage.HDDUnitWriteIops = iops - storage.HDDMaxReadIops = iops - storage.HDDMaxWriteIops = iops - + storage = storage_config_with_default_limits() storage.InactiveClientsTimeout = 10000 return storage diff --git a/cloud/blockstore/tests/loadtest/local-nemesis/test.py b/cloud/blockstore/tests/loadtest/local-nemesis/test.py index ab1c026a71c..ae4c6d45d5a 100644 --- a/cloud/blockstore/tests/loadtest/local-nemesis/test.py +++ b/cloud/blockstore/tests/loadtest/local-nemesis/test.py @@ -5,41 +5,13 @@ from cloud.blockstore.config.client_pb2 import TClientConfig from cloud.blockstore.config.server_pb2 import TServerAppConfig, TServerConfig, TKikimrServiceConfig -from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig +from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test, \ get_restart_interval from cloud.storage.core.protos.endpoints_pb2 import EEndpointStorageType -def default_storage_config_patch(): - bw = 1 << 7 # 128 MB/s - iops = 1 << 16 - - storage = TStorageServiceConfig() - storage.ThrottlingEnabled = True - - storage.SSDUnitReadBandwidth = bw - storage.SSDUnitWriteBandwidth = bw - storage.SSDMaxReadBandwidth = bw - storage.SSDMaxWriteBandwidth = bw - storage.SSDUnitReadIops = iops - storage.SSDUnitWriteIops = iops - storage.SSDMaxReadIops = iops - storage.SSDMaxWriteIops = iops - - storage.HDDUnitReadBandwidth = bw - storage.HDDUnitWriteBandwidth = bw - storage.HDDMaxReadBandwidth = bw - storage.HDDMaxWriteBandwidth = bw - storage.HDDUnitReadIops = iops - storage.HDDUnitWriteIops = iops - storage.HDDMaxReadIops = iops - storage.HDDMaxWriteIops = iops - - return storage - - class TestCase(object): def __init__( @@ -98,7 +70,7 @@ def __run_test(test_case): env = LocalLoadTest( "", server_app_config=server, - storage_config_patches=[default_storage_config_patch()], + storage_config_patches=[storage_config_with_default_limits()], use_in_memory_pdisks=True, restart_interval=test_case.restart_interval, ) diff --git a/cloud/blockstore/tests/loadtest/local-newfeatures/test.py b/cloud/blockstore/tests/loadtest/local-newfeatures/test.py index 4d839305d35..c2a149f55aa 100644 --- a/cloud/blockstore/tests/loadtest/local-newfeatures/test.py +++ b/cloud/blockstore/tests/loadtest/local-newfeatures/test.py @@ -5,7 +5,8 @@ from cloud.blockstore.config.client_pb2 import TClientConfig from cloud.blockstore.config.server_pb2 import TServerAppConfig, TServerConfig, TKikimrServiceConfig -from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig, CT_LOAD +from cloud.blockstore.config.storage_pb2 import CT_LOAD +from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test, \ get_restart_interval @@ -14,35 +15,13 @@ def default_storage_config(): - bw = 1 << 7 # 128 MB/s - iops = 1 << 16 - - storage = TStorageServiceConfig() + storage = storage_config_with_default_limits() storage.SSDCompactionType = CT_LOAD storage.HDDCompactionType = CT_LOAD storage.V1GarbageCompactionEnabled = True - - storage.ThrottlingEnabled = True - storage.ThrottlingEnabledSSD = True - - storage.SSDUnitReadBandwidth = bw - storage.SSDUnitWriteBandwidth = bw - storage.SSDMaxReadBandwidth = bw - storage.SSDMaxWriteBandwidth = bw - storage.SSDUnitReadIops = iops - storage.SSDUnitWriteIops = iops - storage.SSDMaxReadIops = iops - storage.SSDMaxWriteIops = iops - - storage.HDDUnitReadBandwidth = bw - storage.HDDUnitWriteBandwidth = bw - storage.HDDMaxReadBandwidth = bw - storage.HDDMaxWriteBandwidth = bw - storage.HDDUnitReadIops = iops - storage.HDDUnitWriteIops = iops - storage.HDDMaxReadIops = iops - storage.HDDMaxWriteIops = iops + storage.DiskPrefixLengthWithBlockChecksumsInBlobs = 1 << 30 + storage.CheckBlockChecksumsInBlobsUponRead = True return storage @@ -67,6 +46,9 @@ def storage_config_with_incremental_batch_compaction(): def storage_config_with_incremental_compaction_and_patching(): storage = storage_config_with_incremental_compaction() storage.BlobPatchingEnabled = True + # checksums are currently not supported for patched blobs + storage.DiskPrefixLengthWithBlockChecksumsInBlobs = 0 + storage.CheckBlockChecksumsInBlobsUponRead = False return storage @@ -116,6 +98,9 @@ def storage_config_with_mixed_index_cache_enabled(): def storage_config_with_adding_unconfirmed_blobs_enabled(): storage = default_storage_config() storage.AddingUnconfirmedBlobsEnabled = True + # checksums are currently not supported for unconfirmed blobs + storage.DiskPrefixLengthWithBlockChecksumsInBlobs = 0 + storage.CheckBlockChecksumsInBlobsUponRead = False return storage diff --git a/cloud/blockstore/tests/loadtest/local-overlay/test.py b/cloud/blockstore/tests/loadtest/local-overlay/test.py index b4f3174c807..68dd1e6b208 100644 --- a/cloud/blockstore/tests/loadtest/local-overlay/test.py +++ b/cloud/blockstore/tests/loadtest/local-overlay/test.py @@ -3,44 +3,13 @@ import yatest.common as common from cloud.blockstore.config.server_pb2 import TServerAppConfig, TServerConfig, TKikimrServiceConfig -from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig +from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test -import google.protobuf.json_format as protojson - - -def parse_storage_config(param): - if param is None: - return None - - return protojson.Parse(param, TStorageServiceConfig()) - def default_storage_config_patch(tablet_version): - bw = 1 << 7 # 128 MB/s - iops = 1 << 16 - - storage = TStorageServiceConfig() - storage.ThrottlingEnabled = True - - storage.SSDUnitReadBandwidth = bw - storage.SSDUnitWriteBandwidth = bw - storage.SSDMaxReadBandwidth = bw - storage.SSDMaxWriteBandwidth = bw - storage.SSDUnitReadIops = iops - storage.SSDUnitWriteIops = iops - storage.SSDMaxReadIops = iops - storage.SSDMaxWriteIops = iops - - storage.HDDUnitReadBandwidth = bw - storage.HDDUnitWriteBandwidth = bw - storage.HDDMaxReadBandwidth = bw - storage.HDDMaxWriteBandwidth = bw - storage.HDDUnitReadIops = iops - storage.HDDUnitWriteIops = iops - storage.HDDMaxReadIops = iops - storage.HDDMaxWriteIops = iops + storage = storage_config_with_default_limits() storage.InactiveClientsTimeout = 10000 diff --git a/cloud/blockstore/tests/loadtest/local-v2/test.py b/cloud/blockstore/tests/loadtest/local-v2/test.py index e2e9cc0b9e2..cdcc6a39b0f 100644 --- a/cloud/blockstore/tests/loadtest/local-v2/test.py +++ b/cloud/blockstore/tests/loadtest/local-v2/test.py @@ -3,44 +3,13 @@ import yatest.common as common from cloud.blockstore.config.server_pb2 import TServerAppConfig, TServerConfig, TKikimrServiceConfig -from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig +from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test -import google.protobuf.json_format as protojson - - -def parse_storage_config(param): - if param is None: - return None - - return protojson.Parse(param, TStorageServiceConfig()) - def default_storage_config_patch(): - bw = 1 << 7 # 128 MB/s - iops = 1 << 16 - - storage = TStorageServiceConfig() - storage.ThrottlingEnabled = True - - storage.SSDUnitReadBandwidth = bw - storage.SSDUnitWriteBandwidth = bw - storage.SSDMaxReadBandwidth = bw - storage.SSDMaxWriteBandwidth = bw - storage.SSDUnitReadIops = iops - storage.SSDUnitWriteIops = iops - storage.SSDMaxReadIops = iops - storage.SSDMaxWriteIops = iops - - storage.HDDUnitReadBandwidth = bw - storage.HDDUnitWriteBandwidth = bw - storage.HDDMaxReadBandwidth = bw - storage.HDDMaxWriteBandwidth = bw - storage.HDDUnitReadIops = iops - storage.HDDUnitWriteIops = iops - storage.HDDMaxReadIops = iops - storage.HDDMaxWriteIops = iops + storage = storage_config_with_default_limits() storage.InactiveClientsTimeout = 10000 diff --git a/cloud/blockstore/tests/loadtest/local/test.py b/cloud/blockstore/tests/loadtest/local/test.py index f722ae61849..2c17a52eed0 100644 --- a/cloud/blockstore/tests/loadtest/local/test.py +++ b/cloud/blockstore/tests/loadtest/local/test.py @@ -3,46 +3,15 @@ import yatest.common as common from cloud.blockstore.config.server_pb2 import TServerAppConfig, TServerConfig, TKikimrServiceConfig -from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig +from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test from contrib.ydb.tests.library.harness.kikimr_runner import get_unique_path_for_current_test, ensure_path_exists -import google.protobuf.json_format as protojson - - -def parse_storage_config(param): - if param is None: - return None - - return protojson.Parse(param, TStorageServiceConfig()) - def default_storage_config(tablet_version, cache_folder): - bw = 1 << 6 # 64 MiB/s - iops = 1 << 12 # 4096 IOPS - - storage = TStorageServiceConfig() - storage.ThrottlingEnabled = True - - storage.SSDUnitReadBandwidth = bw - storage.SSDUnitWriteBandwidth = bw - storage.SSDMaxReadBandwidth = bw - storage.SSDMaxWriteBandwidth = bw - storage.SSDUnitReadIops = iops - storage.SSDUnitWriteIops = iops - storage.SSDMaxReadIops = iops - storage.SSDMaxWriteIops = iops - - storage.HDDUnitReadBandwidth = bw - storage.HDDUnitWriteBandwidth = bw - storage.HDDMaxReadBandwidth = bw - storage.HDDMaxWriteBandwidth = bw - storage.HDDUnitReadIops = iops - storage.HDDUnitWriteIops = iops - storage.HDDMaxReadIops = iops - storage.HDDMaxWriteIops = iops + storage = storage_config_with_default_limits() storage.InactiveClientsTimeout = 10000 storage.DiskPrefixLengthWithBlockChecksumsInBlobs = 1 << 30 # 1 GiB diff --git a/cloud/blockstore/tests/python/lib/config.py b/cloud/blockstore/tests/python/lib/config.py index c73e19fb078..836a1421de1 100644 --- a/cloud/blockstore/tests/python/lib/config.py +++ b/cloud/blockstore/tests/python/lib/config.py @@ -301,3 +301,32 @@ def generate_disk_agent_txt( ParseDict(storage_discovery_config, TStorageDiscoveryConfig())) return config + + +def storage_config_with_default_limits(): + bw = 1 << 5 # 32 MiB/s + iops = 1 << 16 + + storage = TStorageServiceConfig() + storage.ThrottlingEnabledSSD = True + storage.ThrottlingEnabled = True + + storage.SSDUnitReadBandwidth = bw + storage.SSDUnitWriteBandwidth = bw + storage.SSDMaxReadBandwidth = bw + storage.SSDMaxWriteBandwidth = bw + storage.SSDUnitReadIops = iops + storage.SSDUnitWriteIops = iops + storage.SSDMaxReadIops = iops + storage.SSDMaxWriteIops = iops + + storage.HDDUnitReadBandwidth = bw + storage.HDDUnitWriteBandwidth = bw + storage.HDDMaxReadBandwidth = bw + storage.HDDMaxWriteBandwidth = bw + storage.HDDUnitReadIops = iops + storage.HDDUnitWriteIops = iops + storage.HDDMaxReadIops = iops + storage.HDDMaxWriteIops = iops + + return storage diff --git a/cloud/blockstore/tests/storage_discovery/test.py b/cloud/blockstore/tests/storage_discovery/test.py index cd04839da55..cc4587d8d89 100644 --- a/cloud/blockstore/tests/storage_discovery/test.py +++ b/cloud/blockstore/tests/storage_discovery/test.py @@ -1,6 +1,7 @@ import hashlib import os import pytest +import time from copy import deepcopy @@ -334,6 +335,14 @@ def test_config_comparison( assert crit is not None assert crit['value'] == (1 if cmp == 'mismatch' else 0) + if cmp == 'mismatch': + # Wait for duplicate event. + time.sleep(30) + crit = disk_agent.counters.find( + {'sensor': 'AppCriticalEvents/DiskAgentConfigMismatch'}) + assert crit is not None + assert crit['value'] == 2 + disk_agent.kill() diff --git a/cloud/blockstore/tests/ya.make b/cloud/blockstore/tests/ya.make index d6b8f0bbc64..b134a179c3e 100644 --- a/cloud/blockstore/tests/ya.make +++ b/cloud/blockstore/tests/ya.make @@ -1,4 +1,4 @@ -# https://st.yandex-team.ru/DEVTOOLSSUPPORT-18977#6285fbd36101de4de4e29f48 +# DEVTOOLSSUPPORT-18977 IF (SANITIZER_TYPE != "undefined" AND SANITIZER_TYPE != "memory") RECURSE( plugin diff --git a/cloud/blockstore/tools/analytics/find-perf-bottlenecks/lib/trace.py b/cloud/blockstore/tools/analytics/find-perf-bottlenecks/lib/trace.py index 22a57032709..19d179a18ad 100644 --- a/cloud/blockstore/tools/analytics/find-perf-bottlenecks/lib/trace.py +++ b/cloud/blockstore/tools/analytics/find-perf-bottlenecks/lib/trace.py @@ -113,7 +113,7 @@ def describe_trace(trace): return " ".join(probes) -# https://a.yandex-team.ru/arc/trunk/arcadia/cloud/storage/core/protos/media.proto?rev=r9021410#L11 +# https://github.com/ydb-platform/nbs/blob/main/cloud/storage/core/protos/media.proto#L10 def media_kind2str(media_kind): if media_kind == "0" or media_kind == "2" or media_kind == "3": return "HDD" diff --git a/cloud/blockstore/tools/cms/lib/cms.py b/cloud/blockstore/tools/cms/lib/cms.py index fe16ca78cda..3d0b71a8ae6 100644 --- a/cloud/blockstore/tools/cms/lib/cms.py +++ b/cloud/blockstore/tools/cms/lib/cms.py @@ -24,6 +24,7 @@ def __init__(self, pssh, test_data_dir=None): self.__pssh = pssh self.__dir = test_data_dir self.__request_id = 0 + self.__chowns = set() def execute(self, target, proto): logging.info(f'[cms.execute] {target} proto: {proto}') @@ -34,6 +35,14 @@ def execute(self, target, proto): target_path = '~/cms_request_%s' % os.path.basename(tmp_file.name) + # to be able to copy files to hosts we need to invoke 'chown' on home + # folder + if target not in self.__chowns: + logging.info('[cms.execute] chown...') + output = self.__pssh.run("sudo chown $(whoami) .", target) + logging.info(f'[cms.execute] chown: {output}') + self.__chowns.add(target) + self.__pssh.scp_to(target, tmp_file.name, target_path) cmd = "kikimr admin console execute " + target_path diff --git a/cloud/blockstore/tools/cms/lib/patcher_main.py b/cloud/blockstore/tools/cms/lib/patcher_main.py index a23ed8db219..0ae9417ef91 100644 --- a/cloud/blockstore/tools/cms/lib/patcher_main.py +++ b/cloud/blockstore/tools/cms/lib/patcher_main.py @@ -22,8 +22,8 @@ def __init__(self, conductor_maker, pssh_maker): def make_conductor(self, test_data_dir: str): return self._conductor_maker(test_data_dir) - def make_pssh(self, robot: bool): - return self._pssh_maker(robot) + def make_pssh(self, robot: bool, cluster: str): + return self._pssh_maker(robot, cluster) class BgColors: @@ -167,7 +167,7 @@ def do_main(module_factories: ModuleFactories, default_host_cfg_path: str): scp = libtools.ScpMock(args.test_data_dir) conductor = libconductor.ConductorMock(args.test_data_dir) else: - pssh = module_factories.make_pssh(args.robot) + pssh = module_factories.make_pssh(args.robot, cluster_config.cluster) cms_engine = libcms.CmsEngine(pssh, args.test_data_dir) scp = libtools.Scp(pssh, args.test_data_dir) conductor = module_factories.make_conductor(args.test_data_dir) diff --git a/cloud/blockstore/tools/cms/patcher/__main__.py b/cloud/blockstore/tools/cms/patcher/__main__.py index 0e4bc76bcef..9ade65cf126 100644 --- a/cloud/blockstore/tools/cms/patcher/__main__.py +++ b/cloud/blockstore/tools/cms/patcher/__main__.py @@ -9,7 +9,7 @@ def make_conductor(test_data_dir: str): return ConductorMock(test_data_dir) -def make_pssh(robot: bool): +def make_pssh(robot: bool, cluster: str): return PsshMock() diff --git a/cloud/blockstore/tools/csi_driver/Dockerfile b/cloud/blockstore/tools/csi_driver/Dockerfile new file mode 100644 index 00000000000..69b895705eb --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/Dockerfile @@ -0,0 +1,5 @@ +FROM ubuntu:20.04 + +ADD ./cmd/nbs-csi-driver/nbs-csi-driver . + +ENTRYPOINT ["/nbs-csi-driver"] diff --git a/cloud/blockstore/tools/csi_driver/README.md b/cloud/blockstore/tools/csi_driver/README.md new file mode 100644 index 00000000000..bd6883bb1d2 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/README.md @@ -0,0 +1,139 @@ +# Install local k8s cluster (using Minikube) + +\# Install kubectl + +```bash +curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg +echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list +sudo apt-get update +sudo apt-get install -y kubectl +``` + +\# Install minikube + +```bash +curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 +sudo chmod +x minikube-linux-amd64 +sudo install minikube-linux-amd64 /usr/local/bin/minikube +``` + +\# Install docker + +```bash +sudo apt-get update +sudo apt-get install -y docker.io +sudo usermod -aG docker $USER && newgrp docker +``` + +\# Start a local k8s cluster + +```bash +minikube start +``` + +\# Check k8s cluster + +```bash +minikube status +kubectl cluster-info +kubectl get nodes +``` + +\# Enable nbd devices +```bash +minikube ssh -- sudo modprobe nbd nbds_max=128 +minikube ssh -- ls -l /dev/ | grep nbd +``` +The last command should show the nbd devices, if not then try to restart minikube +```bash +minikube stop +minikube start +``` + +# Pull docker image with csi-driver on the node + +```bash +minikube ssh +docker pull cr.ai.nebius.cloud/crn0l5t3qnnlbpi8de6q/nbs-csi-driver:v0.1 +exit +``` + +# Setup k8s cluster + +```bash +kubectl apply -f ./deploy/manifests/0-ns.yaml +kubectl apply -f ./deploy/manifests/1-priorityclass.yaml +kubectl apply -f ./deploy/manifests/2-rbac.yaml +kubectl apply -f ./deploy/manifests/3-csidriver.yaml +kubectl apply -f ./deploy/manifests/4-storageclass.yaml +# TODO: kubectl secret nbs-puller-secret +kubectl apply -f ./deploy/manifests/5-deployment.yaml +kubectl apply -f ./deploy/manifests/6-daemonset.yaml +``` + +# copy nbsd and blockstore-nbd to minikube and run them +```bash +sudo ./nbsd-lightweight --service local --server-file ./server/server.txt --verbose +sudo ./blockstore-nbd --device-mode endpoint --disk-id my-disk --access-mode rw --mount-mode local --connect-device /dev/nbd0 --listen-path /tmp/nbd.sock +``` + +# Create test pods (1st way) +# - CSI creates pv on controller; +# - CSI mounts pv on node. + +```bash +kubectl apply -f ./deploy/example/pvc-fs.yaml +kubectl apply -f ./deploy/example/pod-fs.yaml + +kubectl apply -f ./deploy/example/pvc-blk.yaml +kubectl apply -f ./deploy/example/pod-blk.yaml +``` + +# Create test pod (2nd way) +# - CSI mounts pv on node. +# +# Compute-API has to create nbs volume before and passes disk-id to volumeHandle of pv.yaml. +# Also Compute-API can pass any arguments to volumeAttributes of pv.yaml. + +```bash +kubectl apply -f ./deploy/example_pv/pvc.yaml +kubectl apply -f ./deploy/example_pv/pv.yaml +kubectl apply -f ./deploy/example_pv/pod.yaml +``` + +=========================================== + +# Update driver in docker image. + +```bash +# Build docker image +ya make -r ./cmd/nbs-csi-driver +docker build -t cr.ai.nebius.cloud/crn0l5t3qnnlbpi8de6q/nbs-csi-driver:v0.1 . +docker push cr.ai.nebius.cloud/crn0l5t3qnnlbpi8de6q/nbs-csi-driver:v0.1 + +# Check docker image +docker pull cr.ai.nebius.cloud/crn0l5t3qnnlbpi8de6q/nbs-csi-driver:v0.1 +mkdir /tmp/csi +docker run --net host -v /tmp/csi:/csi cr.ai.nebius.cloud/crn0l5t3qnnlbpi8de6q/nbs-csi-driver:v0.1 + +# Update docker image on the node +minikube ssh +docker pull cr.ai.nebius.cloud/crn0l5t3qnnlbpi8de6q/nbs-csi-driver:v0.1 +exit +``` + +# Useful commands + +```bash +kubectl get nodes +kubectl -n nbs-csi-ns get all +kubectl -n nbs-csi-ns get pods +kubectl -n nbs-csi-ns describe node +kubectl -n nbs-csi-ns describe pod/nbs-csi-driver-node-xxxxx +kubectl -n nbs-csi-ns logs pod/nbs-csi-driver-node-xxxxx +kubectl -n nbs-csi-ns delete deployment.apps/nbs-csi-driver-controller +kubectl get CSIDriver +minikube ssh + docker ps + docker exec -it $MYCONTAINER /bin/bash +``` diff --git a/cloud/blockstore/tools/csi_driver/cmd/nbs-csi-driver/main.go b/cloud/blockstore/tools/csi_driver/cmd/nbs-csi-driver/main.go new file mode 100644 index 00000000000..9798cb0ce3b --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/cmd/nbs-csi-driver/main.go @@ -0,0 +1,44 @@ +package main + +import ( + "flag" + "log" + + "github.com/ydb-platform/nbs/cloud/blockstore/tools/csi_driver/internal/driver" +) + +//////////////////////////////////////////////////////////////////////////////// + +func main() { + cfg := driver.Config{} + + flag.StringVar(&cfg.DriverName, "name", "nbs.csi.driver", "Driver name") + flag.StringVar(&cfg.VendorVersion, "version", "devel", "Vendor version") + flag.StringVar(&cfg.Endpoint, "endpoint", "/csi/csi.sock", "CSI endpoint") + flag.StringVar(&cfg.NodeID, "node-id", "undefined", "Node ID") + flag.UintVar(&cfg.NbsPort, "nbs-port", 9766, "NBS port") + flag.UintVar(&cfg.NfsServerPort, "nfs-server-port", 9021, "NFS server port") + flag.UintVar(&cfg.NfsVhostPort, "nfs-vhost-port", 9022, "NFS vhost port") + flag.StringVar(&cfg.NbsSocketsDir, + "nbs-sockets-dir", + "/run/nbsd/sockets", + "Path to folder with disk sockets on the node", + ) + flag.StringVar(&cfg.PodSocketsDir, + "pod-sockets-dir", + "/nbsd-sockets", + "Path to folder with disk sockets on the pod", + ) + + flag.Parse() + + log.Printf("Run NBS CSI driver: %s:%s", cfg.DriverName, cfg.VendorVersion) + + srv, err := driver.NewDriver(cfg) + if err != nil { + panic(err) + } + if err := srv.Run(cfg.Endpoint); err != nil { + panic(err) + } +} diff --git a/cloud/blockstore/tools/csi_driver/cmd/nbs-csi-driver/ya.make b/cloud/blockstore/tools/csi_driver/cmd/nbs-csi-driver/ya.make new file mode 100644 index 00000000000..383aeab5160 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/cmd/nbs-csi-driver/ya.make @@ -0,0 +1,9 @@ +OWNER(g:cloud-nbs) + +GO_PROGRAM(nbs-csi-driver) + +SRCS( + main.go +) + +END() diff --git a/cloud/blockstore/tools/csi_driver/cmd/ya.make b/cloud/blockstore/tools/csi_driver/cmd/ya.make new file mode 100644 index 00000000000..8d324a36850 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/cmd/ya.make @@ -0,0 +1,5 @@ +OWNER(g:cloud-nbs) + +RECURSE( + nbs-csi-driver +) diff --git a/cloud/blockstore/tools/csi_driver/deploy/create-pod.sh b/cloud/blockstore/tools/csi_driver/deploy/create-pod.sh new file mode 100755 index 00000000000..28338803de4 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/create-pod.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +MODE=$1 # fs/blk + +echo "=== create pvc ===" +kubectl apply -f ./example/pvc-$MODE.yaml +echo "=== create pod ===" +kubectl apply -f ./example/pod-$MODE.yaml +echo "=== get all ===" +kubectl get all +echo "=== done ===" diff --git a/cloud/blockstore/tools/csi_driver/deploy/delete-pod.sh b/cloud/blockstore/tools/csi_driver/deploy/delete-pod.sh new file mode 100755 index 00000000000..c7ce8803d71 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/delete-pod.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +MODE=$1 # fs/blk + +echo "=== remove pods ===" +kubectl delete pod/my-pod-$MODE +echo "=== remove pvcs ===" +kubectl delete pvc my-pvc-$MODE +echo "=== get all ===" +kubectl get all +echo "=== done ===" diff --git a/cloud/blockstore/tools/csi_driver/deploy/example/pod-blk.yaml b/cloud/blockstore/tools/csi_driver/deploy/example/pod-blk.yaml new file mode 100644 index 00000000000..b6a4792dedb --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/example/pod-blk.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-blk +spec: + containers: + - name: pause + image: ubuntu:latest + command: + - sleep + - "3600" + volumeDevices: + - name: my-pvc + devicePath: /dev/xvda + volumes: + - name: my-pvc + persistentVolumeClaim: + claimName: my-pvc-blk + readOnly: false diff --git a/cloud/blockstore/tools/csi_driver/deploy/example/pod-fs.yaml b/cloud/blockstore/tools/csi_driver/deploy/example/pod-fs.yaml new file mode 100644 index 00000000000..e270afc047d --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/example/pod-fs.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-fs +spec: + containers: + - name: pause + image: ubuntu:latest + command: + - sleep + - "3600" + volumeMounts: + - name: my-pvc + mountPath: /tmp + volumes: + - name: my-pvc + persistentVolumeClaim: + claimName: my-pvc-fs + readOnly: false diff --git a/cloud/blockstore/tools/csi_driver/deploy/example/pvc-blk.yaml b/cloud/blockstore/tools/csi_driver/deploy/example/pvc-blk.yaml new file mode 100644 index 00000000000..b3dcc73f0d3 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/example/pvc-blk.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-pvc-blk +spec: + accessModes: + - ReadWriteOnce + volumeMode: Block + resources: + requests: + storage: 1Gi + limits: + storage: 1Gi + storageClassName: nbs-csi-sc diff --git a/cloud/blockstore/tools/csi_driver/deploy/example/pvc-fs.yaml b/cloud/blockstore/tools/csi_driver/deploy/example/pvc-fs.yaml new file mode 100644 index 00000000000..f95c849fbd6 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/example/pvc-fs.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-pvc-fs +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 1Gi + limits: + storage: 1Gi + storageClassName: nbs-csi-sc diff --git a/cloud/blockstore/tools/csi_driver/deploy/example_pv/pod.yaml b/cloud/blockstore/tools/csi_driver/deploy/example_pv/pod.yaml new file mode 100644 index 00000000000..5bb351ba097 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/example_pv/pod.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - name: pause + image: ubuntu:latest + command: + - sleep + - "3600" + volumeMounts: + - name: my-volume + mountPath: /tmp + volumes: + - name: my-volume + persistentVolumeClaim: + claimName: my-pvc + readOnly: false \ No newline at end of file diff --git a/cloud/blockstore/tools/csi_driver/deploy/example_pv/pv.yaml b/cloud/blockstore/tools/csi_driver/deploy/example_pv/pv.yaml new file mode 100644 index 00000000000..52309a43c00 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/example_pv/pv.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: my-pv +spec: + volumeMode: Filesystem + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: nbs-csi-sc + claimRef: + namespace: nbs-csi-ns + name: my-pvc + csi: + driver: nbs.csi.nebius.ai + volumeHandle: my-nbs-volume-id + volumeAttributes: + key1: value1 + key2: value2 diff --git a/cloud/blockstore/tools/csi_driver/deploy/example_pv/pvc.yaml b/cloud/blockstore/tools/csi_driver/deploy/example_pv/pvc.yaml new file mode 100644 index 00000000000..f22b624328b --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/example_pv/pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-pvc +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 1Gi + limits: + storage: 1Gi + storageClassName: nbs-csi-sc + volumeName: my-pv diff --git a/cloud/blockstore/tools/csi_driver/deploy/manifests/0-ns.yaml b/cloud/blockstore/tools/csi_driver/deploy/manifests/0-ns.yaml new file mode 100644 index 00000000000..c3dd29b3d1c --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/manifests/0-ns.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nbs-csi-ns diff --git a/cloud/blockstore/tools/csi_driver/deploy/manifests/1-priorityclass.yaml b/cloud/blockstore/tools/csi_driver/deploy/manifests/1-priorityclass.yaml new file mode 100644 index 00000000000..7e27a9b707e --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/manifests/1-priorityclass.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: nbs-csi-controller +value: 900000000 +globalDefault: false +description: "This priority class should be used for core kubevirt components only." + +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: nbs-csi-node +value: 900001000 +globalDefault: false +description: "This priority class should be used for core kubevirt components only." diff --git a/cloud/blockstore/tools/csi_driver/deploy/manifests/2-rbac.yaml b/cloud/blockstore/tools/csi_driver/deploy/manifests/2-rbac.yaml new file mode 100644 index 00000000000..96977c587b6 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/manifests/2-rbac.yaml @@ -0,0 +1,264 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nbs-csi-node-sa + namespace: nbs-csi-ns + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nbs-csi-controller-sa + namespace: nbs-csi-ns + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nbs-csi-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + # Access to volumeattachments is only needed when the CSI driver + # has the PUBLISH_UNPUBLISH_VOLUME controller capability. + # In that case, external-provisioner will watch volumeattachments + # to determine when it is safe to delete a volume. + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nbs-csi-controller-provisioner-binding +subjects: + - kind: ServiceAccount + name: nbs-csi-controller-sa + namespace: nbs-csi-ns +roleRef: + kind: ClusterRole + name: nbs-csi-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nbs-csi-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nbs-csi-controller-attacher-binding +subjects: + - kind: ServiceAccount + name: nbs-csi-controller-sa + namespace: nbs-csi-ns +roleRef: + kind: ClusterRole + name: nbs-csi-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- +# Resizer must be able to work with PVCs, PVs, SCs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nbs-csi-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + # If handle-volume-inuse-error=true, the pod specific rbac is needed + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nbs-csi-controller-resizer-binding +subjects: + - kind: ServiceAccount + name: nbs-csi-controller-sa + namespace: nbs-csi-ns +roleRef: + kind: ClusterRole + name: nbs-csi-resizer-role + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nbs-csi-controller-deploy-role +rules: + - apiGroups: ["policy"] + resources: ["podsecuritypolicies"] + verbs: ["use"] + resourceNames: + - nbs-csi-controller-psp + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nbs-csi-controller-deploy-binding +subjects: + - kind: ServiceAccount + name: nbs-csi-controller-sa + namespace: nbs-csi-ns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nbs-csi-controller-deploy-role + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nbs-csi-node-deploy-role +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - nbs-csi-node-psp + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nbs-csi-node-node-deploy-binding +subjects: + - kind: ServiceAccount + name: nbs-csi-node-sa + namespace: nbs-csi-ns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nbs-csi-node-deploy-role + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nbs-csi-controller-node-deploy-binding +subjects: + - kind: ServiceAccount + name: nbs-csi-controller-sa + namespace: nbs-csi-ns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nbs-csi-node-deploy-role + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nbs-csi-snapshotter-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + # Secrets resource omitted since GCE PD snapshots does not require them + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nbs-csi-controller-snapshotter-binding +subjects: + - kind: ServiceAccount + name: nbs-csi-controller-sa + namespace: nbs-csi-ns +roleRef: + kind: ClusterRole + name: nbs-csi-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nbs-csi-leaderelection-role + namespace: nbs-csi-ns +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nbs-csi-controller-leaderelection-binding +subjects: + - kind: ServiceAccount + name: nbs-csi-controller-sa + namespace: nbs-csi-ns +roleRef: + kind: ClusterRole + name: nbs-csi-leaderelection-role + apiGroup: rbac.authorization.k8s.io diff --git a/cloud/blockstore/tools/csi_driver/deploy/manifests/3-csidriver.yaml b/cloud/blockstore/tools/csi_driver/deploy/manifests/3-csidriver.yaml new file mode 100644 index 00000000000..348e21ddf7e --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/manifests/3-csidriver.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: nbs.csi.nebius.ai +spec: + attachRequired: false diff --git a/cloud/blockstore/tools/csi_driver/deploy/manifests/4-storageclass.yaml b/cloud/blockstore/tools/csi_driver/deploy/manifests/4-storageclass.yaml new file mode 100644 index 00000000000..ab13dd346f5 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/manifests/4-storageclass.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: nbs-csi-sc +provisioner: nbs.csi.nebius.ai +volumeBindingMode: Immediate diff --git a/cloud/blockstore/tools/csi_driver/deploy/manifests/5-deployment.yaml b/cloud/blockstore/tools/csi_driver/deploy/manifests/5-deployment.yaml new file mode 100644 index 00000000000..8045f16b53d --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/manifests/5-deployment.yaml @@ -0,0 +1,129 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nbs-csi-driver-controller + namespace: nbs-csi-ns +spec: + replicas: 1 + selector: + matchLabels: + app: "nbs-csi-controller-app" + template: + metadata: + labels: + app: "nbs-csi-controller-app" + spec: + hostNetwork: true + serviceAccountName: nbs-csi-controller-sa + nodeSelector: + kubernetes.io/os: linux + priorityClassName: nbs-csi-controller + containers: + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.2 + imagePullPolicy: IfNotPresent + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--feature-gates=Topology=true" + - "--http-endpoint=:22011" + - "--leader-election-namespace=$(NBS_CSI_NAMESPACE)" + - "--timeout=250s" + - "--leader-election" + env: + - name: NBS_CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 22011 + name: http-endpoint + protocol: TCP + livenessProbe: + failureThreshold: 1 + httpGet: + path: /healthz/leader-election + port: http-endpoint + initialDelaySeconds: 10 + timeoutSeconds: 10 + periodSeconds: 20 + resources: + requests: + memory: "64Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "250m" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:v4.4.2 + imagePullPolicy: IfNotPresent + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--http-endpoint=:22012" + - "--leader-election" + - "--leader-election-namespace=$(NBS_CSI_NAMESPACE)" + - "--timeout=250s" + env: + - name: NBS_CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 22012 + name: http-endpoint + protocol: TCP + livenessProbe: + failureThreshold: 1 + httpGet: + path: /healthz/leader-election + port: http-endpoint + initialDelaySeconds: 10 + timeoutSeconds: 10 + periodSeconds: 20 + resources: + requests: + memory: "64Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "250m" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-nbs-driver + image: cr.ai.nebius.cloud/crn0l5t3qnnlbpi8de6q/nbs-csi-driver:v0.1 + imagePullPolicy: Always + args: + - "--name=nbs.csi.nebius.ai" + - "--version=$(DRIVER_VERSION)" + - "--node-id=$(KUBE_NODE_NAME)" + - "--endpoint=/csi/csi.sock" + - "--nfs-server-port=0" + env: + - name: DRIVER_VERSION + value: "v0.1" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + resources: + requests: + memory: "64Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "250m" + volumeMounts: + - name: socket-dir + mountPath: /csi + imagePullSecrets: + - name: nbs-puller-secret + volumes: + - name: socket-dir + emptyDir: {} diff --git a/cloud/blockstore/tools/csi_driver/deploy/manifests/6-daemonset.yaml b/cloud/blockstore/tools/csi_driver/deploy/manifests/6-daemonset.yaml new file mode 100644 index 00000000000..fb9f9936e2e --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/manifests/6-daemonset.yaml @@ -0,0 +1,99 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nbs-csi-driver-node + namespace: nbs-csi-ns +spec: + selector: + matchLabels: + app: "nbs-csi-node-app" + template: + metadata: + labels: + app: "nbs-csi-node-app" + spec: + hostNetwork: true + serviceAccountName: nbs-csi-node-sa + nodeSelector: + kubernetes.io/os: linux + containers: + - name: csi-node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.2 + imagePullPolicy: IfNotPresent + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/nbs.csi.nebius.ai/csi.sock" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + memory: "64Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "250m" + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: csi-nbs-driver + securityContext: + privileged: true + image: cr.ai.nebius.cloud/crn0l5t3qnnlbpi8de6q/nbs-csi-driver:v0.1 + imagePullPolicy: Always + args: + - "--name=nbs.csi.nebius.ai" + - "--version=$(DRIVER_VERSION)" + - "--node-id=$(KUBE_NODE_NAME)" + - "--endpoint=/csi/csi.sock" + - "--nbs-sockets-dir=/run/nbsd/sockets" + - "--pod-sockets-dir=/nbsd-sockets" + - "--nfs-vhost-port=0" + env: + - name: DRIVER_VERSION + value: "v0.1" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + resources: + requests: + memory: "64Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "250m" + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: sockets-dir + mountPath: /nbsd-sockets + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/nbs.csi.nebius.ai/ + type: DirectoryOrCreate + - name: sockets-dir + hostPath: + path: /run/nbsd/sockets + type: DirectoryOrCreate + imagePullSecrets: + - name: nbs-puller-secret diff --git a/cloud/blockstore/tools/csi_driver/deploy/redeploy.sh b/cloud/blockstore/tools/csi_driver/deploy/redeploy.sh new file mode 100755 index 00000000000..3b9063a6537 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/deploy/redeploy.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +echo "=== remove driver-node ===" +kubectl -n nbs-csi-ns delete daemonset.apps/nbs-csi-driver-node +echo "=== remove driver-controller ===" +kubectl -n nbs-csi-ns delete deployment.apps/nbs-csi-driver-controller +echo "=== get all ===" +kubectl -n nbs-csi-ns get all + +echo "=== update driver ===" +minikube ssh -- docker pull cr.ai.nebius.cloud/crn0l5t3qnnlbpi8de6q/nbs-csi-driver:v0.1 + +echo "=== create driver-controller ===" +kubectl -n nbs-csi-ns apply -f ./manifests/5-deployment.yaml +echo "=== create driver-node ===" +kubectl -n nbs-csi-ns apply -f ./manifests/6-daemonset.yaml +echo "=== get all ===" +kubectl -n nbs-csi-ns get all + +echo "=== done ===" diff --git a/cloud/blockstore/tools/csi_driver/internal/driver/controller.go b/cloud/blockstore/tools/csi_driver/internal/driver/controller.go new file mode 100644 index 00000000000..741a7f91320 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/internal/driver/controller.go @@ -0,0 +1,297 @@ +package driver + +import ( + "context" + "log" + + "github.com/container-storage-interface/spec/lib/go/csi" + nbsapi "github.com/ydb-platform/nbs/cloud/blockstore/public/api/protos" + nbsclient "github.com/ydb-platform/nbs/cloud/blockstore/public/sdk/go/client" + nfsapi "github.com/ydb-platform/nbs/cloud/filestore/public/api/protos" + nfsclient "github.com/ydb-platform/nbs/cloud/filestore/public/sdk/go/client" + storagecoreapi "github.com/ydb-platform/nbs/cloud/storage/core/protos" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +//////////////////////////////////////////////////////////////////////////////// + +const diskBlockSize uint32 = 4 * 1024 + +var nbsServerControllerServiceCapabilities = []*csi.ControllerServiceCapability{ + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + }, + }, + }, + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, + }, + }, + }, +} + +//////////////////////////////////////////////////////////////////////////////// + +type nbsServerControllerService struct { + csi.ControllerServer + + nbsClient nbsclient.ClientIface + nfsClient nfsclient.ClientIface +} + +func newNBSServerControllerService( + nbsClient nbsclient.ClientIface, + nfsClient nfsclient.ClientIface) csi.ControllerServer { + + return &nbsServerControllerService{nbsClient: nbsClient} +} + +func (c *nbsServerControllerService) CreateVolume( + ctx context.Context, + req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + + log.Printf("csi.CreateVolumeRequest: %+v", req) + + if req.Name == "" { + return nil, status.Error( + codes.InvalidArgument, + "Name missing in CreateVolumeRequest") + } + + if req.VolumeCapabilities == nil { + return nil, status.Error( + codes.InvalidArgument, + "VolumeCapabilities missing in CreateVolumeRequest") + } + + var requiredBytes int64 = int64(diskBlockSize) + if req.CapacityRange != nil { + if req.CapacityRange.RequiredBytes < 0 { + return nil, status.Error( + codes.InvalidArgument, + "RequiredBytes must not be negative in CreateVolumeRequest") + } + requiredBytes = req.CapacityRange.RequiredBytes + } + + if uint64(requiredBytes)%uint64(diskBlockSize) != 0 { + return nil, status.Errorf( + codes.InvalidArgument, + "incorrect value: required bytes %d, block size: %d", + requiredBytes, + diskBlockSize, + ) + } + + volumeContext := make(map[string]string) + parameters := make(map[string]string) + if req.Parameters != nil { + for key, value := range req.Parameters { + if key == "backend" { + volumeContext[key] = value + } else { + parameters[key] = value + } + } + } + + var err error + if volumeContext["backend"] == "nfs" { + err = c.createFileStore(ctx, req.Name, requiredBytes) + } else { + err = c.createDisk(ctx, req.Name, requiredBytes, parameters) + } + + if err != nil { + // TODO (issues/464): return codes.AlreadyExists if volume exists + return nil, status.Errorf( + codes.Internal, "Failed to create volume: %+v", err) + } + + return &csi.CreateVolumeResponse{Volume: &csi.Volume{ + CapacityBytes: requiredBytes, + VolumeId: req.Name, + VolumeContext: volumeContext, + }}, nil +} + +func (c *nbsServerControllerService) createDisk( + ctx context.Context, + diskId string, + requiredBytes int64, + parameters map[string]string) error { + + _, err := c.nbsClient.CreateVolume(ctx, &nbsapi.TCreateVolumeRequest{ + DiskId: diskId, + BlockSize: diskBlockSize, + BlocksCount: uint64(requiredBytes) / uint64(diskBlockSize), + StorageMediaKind: storagecoreapi.EStorageMediaKind_STORAGE_MEDIA_SSD, + BaseDiskId: parameters["base-disk-id"], + BaseDiskCheckpointId: parameters["base-disk-checkpoint-id"], + }) + return err +} + +func (c *nbsServerControllerService) createFileStore( + ctx context.Context, + fileSystemId string, + requiredBytes int64) error { + + if c.nfsClient == nil { + return status.Errorf(codes.Internal, "NFS client wasn't created") + } + + _, err := c.nfsClient.CreateFileStore(ctx, &nfsapi.TCreateFileStoreRequest{ + FileSystemId: fileSystemId, + BlockSize: diskBlockSize, + BlocksCount: uint64(requiredBytes) / uint64(diskBlockSize), + StorageMediaKind: storagecoreapi.EStorageMediaKind_STORAGE_MEDIA_SSD, + }) + return err +} + +func (c *nbsServerControllerService) DeleteVolume( + ctx context.Context, + req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + + log.Printf("csi.DeleteVolumeRequest: %+v", req) + + if req.VolumeId == "" { + return nil, status.Error( + codes.InvalidArgument, + "VolumeId missing in DeleteVolumeRequest") + } + + // Trying to destroy both disk and filestore, + // because the resource's type is unknown here. + // When we miss we get S_FALSE/S_ALREADY code (err == nil). + + _, err := c.nbsClient.DestroyVolume(ctx, &nbsapi.TDestroyVolumeRequest{ + DiskId: req.VolumeId, + }) + if err != nil { + return nil, status.Errorf( + codes.Internal, + "Failed to destroy disk: %+v", err) + } + + if c.nfsClient != nil { + _, err = c.nfsClient.DestroyFileStore(ctx, &nfsapi.TDestroyFileStoreRequest{ + FileSystemId: req.VolumeId, + }) + if err != nil { + return nil, status.Errorf( + codes.Internal, + "Failed to destroy filestore: %+v", err) + } + } + + return &csi.DeleteVolumeResponse{}, nil +} + +func (c *nbsServerControllerService) ControllerPublishVolume( + ctx context.Context, + req *csi.ControllerPublishVolumeRequest, +) (*csi.ControllerPublishVolumeResponse, error) { + + log.Printf("csi.ControllerPublishVolumeRequest: %+v", req) + + if req.VolumeId == "" { + return nil, status.Error( + codes.InvalidArgument, + "VolumeId missing in ControllerPublishVolumeRequest") + } + if req.NodeId == "" { + return nil, status.Error( + codes.InvalidArgument, + "NodeId missing in ControllerPublishVolumeRequest") + } + if req.VolumeCapability == nil { + return nil, status.Error( + codes.InvalidArgument, + "VolumeCapability missing in ControllerPublishVolumeRequest") + } + + if !c.doesVolumeExist(ctx, req.VolumeId) { + return nil, status.Errorf( + codes.NotFound, "Volume %q does not exist", req.VolumeId) + } + + // TODO (issues/464): check if req.NodeId exists in the cluster + + return &csi.ControllerPublishVolumeResponse{}, nil +} + +func (c *nbsServerControllerService) ControllerUnpublishVolume( + ctx context.Context, + req *csi.ControllerUnpublishVolumeRequest, +) (*csi.ControllerUnpublishVolumeResponse, error) { + + log.Printf("csi.ControllerUnpublishVolumeRequest: %+v", req) + + if req.VolumeId == "" { + return nil, status.Error( + codes.InvalidArgument, + "VolumeId missing in ControllerUnpublishVolumeRequest") + } + + return &csi.ControllerUnpublishVolumeResponse{}, nil +} + +func (c *nbsServerControllerService) ValidateVolumeCapabilities( + ctx context.Context, + req *csi.ValidateVolumeCapabilitiesRequest, +) (*csi.ValidateVolumeCapabilitiesResponse, error) { + + log.Printf("csi.ValidateVolumeCapabilities: %+v", req) + + if req.VolumeId == "" { + return nil, status.Error( + codes.InvalidArgument, + "VolumeId missing in ValidateVolumeCapabilitiesRequest") + } + if req.VolumeCapabilities == nil { + return nil, status.Error( + codes.InvalidArgument, + "VolumeCapabilities missing in ValidateVolumeCapabilitiesRequest") + } + + if !c.doesVolumeExist(ctx, req.VolumeId) { + return nil, status.Errorf( + codes.NotFound, "Volume %q does not exist", req.VolumeId) + } + + return &csi.ValidateVolumeCapabilitiesResponse{}, nil +} + +func (c *nbsServerControllerService) ControllerGetCapabilities( + ctx context.Context, + req *csi.ControllerGetCapabilitiesRequest, +) (*csi.ControllerGetCapabilitiesResponse, error) { + + return &csi.ControllerGetCapabilitiesResponse{ + Capabilities: nbsServerControllerServiceCapabilities, + }, nil +} + +func (c *nbsServerControllerService) doesVolumeExist( + ctx context.Context, + volumeId string) bool { + + describeVolumeRequest := &nbsapi.TDescribeVolumeRequest{ + DiskId: volumeId, + } + + _, err := c.nbsClient.DescribeVolume(ctx, describeVolumeRequest) + if err != nil { + // TODO (issues/464): check error code + return false + } + + return true +} diff --git a/cloud/blockstore/tools/csi_driver/internal/driver/driver.go b/cloud/blockstore/tools/csi_driver/internal/driver/driver.go new file mode 100644 index 00000000000..d4ba9e675da --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/internal/driver/driver.go @@ -0,0 +1,154 @@ +package driver + +import ( + "context" + "fmt" + "net" + "os" + "os/signal" + "sync" + "syscall" + + "github.com/container-storage-interface/spec/lib/go/csi" + log "github.com/sirupsen/logrus" + nbsclient "github.com/ydb-platform/nbs/cloud/blockstore/public/sdk/go/client" + nfsclient "github.com/ydb-platform/nbs/cloud/filestore/public/sdk/go/client" + "google.golang.org/grpc" +) + +//////////////////////////////////////////////////////////////////////////////// + +type Config struct { + DriverName string + Endpoint string + NodeID string + VendorVersion string + NbsPort uint + NfsServerPort uint + NfsVhostPort uint + NbsSocketsDir string + PodSocketsDir string +} + +//////////////////////////////////////////////////////////////////////////////// + +type Driver struct { + grpcServer *grpc.Server +} + +func NewDriver(cfg Config) (*Driver, error) { + nbsClientID := fmt.Sprintf("%s-%s", cfg.DriverName, cfg.NodeID) + nbsClient, err := nbsclient.NewGrpcClient( + &nbsclient.GrpcClientOpts{ + Endpoint: fmt.Sprintf("localhost:%d", cfg.NbsPort), + ClientId: nbsClientID, + }, nbsclient.NewStderrLog(nbsclient.LOG_DEBUG), + ) + if err != nil { + return nil, err + } + + var nfsClient nfsclient.ClientIface + if cfg.NfsServerPort != 0 { + nfsClient, err = nfsclient.NewGrpcClient( + &nfsclient.GrpcClientOpts{ + Endpoint: fmt.Sprintf("localhost:%d", cfg.NfsServerPort), + }, nfsclient.NewStderrLog(nfsclient.LOG_DEBUG), + ) + if err != nil { + return nil, err + } + } + + var nfsEndpointClient nfsclient.EndpointClientIface + if cfg.NfsVhostPort != 0 { + nfsEndpointClient, err = nfsclient.NewGrpcEndpointClient( + &nfsclient.GrpcClientOpts{ + Endpoint: fmt.Sprintf("localhost:%d", cfg.NfsVhostPort), + }, nfsclient.NewStderrLog(nfsclient.LOG_DEBUG), + ) + if err != nil { + return nil, err + } + } + + errInterceptor := func( + ctx context.Context, + req interface{}, + info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler) (interface{}, error) { + + resp, err := handler(ctx, req) + if err != nil { + log.WithError(err).WithField("method", info.FullMethod).Error("method failed") + } + return resp, err + } + + grpcServer := grpc.NewServer(grpc.UnaryInterceptor(errInterceptor)) + + csi.RegisterIdentityServer( + grpcServer, + newIdentityService(cfg.DriverName, cfg.VendorVersion)) + + csi.RegisterControllerServer( + grpcServer, + newNBSServerControllerService(nbsClient, nfsClient)) + + csi.RegisterNodeServer( + grpcServer, + newNodeService( + cfg.NodeID, + nbsClientID, + cfg.NbsSocketsDir, + cfg.PodSocketsDir, + nbsClient, + nfsEndpointClient)) + + return &Driver{grpcServer: grpcServer}, nil +} + +func (s *Driver) Run(socketPath string) error { + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + return err + } + + listener, err := net.Listen("unix", socketPath) + if err != nil { + return fmt.Errorf("failed to listen: %w", err) + } + + signalChannel := make(chan os.Signal, 1) + signal.Notify(signalChannel, syscall.SIGINT, syscall.SIGTERM) + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGPIPE) + + var wg sync.WaitGroup + defer wg.Wait() + + defer s.grpcServer.GracefulStop() + + done := make(chan error, 1) + + wg.Add(1) + go func() { + defer wg.Done() + defer log.Print("Server finished") + + log.Print("Server started") + if err := s.grpcServer.Serve(listener); err != nil { + done <- err + } + }() + + select { + case err := <-done: + if err != nil { + log.Printf("Server failed: %+v", err) + } + case sig := <-signalChannel: + log.Printf("Got unix signal %q", sig) + } + + return nil +} diff --git a/cloud/blockstore/tools/csi_driver/internal/driver/identity.go b/cloud/blockstore/tools/csi_driver/internal/driver/identity.go new file mode 100644 index 00000000000..5a62e8c69b9 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/internal/driver/identity.go @@ -0,0 +1,52 @@ +package driver + +import ( + "context" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +var pluginCapabilities = []*csi.PluginCapability{ + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, + }, + }, + }, + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS, + }, + }, + }, +} + +type identity struct { + driverName, driverVersion string +} + +func newIdentityService(driverName, driverVersion string) csi.IdentityServer { + return &identity{driverName: driverName, driverVersion: driverVersion} +} + +func (i *identity) GetPluginInfo(context.Context, *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { + return &csi.GetPluginInfoResponse{ + Name: i.driverName, + VendorVersion: i.driverVersion, + }, nil +} + +func (i *identity) GetPluginCapabilities(context.Context, *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { + return &csi.GetPluginCapabilitiesResponse{Capabilities: pluginCapabilities}, nil +} + +func (*identity) Probe(context.Context, *csi.ProbeRequest) (*csi.ProbeResponse, error) { + return &csi.ProbeResponse{ + Ready: &wrapperspb.BoolValue{ + Value: true, + }, + }, nil +} diff --git a/cloud/blockstore/tools/csi_driver/internal/driver/node.go b/cloud/blockstore/tools/csi_driver/internal/driver/node.go new file mode 100644 index 00000000000..df3185c0548 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/internal/driver/node.go @@ -0,0 +1,457 @@ +package driver + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/container-storage-interface/spec/lib/go/csi" + nbsapi "github.com/ydb-platform/nbs/cloud/blockstore/public/api/protos" + nbsclient "github.com/ydb-platform/nbs/cloud/blockstore/public/sdk/go/client" + nfsapi "github.com/ydb-platform/nbs/cloud/filestore/public/api/protos" + nfsclient "github.com/ydb-platform/nbs/cloud/filestore/public/sdk/go/client" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +//////////////////////////////////////////////////////////////////////////////// + +const topologyKeyNode = "topology.nbs.csi/node" + +const socketName = "nbs-volume.sock" + +var capabilities = []*csi.NodeServiceCapability{ + &csi.NodeServiceCapability{ + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, + }, + }, + }, +} + +//////////////////////////////////////////////////////////////////////////////// + +type nodeService struct { + csi.NodeServer + + nodeID string + clientID string + nbsSocketsDir string + podSocketsDir string + nbsClient nbsclient.ClientIface + nfsClient nfsclient.EndpointClientIface +} + +func newNodeService( + nodeID string, + clientID string, + nbsSocketsDir string, + podSocketsDir string, + nbsClient nbsclient.ClientIface, + nfsClient nfsclient.EndpointClientIface) csi.NodeServer { + + return &nodeService{ + nodeID: nodeID, + clientID: clientID, + nbsSocketsDir: nbsSocketsDir, + podSocketsDir: podSocketsDir, + nbsClient: nbsClient, + nfsClient: nfsClient, + } +} + +func (s *nodeService) NodeStageVolume( + ctx context.Context, + req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { + + log.Printf("csi.NodeStageVolumeRequest: %+v", req) + + if req.VolumeId == "" { + return nil, status.Error( + codes.InvalidArgument, + "VolumeId missing in NodeStageVolumeRequest") + } + if req.StagingTargetPath == "" { + return nil, status.Error( + codes.InvalidArgument, + "StagingTargetPath missing in NodeStageVolumeRequest") + } + if req.VolumeCapability == nil { + return nil, status.Error( + codes.InvalidArgument, + "VolumeCapability missing im NodeStageVolumeRequest") + } + + return &csi.NodeStageVolumeResponse{}, nil +} + +func (s *nodeService) NodeUnstageVolume( + ctx context.Context, + req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { + + log.Printf("csi.NodeUnstageVolumeRequest: %+v", req) + + if req.VolumeId == "" { + return nil, status.Error( + codes.InvalidArgument, + "VolumeId missing in NodeUnstageVolumeRequest") + } + if req.StagingTargetPath == "" { + return nil, status.Error( + codes.InvalidArgument, + "StagingTargetPath missing in NodeUnstageVolumeRequest") + } + + return &csi.NodeUnstageVolumeResponse{}, nil +} + +func (s *nodeService) NodePublishVolume( + ctx context.Context, + req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + + log.Printf("csi.NodePublishVolumeRequest: %+v", req) + + if req.VolumeId == "" { + return nil, status.Error( + codes.InvalidArgument, + "VolumeId missing in NodePublishVolumeRequest") + } + if req.VolumeCapability == nil { + return nil, status.Error( + codes.InvalidArgument, + "VolumeCapability missing im NodePublishVolumeRequest") + } + + var err error + nfsBackend := (req.VolumeContext != nil && req.VolumeContext["backend"] == "nfs") + + switch req.VolumeCapability.GetAccessType().(type) { + case *csi.VolumeCapability_Mount: + if nfsBackend { + err = s.nodePublishFileStoreAsVhostSocket(ctx, req) + } else { + err = s.nodePublishDiskAsVhostSocket(ctx, req) + } + case *csi.VolumeCapability_Block: + if nfsBackend { + err = status.Error(codes.InvalidArgument, + "'Block' volume mode is not supported for nfs backend") + } else { + err = s.nodePublishDiskAsBlockDevice(ctx, req) + } + default: + err = status.Error(codes.InvalidArgument, "Unknown access type") + } + + if err != nil { + return nil, err + } + + return &csi.NodePublishVolumeResponse{}, nil +} + +func (s *nodeService) NodeUnpublishVolume( + ctx context.Context, + req *csi.NodeUnpublishVolumeRequest, +) (*csi.NodeUnpublishVolumeResponse, error) { + + log.Printf("csi.NodeUnpublishVolumeRequest: %+v", req) + + if req.VolumeId == "" { + return nil, status.Error( + codes.InvalidArgument, + "Volume ID missing in NodeUnpublishVolumeRequest") + } + if req.TargetPath == "" { + return nil, status.Error( + codes.InvalidArgument, + "Target Path missing in NodeUnpublishVolumeRequest") + } + + if err := s.nodeUnpublishVolume(ctx, req); err != nil { + return nil, err + } + + return &csi.NodeUnpublishVolumeResponse{}, nil +} + +func (s *nodeService) NodeGetCapabilities( + ctx context.Context, + req *csi.NodeGetCapabilitiesRequest, +) (*csi.NodeGetCapabilitiesResponse, error) { + + return &csi.NodeGetCapabilitiesResponse{ + Capabilities: capabilities, + }, nil +} + +func (s *nodeService) NodeGetInfo( + ctx context.Context, + req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { + + return &csi.NodeGetInfoResponse{ + NodeId: s.nodeID, + AccessibleTopology: &csi.Topology{ + Segments: map[string]string{topologyKeyNode: s.nodeID}, + }, + }, nil +} + +func (s *nodeService) nodePublishDiskAsVhostSocket( + ctx context.Context, + req *csi.NodePublishVolumeRequest) error { + + _, err := s.startNbsEndpoint(ctx, req, nbsapi.EClientIpcType_IPC_VHOST) + if err != nil { + return status.Errorf(codes.Internal, + "Failed to start NBS endpoint: %+v", err) + } + + return s.mountSocketDir(req) +} + +func (s *nodeService) nodePublishDiskAsBlockDevice( + ctx context.Context, + req *csi.NodePublishVolumeRequest) error { + + resp, err := s.startNbsEndpoint(ctx, req, nbsapi.EClientIpcType_IPC_NBD) + if err != nil { + return status.Errorf(codes.Internal, + "Failed to start NBS endpoint: %+v", err) + } + + if resp.NbdDeviceFile != "" { + log.Printf("Endpoint started with device file: %q", resp.NbdDeviceFile) + } + + return s.mountBlockDevice(resp.NbdDeviceFile, req.TargetPath) +} + +func (s *nodeService) startNbsEndpoint( + ctx context.Context, + req *csi.NodePublishVolumeRequest, + ipcType nbsapi.EClientIpcType) (*nbsapi.TStartEndpointResponse, error) { + + endpointDir := filepath.Join(s.podSocketsDir, req.VolumeId) + if err := os.MkdirAll(endpointDir, 0777); err != nil { + return nil, err + } + + socketPath := filepath.Join(s.nbsSocketsDir, req.VolumeId, socketName) + hostType := nbsapi.EHostType_HOST_TYPE_DEFAULT + return s.nbsClient.StartEndpoint(ctx, &nbsapi.TStartEndpointRequest{ + UnixSocketPath: socketPath, + DiskId: req.VolumeId, + ClientId: s.clientID, + DeviceName: req.VolumeId, + IpcType: ipcType, + VhostQueuesCount: 8, + VolumeAccessMode: nbsapi.EVolumeAccessMode_VOLUME_ACCESS_READ_WRITE, + VolumeMountMode: nbsapi.EVolumeMountMode_VOLUME_MOUNT_REMOTE, + Persistent: true, + NbdDevice: &nbsapi.TStartEndpointRequest_UseFreeNbdDeviceFile{ + ipcType == nbsapi.EClientIpcType_IPC_NBD, + }, + ClientProfile: &nbsapi.TClientProfile{ + HostType: &hostType, + }, + }) +} + +func (s *nodeService) nodePublishFileStoreAsVhostSocket( + ctx context.Context, + req *csi.NodePublishVolumeRequest) error { + + endpointDir := filepath.Join(s.podSocketsDir, req.VolumeId) + if err := os.MkdirAll(endpointDir, 0777); err != nil { + return err + } + + if s.nfsClient == nil { + return status.Errorf(codes.Internal, "NFS client wasn't created") + } + + socketPath := filepath.Join(s.nbsSocketsDir, req.VolumeId, socketName) + _, err := s.nfsClient.StartEndpoint(ctx, &nfsapi.TStartEndpointRequest{ + Endpoint: &nfsapi.TEndpointConfig{ + SocketPath: socketPath, + FileSystemId: req.VolumeId, + ClientId: s.clientID, + VhostQueuesCount: 8, + Persistent: true, + }, + }) + if err != nil { + return status.Errorf(codes.Internal, + "Failed to start NFS endpoint: %+v", err) + } + + return s.mountSocketDir(req) +} + +func (s *nodeService) nodeUnpublishVolume( + ctx context.Context, + req *csi.NodeUnpublishVolumeRequest) error { + + // Trying to stop both NBS and NFS endpoints, + // because the endpoint's backend service is unknown here. + // When we miss we get S_FALSE/S_ALREADY code (err == nil). + + socketPath := filepath.Join(s.nbsSocketsDir, req.VolumeId, socketName) + _, err := s.nbsClient.StopEndpoint(ctx, &nbsapi.TStopEndpointRequest{ + UnixSocketPath: socketPath, + }) + if err != nil { + return status.Errorf(codes.Internal, + "Failed to stop nbs endpoint: %+v", err) + } + + if s.nfsClient != nil { + _, err = s.nfsClient.StopEndpoint(ctx, &nfsapi.TStopEndpointRequest{ + SocketPath: socketPath, + }) + if err != nil { + return status.Errorf(codes.Internal, + "Failed to stop nfs endpoint: %+v", err) + } + } + + endpointDir := filepath.Join(s.podSocketsDir, req.VolumeId) + if err := os.RemoveAll(endpointDir); err != nil { + return err + } + + if err := s.unmount(req.TargetPath); err != nil { + return err + } + + if err := os.RemoveAll(req.TargetPath); err != nil { + return err + } + + return nil +} + +func (s *nodeService) mountSocketDir(req *csi.NodePublishVolumeRequest) error { + + endpointDir := filepath.Join(s.podSocketsDir, req.VolumeId) + + podSocketPath := filepath.Join(endpointDir, socketName) + if err := os.Chmod(podSocketPath, 0666); err != nil { + return err + } + + // https://kubevirt.io/user-guide/virtual_machines/disks_and_volumes/#persistentvolumeclaim + // "If the disk.img image file has not been created manually before starting a VM + // then it will be created automatically with the PersistentVolumeClaim size." + // So, let's create an empty disk.img to avoid automatic creation and save disk space. + diskImgPath := filepath.Join(endpointDir, "disk.img") + file, err := os.OpenFile(diskImgPath, os.O_CREATE, 0660) + if err != nil { + return status.Errorf(codes.Internal, "Failed to create disk.img: %+v", err) + } + file.Close() + + source := endpointDir + target := req.TargetPath + + fsType := "ext4" + mountOptions := []string{"bind"} + + mnt := req.VolumeCapability.GetMount() + if mnt != nil { + for _, flag := range mnt.MountFlags { + mountOptions = append(mountOptions, flag) + } + + if mnt.FsType != "" { + fsType = mnt.FsType + } + } + + return s.mount(source, target, fsType, mountOptions...) +} + +func (s *nodeService) mountBlockDevice(source string, target string) error { + mountOptions := []string{"bind"} + return s.mount(source, target, "", mountOptions...) +} + +func (s *nodeService) mount( + source string, + target string, + fsType string, + opts ...string) error { + + mountCmd := "mount" + mountArgs := []string{} + + if source == "" { + return status.Error( + codes.Internal, + "source is not specified for mounting the volume") + } + + if target == "" { + return status.Error( + codes.Internal, + "target is not specified for mounting the volume") + } + + // This is a raw block device mount. Create the mount point as a file + // since bind mount device node requires it to be a file + if fsType == "" { + err := os.MkdirAll(filepath.Dir(target), 0750) + if err != nil { + return fmt.Errorf("failed to create target directory: %v", err) + } + + file, err := os.OpenFile(target, os.O_CREATE, 0660) + if err != nil { + return fmt.Errorf("failed to create target file: %v", err) + } + file.Close() + } else { + mountArgs = append(mountArgs, "-t", fsType) + + err := os.MkdirAll(target, 0755) + if err != nil { + return err + } + } + + if len(opts) > 0 { + mountArgs = append(mountArgs, "-o", strings.Join(opts, ",")) + } + + mountArgs = append(mountArgs, source) + mountArgs = append(mountArgs, target) + + out, err := exec.Command(mountCmd, mountArgs...).CombinedOutput() + if err != nil { + return fmt.Errorf("mounting failed: %v cmd: '%s %s' output: %q", + err, mountCmd, strings.Join(mountArgs, " "), string(out)) + } + + return nil +} + +func (s *nodeService) unmount(target string) error { + unmountCmd := "umount" + unmountArgs := []string{} + + unmountArgs = append(unmountArgs, target) + + out, err := exec.Command(unmountCmd, unmountArgs...).CombinedOutput() + if err != nil { + return fmt.Errorf("unmounting failed: %v cmd: '%s %s' output: %q", + err, unmountCmd, strings.Join(unmountArgs, " "), string(out)) + } + + return nil +} diff --git a/cloud/blockstore/tools/csi_driver/internal/driver/ya.make b/cloud/blockstore/tools/csi_driver/internal/driver/ya.make new file mode 100644 index 00000000000..63e2106d4db --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/internal/driver/ya.make @@ -0,0 +1,12 @@ +OWNER(g:cloud-nbs) + +GO_LIBRARY() + +SRCS( + controller.go + driver.go + identity.go + node.go +) + +END() diff --git a/cloud/blockstore/tools/csi_driver/internal/ya.make b/cloud/blockstore/tools/csi_driver/internal/ya.make new file mode 100644 index 00000000000..ff66d7da8d5 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/internal/ya.make @@ -0,0 +1,5 @@ +OWNER(g:cloud-nbs) + +RECURSE( + driver +) diff --git a/cloud/blockstore/tools/csi_driver/rebuild.sh b/cloud/blockstore/tools/csi_driver/rebuild.sh new file mode 100755 index 00000000000..92da9d394c3 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/rebuild.sh @@ -0,0 +1,16 @@ +#!/bin/bash -e + +echo "=== build driver ===" +rm -f cmd/nbs-csi-driver/nbs-csi-driver +~/workspace/nbs/ya make -r ./cmd/nbs-csi-driver/ + +echo "=== prepare driver ===" +DRIVER_PATH=$(readlink -f "./cmd/nbs-csi-driver/nbs-csi-driver") +rm cmd/nbs-csi-driver/nbs-csi-driver +cp $DRIVER_PATH cmd/nbs-csi-driver/ + +echo "=== upload driver ===" +docker build -t cr.ai.nebius.cloud/crn0l5t3qnnlbpi8de6q/nbs-csi-driver:v0.1 . +docker push cr.ai.nebius.cloud/crn0l5t3qnnlbpi8de6q/nbs-csi-driver:v0.1 + +echo "=== done ===" diff --git a/cloud/blockstore/tools/csi_driver/ya.make b/cloud/blockstore/tools/csi_driver/ya.make new file mode 100644 index 00000000000..95487b89fc2 --- /dev/null +++ b/cloud/blockstore/tools/csi_driver/ya.make @@ -0,0 +1,6 @@ +OWNER(g:cloud-nbs) + +RECURSE( + cmd + internal +) diff --git a/cloud/blockstore/tools/debug/csum/main.cpp b/cloud/blockstore/tools/debug/csum/main.cpp new file mode 100644 index 00000000000..d60f316a97f --- /dev/null +++ b/cloud/blockstore/tools/debug/csum/main.cpp @@ -0,0 +1,67 @@ +#include + +#include + +#include +#include + +namespace { + +using namespace NCloud; +using namespace NBlockStore; +using namespace NLastGetopt; + +//////////////////////////////////////////////////////////////////////////////// + +struct TOptions +{ + TString DataPath; + ui32 BlockSize = 0; + + void Parse(int argc, char** argv) + { + TOpts opts; + opts.AddHelpOption(); + + opts.AddLongOption("data-path", "path to the file with data") + .RequiredArgument() + .Required() + .StoreResult(&DataPath); + + opts.AddLongOption("block-size", "split data into blocks of this size") + .RequiredArgument() + .DefaultValue(4_KB) + .StoreResult(&BlockSize); + + TOptsParseResultException res(&opts, argc, argv); + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +int main(int argc, char** argv) +{ + TOptions opts; + opts.Parse(argc, argv); + + auto data = TIFStream(opts.DataPath).ReadAll(); + if (data.Size() % opts.BlockSize != 0) { + Cerr << "data size is not divisible by block size" << Endl; + return 1; + } + + ui32 i = 0; + while (i < data.Size()) { + auto csum = ComputeDefaultDigest(TBlockDataRef( + data.begin() + i, + opts.BlockSize)); + + Cout << i << "\t" << csum << Endl; + + i += opts.BlockSize; + } + + return 0; +} diff --git a/cloud/blockstore/tools/debug/csum/ya.make b/cloud/blockstore/tools/debug/csum/ya.make new file mode 100644 index 00000000000..4b60e864306 --- /dev/null +++ b/cloud/blockstore/tools/debug/csum/ya.make @@ -0,0 +1,13 @@ +PROGRAM(blockstore-csum-calculator) + +SRCS( + main.cpp +) + +PEERDIR( + cloud/blockstore/libs/diagnostics + + library/cpp/getopt +) + +END() diff --git a/cloud/blockstore/tools/debug/ya.make b/cloud/blockstore/tools/debug/ya.make index 3d0efc7256d..ea24008b698 100644 --- a/cloud/blockstore/tools/debug/ya.make +++ b/cloud/blockstore/tools/debug/ya.make @@ -1,4 +1,5 @@ RECURSE( + csum featureconfig formatter readbdev diff --git a/cloud/blockstore/tools/nbd/bootstrap.cpp b/cloud/blockstore/tools/nbd/bootstrap.cpp index b2b94c696c0..2c6e1b96e29 100644 --- a/cloud/blockstore/tools/nbd/bootstrap.cpp +++ b/cloud/blockstore/tools/nbd/bootstrap.cpp @@ -436,6 +436,10 @@ void TBootstrap::Start() void TBootstrap::Stop() { + if (NbdDevice) { + NbdDevice->Stop(); + } + if (Options->DeviceMode == EDeviceMode::Endpoint) { auto ctx = MakeIntrusive(); auto request = std::make_shared(); @@ -447,10 +451,6 @@ void TBootstrap::Stop() CheckError(future.GetValue(WaitTimeout)); } - if (NbdDevice) { - NbdDevice->Stop(); - } - if (NbdServer) { NbdServer->Stop(); } diff --git a/cloud/blockstore/tools/testing/bad-guest/bin/app.cpp b/cloud/blockstore/tools/testing/bad-guest/bin/app.cpp new file mode 100644 index 00000000000..27637525423 --- /dev/null +++ b/cloud/blockstore/tools/testing/bad-guest/bin/app.cpp @@ -0,0 +1,99 @@ +#include "app.h" + +#include "options.h" + +#include + +#include + +#include + +namespace NCloud::NBlockStore { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +class TApp +{ +private: + std::unique_ptr IO; + std::atomic ShouldStop = false; + +public: + static TApp* GetInstance() + { + return Singleton(); + } + + int Run(const TOptions& options) + { + IO = std::make_unique(options.FilePath, options.BlockSize); + + ui64 blockIndex = 0; + ui32 i = 0; + while (!ShouldStop.load() && blockIndex < options.TotalBlockCount) { + const ui64 sz = options.BlockSize * options.ChunkBlockCount; + const char c1 = 'a' + i % ('z' - 'a' + 1); + const char c2 = 'A' + i % ('Z' - 'A' + 1); + + TString data1(sz, c1); + TString data2(sz, c2); + data1[sz - 1] = '\n'; + data2[sz - 1] = '\n'; + + IO->AlternatingWrite( + blockIndex, + {TStringBuf(data1), TStringBuf(data2)}); + + blockIndex += options.ChunkBlockCount; + ++i; + } + + return 0; + } + + void Stop() + { + ShouldStop.store(true); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +void ProcessSignal(int signum) +{ + if (signum == SIGINT || signum == SIGTERM) { + AppStop(); + } +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +void ConfigureSignals() +{ + std::set_new_handler(abort); + + // make sure that errors can be seen by everybody :) + (void) setvbuf(stdout, nullptr, _IONBF, 0); + (void) setvbuf(stderr, nullptr, _IONBF, 0); + + // mask signals + (void) signal(SIGPIPE, SIG_IGN); + (void) signal(SIGINT, ProcessSignal); + (void) signal(SIGTERM, ProcessSignal); +} + +int AppMain(const TOptions& options) +{ + return TApp::GetInstance()->Run(options); +} + +void AppStop() +{ + TApp::GetInstance()->Stop(); +} + +} // namespace NCloud::NBlockStore diff --git a/cloud/blockstore/tools/testing/bad-guest/bin/app.h b/cloud/blockstore/tools/testing/bad-guest/bin/app.h new file mode 100644 index 00000000000..29408b468bd --- /dev/null +++ b/cloud/blockstore/tools/testing/bad-guest/bin/app.h @@ -0,0 +1,14 @@ +#pragma once + +#include "options.h" + +namespace NCloud::NBlockStore { + +//////////////////////////////////////////////////////////////////////////////// + +void ConfigureSignals(); + +int AppMain(const TOptions& options); +void AppStop(); + +} // namespace NCloud::NBlockStore diff --git a/cloud/blockstore/tools/testing/bad-guest/bin/main.cpp b/cloud/blockstore/tools/testing/bad-guest/bin/main.cpp new file mode 100644 index 00000000000..c3e9f4dbe72 --- /dev/null +++ b/cloud/blockstore/tools/testing/bad-guest/bin/main.cpp @@ -0,0 +1,24 @@ +#include "app.h" +#include "options.h" + +#include + +//////////////////////////////////////////////////////////////////////////////// + +int main(int argc, char** argv) +{ + using namespace NCloud::NBlockStore; + + ConfigureSignals(); + + TOptions options; + try { + options.Parse(argc, argv); + } catch (...) { + Cerr << CurrentExceptionMessage() << Endl; + return 1; + } + + return AppMain(options); +} + diff --git a/cloud/blockstore/tools/testing/bad-guest/bin/options.cpp b/cloud/blockstore/tools/testing/bad-guest/bin/options.cpp new file mode 100644 index 00000000000..79c984682e7 --- /dev/null +++ b/cloud/blockstore/tools/testing/bad-guest/bin/options.cpp @@ -0,0 +1,41 @@ +#include "options.h" + +#include + +#include + +namespace NCloud::NBlockStore { + +using namespace NLastGetopt; + +//////////////////////////////////////////////////////////////////////////////// + +void TOptions::Parse(int argc, char** argv) +{ + TOpts opts; + opts.AddHelpOption(); + + opts.AddLongOption("file", "path to file or block device") + .RequiredArgument("STR") + .Required() + .StoreResult(&FilePath); + + opts.AddLongOption("total-block-count", "total block count") + .RequiredArgument("NUM") + .Required() + .StoreResult(&TotalBlockCount); + + opts.AddLongOption("chunk-block-count", "chunk block count") + .RequiredArgument("NUM") + .Required() + .StoreResult(&ChunkBlockCount); + + opts.AddLongOption("block-size", "block size") + .RequiredArgument("NUM") + .DefaultValue(4_KB) + .StoreResult(&BlockSize); + + TOptsParseResultException(&opts, argc, argv); +} + +} // namespace NCloud::NBlockStore diff --git a/cloud/blockstore/tools/testing/bad-guest/bin/options.h b/cloud/blockstore/tools/testing/bad-guest/bin/options.h new file mode 100644 index 00000000000..7da8c11c83f --- /dev/null +++ b/cloud/blockstore/tools/testing/bad-guest/bin/options.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +namespace NCloud::NBlockStore { + +//////////////////////////////////////////////////////////////////////////////// + +struct TOptions +{ + TString FilePath; + ui64 TotalBlockCount; + ui64 ChunkBlockCount; + ui32 BlockSize; + + void Parse(int argc, char** argv); +}; + +} // namespace NCloud::NBlockStore diff --git a/cloud/blockstore/tools/testing/bad-guest/bin/ya.make b/cloud/blockstore/tools/testing/bad-guest/bin/ya.make new file mode 100644 index 00000000000..9ad014ec335 --- /dev/null +++ b/cloud/blockstore/tools/testing/bad-guest/bin/ya.make @@ -0,0 +1,16 @@ +PROGRAM(blockstore-bad-guest) + +SRCS( + app.cpp + main.cpp + options.cpp +) + +PEERDIR( + cloud/blockstore/tools/testing/bad-guest/lib + + library/cpp/getopt + library/cpp/threading/future +) + +END() diff --git a/cloud/blockstore/tools/testing/bad-guest/lib/io.cpp b/cloud/blockstore/tools/testing/bad-guest/lib/io.cpp new file mode 100644 index 00000000000..718f120f477 --- /dev/null +++ b/cloud/blockstore/tools/testing/bad-guest/lib/io.cpp @@ -0,0 +1,103 @@ +#include "io.h" + +#include + +#include + +namespace NCloud::NBlockStore { + +//////////////////////////////////////////////////////////////////////////////// + +struct TIO::TImpl +{ + mutable TFileHandle File; + mutable NAsyncIO::TAsyncIOService AsyncIO; + const ui32 BlockSize; + +public: + TImpl(const TString& filePath, ui32 blockSize) + : File(filePath, EOpenModeFlag::DirectAligned | EOpenModeFlag::RdWr) + , BlockSize(blockSize) + { + Y_ABORT_UNLESS(File.IsOpen()); + AsyncIO.Start(); + } + + ~TImpl() + { + AsyncIO.Stop(); + File.Close(); + } + + struct TAlignedBuffer + { + TString Buffer; + char* AlignedPtr; + + TAlignedBuffer(ui64 sz, ui32 align) + : Buffer(sz + align, 0) + { + const ui64 baseAddr = reinterpret_cast(Buffer.data()); + const ui64 alignedAddr = AlignUp(baseAddr, align); + const auto alignmentOffset = alignedAddr - baseAddr; + AlignedPtr = Buffer.begin() + alignmentOffset; + } + }; + + TString Read(ui64 blockIndex, ui32 blockCount) const + { + const auto sz = blockCount * BlockSize; + TAlignedBuffer buffer(sz, BlockSize); + const auto offset = blockIndex * BlockSize; + + auto f = AsyncIO.Read(File, buffer.AlignedPtr, sz, offset); + auto res = f.GetValueSync(); + Y_ABORT_UNLESS(res == sz); + return {buffer.AlignedPtr, sz}; + } + + void AlternatingWrite(ui64 blockIndex, const TVector& datas) + { + ui64 sz = 0; + for (const auto& data: datas) { + sz = Max(sz, data.Size()); + } + sz = AlignUp(sz); + TAlignedBuffer buffer(sz, BlockSize); + const auto offset = blockIndex * BlockSize; + memcpy(buffer.AlignedPtr, datas[0].Data(), datas[0].Size()); + + auto f = AsyncIO.Write(File, buffer.AlignedPtr, sz, offset); + // corrupting the data buffer that we have just sent to the storage + // layer via AsyncIO.Write - we want to test how nbs would handle such + // a situation - nbs should not break in any way + ui32 i = 0; + while (!f.HasValue()) { + const auto& data = datas[++i % datas.size()]; + memcpy(buffer.AlignedPtr, data.Data(), data.Size()); + } + + Y_ABORT_UNLESS(f.GetValue() == sz); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +TIO::TIO(const TString& filePath, ui32 blockSize) + : Impl(new TImpl(filePath, blockSize)) +{ +} + +TIO::~TIO() = default; + +TString TIO::Read(ui64 blockIndex, ui32 blockCount) const +{ + return Impl->Read(blockIndex, blockCount); +} + +void TIO::AlternatingWrite(ui64 blockIndex, const TVector& datas) +{ + return Impl->AlternatingWrite(blockIndex, datas); +} + +} // namespace NCloud::NBlockStore diff --git a/cloud/blockstore/tools/testing/bad-guest/lib/io.h b/cloud/blockstore/tools/testing/bad-guest/lib/io.h new file mode 100644 index 00000000000..26cc3106264 --- /dev/null +++ b/cloud/blockstore/tools/testing/bad-guest/lib/io.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include +#include + +namespace NCloud::NBlockStore { + +//////////////////////////////////////////////////////////////////////////////// + +struct TIO +{ + explicit TIO(const TString& filePath, ui32 blockSize); + ~TIO(); + + [[nodiscard]] TString Read(ui64 blockIndex, ui32 blockCount) const; + void AlternatingWrite(ui64 blockIndex, const TVector& datas); + +private: + struct TImpl; + std::unique_ptr Impl; +}; + +} // namespace NCloud::NBlockStore diff --git a/cloud/blockstore/tools/testing/bad-guest/lib/ya.make b/cloud/blockstore/tools/testing/bad-guest/lib/ya.make new file mode 100644 index 00000000000..879b8b35605 --- /dev/null +++ b/cloud/blockstore/tools/testing/bad-guest/lib/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +SRCS( + io.cpp +) + +PEERDIR( + library/cpp/aio + library/cpp/threading/future +) + +END() diff --git a/cloud/blockstore/tools/testing/bad-guest/ya.make b/cloud/blockstore/tools/testing/bad-guest/ya.make new file mode 100644 index 00000000000..25fc2d1dbf7 --- /dev/null +++ b/cloud/blockstore/tools/testing/bad-guest/ya.make @@ -0,0 +1,4 @@ +RECURSE( + bin + lib +) diff --git a/cloud/blockstore/tools/testing/eternal_tests/test_runner/lib/main.py b/cloud/blockstore/tools/testing/eternal_tests/test_runner/lib/main.py index 06312c82e8b..740fdae5c1b 100644 --- a/cloud/blockstore/tools/testing/eternal_tests/test_runner/lib/main.py +++ b/cloud/blockstore/tools/testing/eternal_tests/test_runner/lib/main.py @@ -346,10 +346,10 @@ def create_fs(self, instance: Ycp.Instance): def _mount_fs(self, instance: Ycp.Instance): self.logger.info('Mounting fs') config = self.test_config.fs_config - with self.module_factories.make_ssh_client(self.args.dry_run, instance.ip) as ssh: + with self.module_factories.make_ssh_client(self.args.dry_run, instance.ip, ssh_key_path=self.args.ssh_key_path) as ssh: cmd = (f'mkdir -p {config.mount_path} && {{ sudo umount {config.mount_path} || true; }} &&' f' mount -t virtiofs {config.device_name} {config.mount_path}') - if hasattr(self.test_config.load_config, 'test_file'): + if hasattr(self.test_config, 'load_config') and hasattr(self.test_config.load_config, 'test_file'): cmd += f' && touch {self.test_config.load_config.test_file}' _, _, stderr = ssh.exec_command(cmd) @@ -581,14 +581,14 @@ def handle_setup_new_db_test(self): instance = self.create_and_configure_vm() self.logger.info('Prepare database, executing script') - with self.module_factories.make_sftp_client(self.args.dry_run, instance.ip) as sftp: + with self.module_factories.make_sftp_client(self.args.dry_run, instance.ip, ssh_key_path=self.args.ssh_key_path) as sftp: script_name = self._DB_TEST_INIT_SCRIPT % self.test_config.db file = sftp.file(f'{self._DB_TEST_INIT_SCRIPT_PATH}/{script_name}', 'w') file.write(resource.find(script_name).decode('utf8')) file.flush() sftp.chmod(f'{self._DB_TEST_INIT_SCRIPT_PATH}/{script_name}', 0o755) - with self.module_factories.make_ssh_client(self.args.dry_run, instance.ip) as ssh: + with self.module_factories.make_ssh_client(self.args.dry_run, instance.ip, ssh_key_path=self.args.ssh_key_path) as ssh: _, stdout, stderr = ssh.exec_command(f'{self._DB_TEST_INIT_SCRIPT_PATH}/{script_name}') exit_code = stdout.channel.recv_exit_status() if exit_code != 0: diff --git a/cloud/blockstore/tools/testing/eternal_tests/test_runner/lib/test_configs.py b/cloud/blockstore/tools/testing/eternal_tests/test_runner/lib/test_configs.py index 3cf48a48d86..97ba39b3459 100644 --- a/cloud/blockstore/tools/testing/eternal_tests/test_runner/lib/test_configs.py +++ b/cloud/blockstore/tools/testing/eternal_tests/test_runner/lib/test_configs.py @@ -316,7 +316,8 @@ def service_name(self) -> tp.Optional[str]: 'eternal-279gb-ssd-local-different-size-requests': LoadConfig(True, False, 64, 50, 279, 4096), 'eternal-367gb-ssd-local-v3': LoadConfig(False, False, 128, 50, 186, 4096), - 'eternal-734gb-ssd-local-different-size-requests-v3': LoadConfig(True, False, 64, 50, 279, 4096), + 'eternal-734gb-ssd-local-different-size-requests-v3': LoadConfig( + True, False, 64, 10, 279, 4096), 'eternal-320gb-encrypted': LoadConfig(False, True, 32, 50, 320, 4096), 'eternal-1023gb-nonrepl-encrypted': LoadConfig(False, False, 32, 50, 1023, 4096), diff --git a/cloud/blockstore/tools/testing/ya.make b/cloud/blockstore/tools/testing/ya.make index 7ca46d879d1..835b9f6471d 100644 --- a/cloud/blockstore/tools/testing/ya.make +++ b/cloud/blockstore/tools/testing/ya.make @@ -1,4 +1,5 @@ RECURSE( + bad-guest build-fresh-blob chaos-monkey eternal_tests diff --git a/cloud/blockstore/tools/ya.make b/cloud/blockstore/tools/ya.make index 6244fc75a18..f4f9ea704b2 100644 --- a/cloud/blockstore/tools/ya.make +++ b/cloud/blockstore/tools/ya.make @@ -2,6 +2,7 @@ RECURSE( analytics ci cms + csi_driver debug fs http_proxy diff --git a/cloud/blockstore/vhost-server/server_ut.cpp b/cloud/blockstore/vhost-server/server_ut.cpp index 9d472a8a49a..d9e2a0b7521 100644 --- a/cloud/blockstore/vhost-server/server_ut.cpp +++ b/cloud/blockstore/vhost-server/server_ut.cpp @@ -168,6 +168,26 @@ struct TFixture Files.clear(); Options.Layout.clear(); } + + TSimpleStats GetStats(ui64 expectedCompleted) const + { + // Without I/O, stats are synced every second and only if there is a + // pending GetStats call. The first call to GetStats might not bring the + // latest stats; therefore, you need at least two calls so that the AIO + // backend will sync the stats. + + TSimpleStats prevStats; + TSimpleStats stats; + for (int i = 0; i != 5; ++i) { + stats = Server->GetStats(prevStats); + if (stats.Completed == expectedCompleted) { + break; + } + Sleep(TDuration::Seconds(1)); + } + + return stats; + } }; //////////////////////////////////////////////////////////////////////////////// @@ -252,8 +272,7 @@ Y_UNIT_TEST_SUITE(TServerTest) UNIT_ASSERT_VALUES_EQUAL(expectedData, buffer); } - TSimpleStats prevStats; - auto stats = Server->GetStats(prevStats); + const auto stats = GetStats(sectorCount); UNIT_ASSERT_VALUES_EQUAL(0, stats.CompFailed); UNIT_ASSERT_VALUES_EQUAL(0, stats.SubFailed); @@ -422,8 +441,7 @@ Y_UNIT_TEST_SUITE(TServerTest) WaitAll(futures).Wait(); - TSimpleStats prevStats; - auto stats = Server->GetStats(prevStats); + const auto stats = GetStats(requestCount); UNIT_ASSERT_VALUES_EQUAL(requestCount, stats.Submitted); UNIT_ASSERT_VALUES_EQUAL(requestCount, stats.Completed); diff --git a/cloud/contrib/vhost b/cloud/contrib/vhost index 9ff7fb2ecdc..5e464ce321d 160000 --- a/cloud/contrib/vhost +++ b/cloud/contrib/vhost @@ -1 +1 @@ -Subproject commit 9ff7fb2ecdc3dab230550eadcf521912c632af86 +Subproject commit 5e464ce321d545f065bca9b87cf68c3cd16c9653 diff --git a/cloud/disk_manager/internal/pkg/clients/nbs/client.go b/cloud/disk_manager/internal/pkg/clients/nbs/client.go index 0eda84466b4..8ba399d24c3 100644 --- a/cloud/disk_manager/internal/pkg/clients/nbs/client.go +++ b/cloud/disk_manager/internal/pkg/clients/nbs/client.go @@ -77,6 +77,40 @@ func getStorageMediaKind( } } +func getDiskKind( + mediaKind core_protos.EStorageMediaKind, +) (types.DiskKind, error) { + + switch mediaKind { + case core_protos.EStorageMediaKind_STORAGE_MEDIA_DEFAULT: + return 0, errors.NewNonRetriableErrorf( + "unsupported media kind %v", + mediaKind, + ) + case core_protos.EStorageMediaKind_STORAGE_MEDIA_SSD: + return types.DiskKind_DISK_KIND_SSD, nil + case core_protos.EStorageMediaKind_STORAGE_MEDIA_HYBRID: + return types.DiskKind_DISK_KIND_HDD, nil + case core_protos.EStorageMediaKind_STORAGE_MEDIA_HDD: + return types.DiskKind_DISK_KIND_HDD, nil + case core_protos.EStorageMediaKind_STORAGE_MEDIA_SSD_NONREPLICATED: + return types.DiskKind_DISK_KIND_SSD_NONREPLICATED, nil + case core_protos.EStorageMediaKind_STORAGE_MEDIA_SSD_MIRROR2: + return types.DiskKind_DISK_KIND_SSD_MIRROR2, nil + case core_protos.EStorageMediaKind_STORAGE_MEDIA_SSD_LOCAL: + return types.DiskKind_DISK_KIND_SSD_LOCAL, nil + case core_protos.EStorageMediaKind_STORAGE_MEDIA_SSD_MIRROR3: + return types.DiskKind_DISK_KIND_SSD_MIRROR3, nil + case core_protos.EStorageMediaKind_STORAGE_MEDIA_HDD_NONREPLICATED: + return types.DiskKind_DISK_KIND_HDD_NONREPLICATED, nil + default: + return 0, errors.NewNonRetriableErrorf( + "unknown media kind %v", + mediaKind, + ) + } +} + func isDiskRegistryBasedDisk(mediaKind core_protos.EStorageMediaKind) bool { switch mediaKind { case core_protos.EStorageMediaKind_STORAGE_MEDIA_SSD_NONREPLICATED, @@ -538,6 +572,32 @@ func (t CheckpointType) toProto() protos.ECheckpointType { }[t] } +func parseCheckpointStatus(protoType protos.ECheckpointStatus) CheckpointStatus { + switch protoType { + case protos.ECheckpointStatus_NOT_READY: + return CheckpointStatusNotReady + case protos.ECheckpointStatus_READY: + return CheckpointStatusReady + case protos.ECheckpointStatus_ERROR: + return CheckpointStatusError + default: + return CheckpointStatusError + } +} + +func (m CheckpointStatus) String() string { + switch m { + case CheckpointStatusNotReady: + return "CheckpointStatusNotReady" + case CheckpointStatusReady: + return "CheckpointStatusReady" + case CheckpointStatusError: + return "CheckpointStatusError" + default: + return "CheckpointStatusUnknown" + } +} + //////////////////////////////////////////////////////////////////////////////// func (c *client) Ping(ctx context.Context) (err error) { @@ -716,6 +776,16 @@ func (c *client) CreateCheckpoint( return wrapError(err) } +func (c *client) GetCheckpointStatus( + ctx context.Context, + diskID string, + checkpointID string, +) (CheckpointStatus, error) { + + status, err := c.nbs.GetCheckpointStatus(ctx, diskID, checkpointID) + return parseCheckpointStatus(status), wrapError(err) +} + func (c *client) DeleteCheckpoint( ctx context.Context, diskID string, @@ -966,9 +1036,15 @@ func (c *client) Describe( return DiskParams{}, err } + diskKind, err := getDiskKind(volume.StorageMediaKind) + if err != nil { + return DiskParams{}, err + } + return DiskParams{ BlockSize: volume.BlockSize, BlocksCount: volume.BlocksCount, + Kind: diskKind, EncryptionDesc: encryptionDesc, CloudID: volume.CloudId, FolderID: volume.FolderId, diff --git a/cloud/disk_manager/internal/pkg/clients/nbs/interface.go b/cloud/disk_manager/internal/pkg/clients/nbs/interface.go index ed6be52ee9a..3c3daedae3a 100644 --- a/cloud/disk_manager/internal/pkg/clients/nbs/interface.go +++ b/cloud/disk_manager/internal/pkg/clients/nbs/interface.go @@ -61,6 +61,7 @@ type DiskModel struct { type DiskParams struct { BlockSize uint32 BlocksCount uint64 + Kind types.DiskKind EncryptionDesc *types.EncryptionDesc CloudID string FolderID string @@ -98,6 +99,14 @@ const ( CheckpointTypeWithoutData ) +type CheckpointStatus uint32 + +const ( + CheckpointStatusNotReady CheckpointStatus = iota + CheckpointStatusReady + CheckpointStatusError +) + type CheckpointParams struct { DiskID string CheckpointID string @@ -126,6 +135,12 @@ type Client interface { CreateCheckpoint(ctx context.Context, params CheckpointParams) error + GetCheckpointStatus( + ctx context.Context, + diskID string, + checkpointID string, + ) (CheckpointStatus, error) + DeleteCheckpoint( ctx context.Context, diskID string, diff --git a/cloud/disk_manager/internal/pkg/clients/nbs/mocks/client_mock.go b/cloud/disk_manager/internal/pkg/clients/nbs/mocks/client_mock.go index 6ef9437fc47..7548822ba29 100644 --- a/cloud/disk_manager/internal/pkg/clients/nbs/mocks/client_mock.go +++ b/cloud/disk_manager/internal/pkg/clients/nbs/mocks/client_mock.go @@ -76,6 +76,16 @@ func (c *ClientMock) CreateCheckpoint( return args.Error(0) } +func (c *ClientMock) GetCheckpointStatus( + ctx context.Context, + diskID string, + checkpointID string, +) (nbs.CheckpointStatus, error) { + + args := c.Called(ctx, diskID, checkpointID) + return args.Get(0).(nbs.CheckpointStatus), args.Error(1) +} + func (c *ClientMock) DeleteCheckpoint( ctx context.Context, diskID string, diff --git a/cloud/disk_manager/internal/pkg/clients/nbs/session.go b/cloud/disk_manager/internal/pkg/clients/nbs/session.go index a140c67b456..bfb1c988b86 100644 --- a/cloud/disk_manager/internal/pkg/clients/nbs/session.go +++ b/cloud/disk_manager/internal/pkg/clients/nbs/session.go @@ -170,12 +170,14 @@ func (s *Session) init(ctx context.Context) error { } }() - err := s.discoverAndMount(ctx) + volume, err := s.discoverAndMount(ctx) if err != nil { s.closeImpl(ctx) + return err } - return err + s.volume = volume + return nil } //////////////////////////////////////////////////////////////////////////////// @@ -245,6 +247,15 @@ func (s *Session) Write( return errors.NewRetriableErrorf("last rediscover was failed") } + blocksCount := uint32(len(data)) / s.BlockSize() + logging.Debug( + ctx, + "WriteBlocks on disk %v, offset %v, blocks count %v", + s.diskID, + startIndex, + blocksCount, + ) + defer s.metrics.StatRequest("Write")(&err) err = s.session.WriteBlocks( @@ -297,10 +308,10 @@ func (s *Session) closeImpl(ctx context.Context) { } // Not thread-safe. -func (s *Session) discoverAndMount(ctx context.Context) error { +func (s *Session) discoverAndMount(ctx context.Context) (*protos.TVolume, error) { client, host, err := s.nbs.DiscoverInstance(ctx) if err != nil { - return wrapError(err) + return nil, wrapError(err) } s.client = client @@ -315,11 +326,11 @@ func (s *Session) discoverAndMount(ctx context.Context) error { s.diskID, &s.mountOpts, ) - if err == nil { - s.volume = s.session.Volume() + if err != nil { + return nil, wrapError(err) } - return wrapError(err) + return s.session.Volume(), nil } func (s *Session) rediscover(ctx context.Context) { @@ -332,7 +343,7 @@ func (s *Session) rediscover(ctx context.Context) { s.closeSession(ctx) - err := s.discoverAndMount(ctx) + _, err := s.discoverAndMount(ctx) if err != nil { logging.Warn(ctx, "rediscover failed: %v", err) // Stop subsequent rediscovers. diff --git a/cloud/disk_manager/internal/pkg/clients/nbs/tests/client_test.go b/cloud/disk_manager/internal/pkg/clients/nbs/tests/client_test.go index bb6358276ce..b8c4abd55fa 100644 --- a/cloud/disk_manager/internal/pkg/clients/nbs/tests/client_test.go +++ b/cloud/disk_manager/internal/pkg/clients/nbs/tests/client_test.go @@ -15,6 +15,7 @@ import ( "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/auth" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/clients/nbs" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/clients/nbs/config" + "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/facade/testcommon" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/monitoring/metrics" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/types" "github.com/ydb-platform/nbs/cloud/tasks/errors" @@ -1167,6 +1168,67 @@ func TestCloneDiskFromOneZoneToAnother(t *testing.T) { require.True(t, !errors.CanRetry(err)) } +func TestCloneDiskFromOneZoneToAnotherConcurrently(t *testing.T) { + ctx := newContext() + client := newClient(t, ctx) + otherZoneClient := newOtherZoneClient(t, ctx) + multiZoneClient := newMultiZoneClient(t, ctx) + + diskID := t.Name() + + err := client.Create(ctx, nbs.CreateDiskParams{ + ID: diskID, + BlocksCount: 4096, + BlockSize: 4096, + Kind: types.DiskKind_DISK_KIND_SSD, + }) + require.NoError(t, err) + + err = multiZoneClient.Clone( + ctx, + diskID, + "", // dstPlacementGroupID + 0, // dstPlacementPartitionIndex + 1, // fillGeneration + "", // baseDiskID + ) + require.NoError(t, err) + + errs := make(chan error) + + go func() { + // Need to add some variance for better testing. + testcommon.WaitForRandomDuration(1*time.Millisecond, 10*time.Millisecond) + + errs <- multiZoneClient.Clone( + ctx, + diskID, + "", // dstPlacementGroupID + 0, // dstPlacementPartitionIndex + 2, // fillGeneration + "", // baseDiskID + ) + }() + + go func() { + // Need to add some variance for better testing. + testcommon.WaitForRandomDuration(1*time.Millisecond, 10*time.Millisecond) + + errs <- otherZoneClient.DeleteWithFillGeneration( + ctx, + diskID, + 1, // fillGeneration + ) + }() + + for i := 0; i < 2; i++ { + err := <-errs + if err != nil { + require.True(t, errors.Is(err, errors.NewEmptyRetriableError())) + } + } +} + func TestFinishFillDisk(t *testing.T) { ctx := newContext() client := newClient(t, ctx) diff --git a/cloud/disk_manager/internal/pkg/dataplane/collect_snapshot_metrics_task.go b/cloud/disk_manager/internal/pkg/dataplane/collect_snapshot_metrics_task.go index e14b6842238..0e166aa9ac0 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/collect_snapshot_metrics_task.go +++ b/cloud/disk_manager/internal/pkg/dataplane/collect_snapshot_metrics_task.go @@ -75,7 +75,6 @@ func (c collectSnapshotMetricsTask) Cancel( func (c collectSnapshotMetricsTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/dataplane/collect_snapshots_task.go b/cloud/disk_manager/internal/pkg/dataplane/collect_snapshots_task.go index d41eca9edf5..f236f81e2d8 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/collect_snapshots_task.go +++ b/cloud/disk_manager/internal/pkg/dataplane/collect_snapshots_task.go @@ -171,7 +171,6 @@ func (t *collectSnapshotsTask) Cancel( func (t *collectSnapshotsTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_disk_task.go b/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_disk_task.go index ff2ad3bcc6c..48483bcf07e 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_disk_task.go +++ b/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_disk_task.go @@ -67,7 +67,6 @@ func (t *createSnapshotFromDiskTask) Cancel( func (t *createSnapshotFromDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &protos.CreateSnapshotFromDiskMetadata{ diff --git a/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_legacy_snapshot_task.go b/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_legacy_snapshot_task.go index 11bd293e0f7..13f6c2e29d6 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_legacy_snapshot_task.go +++ b/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_legacy_snapshot_task.go @@ -155,7 +155,6 @@ func (t *createSnapshotFromLegacySnapshotTask) Cancel( func (t *createSnapshotFromLegacySnapshotTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &protos.CreateSnapshotFromLegacySnapshotMetadata{ diff --git a/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_snapshot_task.go b/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_snapshot_task.go index 64bac73c73d..4384687c7a0 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_snapshot_task.go +++ b/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_snapshot_task.go @@ -100,7 +100,6 @@ func (t *createSnapshotFromSnapshotTask) Cancel( func (t *createSnapshotFromSnapshotTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &protos.CreateSnapshotFromSnapshotMetadata{ diff --git a/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_url_task.go b/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_url_task.go index e7ee83df5a7..755abf18cef 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_url_task.go +++ b/cloud/disk_manager/internal/pkg/dataplane/create_snapshot_from_url_task.go @@ -176,7 +176,6 @@ func (t *createSnapshotFromURLTask) Cancel( func (t *createSnapshotFromURLTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &protos.CreateSnapshotFromURLMetadata{ diff --git a/cloud/disk_manager/internal/pkg/dataplane/delete_snapshot_data_task.go b/cloud/disk_manager/internal/pkg/dataplane/delete_snapshot_data_task.go index 1d7dd6ba342..fc6819199a7 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/delete_snapshot_data_task.go +++ b/cloud/disk_manager/internal/pkg/dataplane/delete_snapshot_data_task.go @@ -65,7 +65,6 @@ func (t *deleteSnapshotDataTask) Cancel( func (t *deleteSnapshotDataTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/dataplane/delete_snapshot_task.go b/cloud/disk_manager/internal/pkg/dataplane/delete_snapshot_task.go index 58ca53aea60..3dbddc1aec3 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/delete_snapshot_task.go +++ b/cloud/disk_manager/internal/pkg/dataplane/delete_snapshot_task.go @@ -65,7 +65,6 @@ func (t *deleteSnapshotTask) Cancel( func (t *deleteSnapshotTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/dataplane/protos/replicate_disk_task.proto b/cloud/disk_manager/internal/pkg/dataplane/protos/replicate_disk_task.proto index 2e21308a093..086b6af234e 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/protos/replicate_disk_task.proto +++ b/cloud/disk_manager/internal/pkg/dataplane/protos/replicate_disk_task.proto @@ -15,11 +15,12 @@ enum ReplicateDiskTaskEvents { } message ReplicateDiskTaskRequest { + reserved 4; + types.Disk SrcDisk = 1; types.Disk DstDisk = 2; uint64 FillGeneration = 3; - bool UseLightCheckpoint = 4; // Use light or withoutData checkpoint. bool IgnoreBaseDisk = 5; bool UseProxyOverlayDisk = 6; diff --git a/cloud/disk_manager/internal/pkg/dataplane/replicate_disk_task.go b/cloud/disk_manager/internal/pkg/dataplane/replicate_disk_task.go index 618e1213d31..4ba06057f95 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/replicate_disk_task.go +++ b/cloud/disk_manager/internal/pkg/dataplane/replicate_disk_task.go @@ -14,6 +14,7 @@ import ( "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/dataplane/protos" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/performance" performance_config "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/performance/config" + "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/types" "github.com/ydb-platform/nbs/cloud/disk_manager/pkg/client/codes" "github.com/ydb-platform/nbs/cloud/tasks" "github.com/ydb-platform/nbs/cloud/tasks/errors" @@ -51,6 +52,11 @@ func (t *replicateDiskTask) Run( execCtx tasks.ExecutionContext, ) error { + useLightCheckpoint, err := t.useLightCheckpoint(ctx) + if err != nil { + return err + } + for { _, currentCheckpointID, _ := t.getCheckpointIDs(execCtx) err := t.deleteProxyOverlayDisk( @@ -78,7 +84,7 @@ func (t *replicateDiskTask) Run( return nil } - err = t.updateCheckpoints(ctx, execCtx, t.request.UseLightCheckpoint) + err = t.updateCheckpoints(ctx, execCtx, useLightCheckpoint) if err != nil { return err } @@ -107,7 +113,6 @@ func (t *replicateDiskTask) Cancel( func (t *replicateDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { metadata := &protos.ReplicateDiskTaskMetadata{ @@ -255,6 +260,28 @@ func (t *replicateDiskTask) checkReplicationProgress( return nil } +func (t *replicateDiskTask) useLightCheckpoint( + ctx context.Context, +) (bool, error) { + + client, err := t.nbsFactory.GetClient(ctx, t.request.SrcDisk.ZoneId) + if err != nil { + return false, err + } + + diskParams, err := client.Describe(ctx, t.request.SrcDisk.DiskId) + if err != nil { + return false, err + } + + diskKind := diskParams.Kind + + return diskKind == types.DiskKind_DISK_KIND_SSD_NONREPLICATED || + diskKind == types.DiskKind_DISK_KIND_SSD_MIRROR2 || + diskKind == types.DiskKind_DISK_KIND_SSD_MIRROR3 || + diskKind == types.DiskKind_DISK_KIND_HDD_NONREPLICATED, nil +} + func (t *replicateDiskTask) replicate( ctx context.Context, execCtx tasks.ExecutionContext, diff --git a/cloud/disk_manager/internal/pkg/dataplane/transfer_from_disk_to_disk_task.go b/cloud/disk_manager/internal/pkg/dataplane/transfer_from_disk_to_disk_task.go index 0e521b189fc..9e622094a58 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/transfer_from_disk_to_disk_task.go +++ b/cloud/disk_manager/internal/pkg/dataplane/transfer_from_disk_to_disk_task.go @@ -133,7 +133,6 @@ func (t *transferFromDiskToDiskTask) Cancel( func (t *transferFromDiskToDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &protos.TransferFromDiskToDiskMetadata{ diff --git a/cloud/disk_manager/internal/pkg/dataplane/transfer_from_snapshot_to_disk_task.go b/cloud/disk_manager/internal/pkg/dataplane/transfer_from_snapshot_to_disk_task.go index 484e9814d8a..356cc90627b 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/transfer_from_snapshot_to_disk_task.go +++ b/cloud/disk_manager/internal/pkg/dataplane/transfer_from_snapshot_to_disk_task.go @@ -113,7 +113,6 @@ func (t *transferFromSnapshotToDiskTask) Cancel( func (t *transferFromSnapshotToDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &protos.TransferFromSnapshotToDiskMetadata{ diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/formats.go b/cloud/disk_manager/internal/pkg/dataplane/url/formats.go index b5f98456900..0a5e28151bb 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/url/formats.go +++ b/cloud/disk_manager/internal/pkg/dataplane/url/formats.go @@ -56,7 +56,7 @@ const ( func (f ImageFormat) IsSupported() bool { switch f { - case ImageFormatRaw, ImageFormatQCOW2, ImageFormatVMDK: + case ImageFormatRaw, ImageFormatQCOW2, ImageFormatVMDK, ImageFormatVHD: return true } diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/source.go b/cloud/disk_manager/internal/pkg/dataplane/url/source.go index 0b45902aa30..ee3af708312 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/url/source.go +++ b/cloud/disk_manager/internal/pkg/dataplane/url/source.go @@ -9,6 +9,7 @@ import ( dataplane_common "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/dataplane/common" url_common "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/dataplane/url/common" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/dataplane/url/qcow2" + "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/dataplane/url/vhd" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/dataplane/url/vmdk" task_errors "github.com/ydb-platform/nbs/cloud/tasks/errors" ) @@ -253,6 +254,16 @@ func newImageMapReader( urlReader.EnableCache() return reader, nil + case ImageFormatVHD: + reader := vhd.NewImageMapReader(urlReader) + err := reader.ReadFooter(ctx) + if err != nil { + return nil, err + } + + urlReader.EnableCache() + return reader, nil + case ImageFormatRaw: return newRawImageMapReader(urlReader.Size()), nil diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/tests/url_test.go b/cloud/disk_manager/internal/pkg/dataplane/url/tests/url_test.go index 11a906c8960..821aa463fcd 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/url/tests/url_test.go +++ b/cloud/disk_manager/internal/pkg/dataplane/url/tests/url_test.go @@ -43,6 +43,14 @@ func parseUint64(t *testing.T, str string) uint64 { return value } +func getExpectedChunkCount(imageSize uint64) uint32 { + if imageSize%chunkSize == 0 { + return uint32(imageSize / chunkSize) + } + + return uint32(imageSize/chunkSize) + 1 +} + //////////////////////////////////////////////////////////////////////////////// const ( @@ -142,6 +150,12 @@ func TestImageReading(t *testing.T) { imageSize: parseUint64(t, os.Getenv("DISK_MANAGER_RECIPE_VMDK_IMAGE_SIZE")), imageCRC32: parseUint32(t, os.Getenv("DISK_MANAGER_RECIPE_VMDK_IMAGE_CRC32")), }, + { + name: "vhd image", + imageURL: getImageFileURL(os.Getenv("DISK_MANAGER_RECIPE_VHD_IMAGE_FILE_SERVER_PORT")), + imageSize: parseUint64(t, os.Getenv("DISK_MANAGER_RECIPE_VHD_IMAGE_SIZE")), + imageCRC32: parseUint32(t, os.Getenv("DISK_MANAGER_RECIPE_VHD_IMAGE_CRC32")), + }, { name: "vmdk stream optimized image", imageURL: getImageFileURL(os.Getenv("DISK_MANAGER_RECIPE_VMDK_STREAM_OPTIMIZED_IMAGE_FILE_SERVER_PORT")), @@ -155,12 +169,6 @@ func TestImageReading(t *testing.T) { imageSize: parseUint64(t, os.Getenv("DISK_MANAGER_RECIPE_VMDK_UBUNTU2204_IMAGE_SIZE")), imageCRC32: parseUint32(t, os.Getenv("DISK_MANAGER_RECIPE_VMDK_UBUNTU2204_IMAGE_CRC32")), }, - { - name: "qcow2 image which reproduces panic issue (NBS-4635)", - imageURL: getImageFileURL(os.Getenv("DISK_MANAGER_RECIPE_QCOW2_PANIC_IMAGE_FILE_SERVER_PORT")), - imageSize: parseUint64(t, os.Getenv("DISK_MANAGER_RECIPE_QCOW2_PANIC_IMAGE_SIZE")), - imageCRC32: parseUint32(t, os.Getenv("DISK_MANAGER_RECIPE_QCOW2_PANIC_IMAGE_CRC32")), - }, { name: "vmdk stream optimized windows image with multiple grains", imageURL: getImageFileURL(os.Getenv("DISK_MANAGER_RECIPE_VMDK_WINDOWS_FILE_SERVER_PORT")), @@ -188,7 +196,7 @@ func TestImageReading(t *testing.T) { chunkCount, err := source.ChunkCount(ctx) require.NoError(t, err) - expectedChunkCount := uint32(testCase.imageSize / chunkSize) + expectedChunkCount := getExpectedChunkCount(testCase.imageSize) require.Equal(t, expectedChunkCount, chunkCount) checkChunks(t, ctx, testCase.imageCRC32, chunkCount, source) diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/vhd/footer.go b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/footer.go index 2afeeb0d92b..56b0ececc4f 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/url/vhd/footer.go +++ b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/footer.go @@ -5,7 +5,9 @@ import "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/dataplane/ur //////////////////////////////////////////////////////////////////////////////// const ( - footerCookie = "conectix" + footerCookie = "conectix" + fileFormatVersion = uint32(0x00010000) + dynamicHardDiskType = uint32(3) ) //////////////////////////////////////////////////////////////////////////////// @@ -13,31 +15,47 @@ const ( // https://learn.microsoft.com/en-us/windows/win32/vstor/about-vhd type footer struct { Cookie [8]byte - Features uint32 - FileFormatVersion uint32 - DataOffset uint64 - Timestamp uint32 - CreatorApplication uint32 - CreatorVersion uint32 - CreatorHostOS uint32 - OriginalSize uint64 // Stores the size of the hard disk in bytes at creation time. - CurrentSize uint64 // Stores the current size of the hard disk in bytes. - DiskGeometry uint32 - DiskType uint32 - Checksum uint32 - UniqueID [16]byte - SavedState byte + Features uint32 // A bit field used to indicate specific feature support. + FileFormatVersion uint32 // Is divided into a major/minor version and matches the version of the specification used in creating the file. + DataOffset uint64 // Absolute byte offset, from the beginning of the file, to the next structure. + Timestamp uint32 // Creation time of a hard disk image. + CreatorApplication [4]byte // Used to document which application created the hard disk. + CreatorVersion uint32 // Major/minor version of the application that created the hard disk image. + CreatorHostOS uint32 // Type of host operating system this disk image is created on. + OriginalSize uint64 // Size of the hard disk in bytes at creation time. + CurrentSize uint64 // Current size of the hard disk in bytes. + DiskGeometry uint32 // Cylinder, heads, and sectors per track value for the hard disk. + DiskType uint32 // Type of the disk. + Checksum uint32 // A basic checksum of the hard disk footer. + UniqueID [16]byte // А unique ID stored in the hard disk which is used to identify the hard disk. + SavedState byte // A one-byte flag that describes whether the system is in saved state. Reserved [427]byte } func (f footer) validate() error { if string(f.Cookie[:]) != footerCookie { return common.NewSourceInvalidError( - "Failed to check vhd footer cookie: expected - %s, actual - %s", + "failed to check vhd footer cookie: expected %s, actual %s", footerCookie, f.Cookie, ) } - return nil // TODO: Implement. + if f.FileFormatVersion != fileFormatVersion { + return common.NewSourceInvalidError( + "failed to check vhd file format version: expected %v, actual %v", + fileFormatVersion, + f.FileFormatVersion, + ) + } + + if f.DiskType != dynamicHardDiskType { + return common.NewSourceInvalidError( + "failed to check vhd disk type: expected %v, actual %v", + dynamicHardDiskType, + f.DiskType, + ) + } + + return nil } diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/vhd/header.go b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/header.go index 1aaffbd370d..57aa6f3b1a2 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/url/vhd/header.go +++ b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/header.go @@ -6,36 +6,53 @@ import "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/dataplane/ur const ( // The dynamic disk header should appear on a sector (512-byte) boundary. - headerOffset = uint64(512) - - headerCookie = "cxsparse" + headerOffset = uint64(512) + headerCookie = "cxsparse" + batOffset = uint64(1536) // Block allocation table offset. + headerVersion = uint32(0x00010000) ) // https://learn.microsoft.com/en-us/windows/win32/vstor/about-vhd type header struct { Cookie [8]byte - DataOffset uint64 - TableOffset uint64 - HeaderVersion uint32 - MaxTableEntries uint32 - BlockSize uint32 - Checksum uint32 - ParentUniqueID [16]byte - ParentTimeStamp uint32 + DataOffset uint64 // Absolute byte offset to the next structure in the hard disk image. + TableOffset uint64 // Absolute byte offset of the Block Allocation Table (BAT) in the file. + HeaderVersion uint32 // Version of the dynamic disk header. + MaxTableEntries uint32 // Maximum entries present in the BAT. + BlockSize uint32 // Unit of expansion for dynamic and differencing hard disks. + Checksum uint32 // Basic checksum of the dynamic header. + ParentUniqueID [16]byte // Used for differencing hard disks. Not supported. + ParentTimeStamp uint32 // Used for differencing hard disks. Not supported. Reserved1 uint32 - ParentUnicodeName [512]byte - ParentLocatorEntries [192]byte + ParentUnicodeName [512]byte // Used for differencing hard disks. Not supported. + ParentLocatorEntries [192]byte // Used for differencing hard disks. Not supported. Reserved2 [256]byte } func (h header) validate() error { if string(h.Cookie[:]) != headerCookie { return common.NewSourceInvalidError( - "Failed to check vhd header cookie: expected - %s, actual - %s", + "failed to check vhd header cookie: expected %s, actual %s", headerCookie, h.Cookie, ) } - return nil // TODO: Implement. + if h.TableOffset != batOffset { + return common.NewSourceInvalidError( + "failed to check vhd header bat offset: expected %v, actual %v", + batOffset, + h.TableOffset, + ) + } + + if h.HeaderVersion != headerVersion { + return common.NewSourceInvalidError( + "failed to check vhd header version: expected %v, actual %v", + headerVersion, + h.HeaderVersion, + ) + } + + return nil } diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/vhd/image_map_reader.go b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/image_map_reader.go index 99a6ab98391..895c7109670 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/url/vhd/image_map_reader.go +++ b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/image_map_reader.go @@ -32,7 +32,7 @@ type ImageMapReader struct { reader common.Reader header header footer footer - bat []uint32 + bat []uint32 // Block allocation table. } func (r *ImageMapReader) Size() uint64 { @@ -98,6 +98,10 @@ func (r *ImageMapReader) Read(ctx context.Context) ([]common.ImageMapItem, error return items, nil } +func (r *ImageMapReader) ReadFooter(ctx context.Context) error { + return r.readFooter(ctx) +} + //////////////////////////////////////////////////////////////////////////////// func (r *ImageMapReader) readFooter(ctx context.Context) error { diff --git a/cloud/disk_manager/internal/pkg/facade/disk_service_test/.gitignore b/cloud/disk_manager/internal/pkg/facade/disk_service_test/.gitignore new file mode 100644 index 00000000000..dc5c8d46a89 --- /dev/null +++ b/cloud/disk_manager/internal/pkg/facade/disk_service_test/.gitignore @@ -0,0 +1 @@ +disk_service_test diff --git a/cloud/disk_manager/internal/pkg/facade/image_service_test/image_service_test.go b/cloud/disk_manager/internal/pkg/facade/image_service_test/image_service_test.go index bd519e3bf31..fb05c53b0e9 100644 --- a/cloud/disk_manager/internal/pkg/facade/image_service_test/image_service_test.go +++ b/cloud/disk_manager/internal/pkg/facade/image_service_test/image_service_test.go @@ -493,18 +493,6 @@ func TestImageServiceCreateGeneratedVMDKImageFromURL(t *testing.T) { //////////////////////////////////////////////////////////////////////////////// -// Reproduces panic issue (NBS-4635). -func TestImageServiceCreateQCOW2PanicImageFromURL(t *testing.T) { - testCreateImageFromURL( - t, - testcommon.GetQCOW2PanicImageFileURL(), - testcommon.GetQCOW2PanicImageSize(t), - testcommon.GetQCOW2PanicImageCrc32(t), - ) -} - -//////////////////////////////////////////////////////////////////////////////// - func TestImageServiceCancelCreateImageFromURL(t *testing.T) { ctx := testcommon.NewContext() diff --git a/cloud/disk_manager/internal/pkg/facade/snapshot_service_test/.gitignore b/cloud/disk_manager/internal/pkg/facade/snapshot_service_test/.gitignore new file mode 100644 index 00000000000..f8af8e2e538 --- /dev/null +++ b/cloud/disk_manager/internal/pkg/facade/snapshot_service_test/.gitignore @@ -0,0 +1 @@ +snapshot_service_test diff --git a/cloud/disk_manager/internal/pkg/facade/testcommon/common.go b/cloud/disk_manager/internal/pkg/facade/testcommon/common.go index 2653bfa5734..4248a0d39eb 100644 --- a/cloud/disk_manager/internal/pkg/facade/testcommon/common.go +++ b/cloud/disk_manager/internal/pkg/facade/testcommon/common.go @@ -160,23 +160,6 @@ func GetGeneratedVMDKImageCrc32(t *testing.T) uint32 { return uint32(value) } -func GetQCOW2PanicImageFileURL() string { - port := os.Getenv("DISK_MANAGER_RECIPE_QCOW2_PANIC_IMAGE_FILE_SERVER_PORT") - return fmt.Sprintf("http://localhost:%v", port) -} - -func GetQCOW2PanicImageSize(t *testing.T) uint64 { - value, err := strconv.ParseUint(os.Getenv("DISK_MANAGER_RECIPE_QCOW2_PANIC_IMAGE_SIZE"), 10, 64) - require.NoError(t, err) - return uint64(value) -} - -func GetQCOW2PanicImageCrc32(t *testing.T) uint32 { - value, err := strconv.ParseUint(os.Getenv("DISK_MANAGER_RECIPE_QCOW2_PANIC_IMAGE_CRC32"), 10, 32) - require.NoError(t, err) - return uint32(value) -} - //////////////////////////////////////////////////////////////////////////////// func CancelOperation( diff --git a/cloud/disk_manager/internal/pkg/resources/disks.go b/cloud/disk_manager/internal/pkg/resources/disks.go index 685a11dd480..d29103240e8 100644 --- a/cloud/disk_manager/internal/pkg/resources/disks.go +++ b/cloud/disk_manager/internal/pkg/resources/disks.go @@ -95,9 +95,6 @@ func (s *diskState) toDiskMeta() *DiskMeta { FolderID: s.folderID, PlacementGroupID: s.placementGroupID, - BaseDiskID: s.baseDiskID, - BaseDiskCheckpointID: s.baseDiskCheckpointID, - CreateTaskID: s.createTaskID, CreatingAt: s.creatingAt, CreatedAt: s.createdAt, @@ -381,9 +378,6 @@ func (s *storageYDB) createDisk( folderID: disk.FolderID, placementGroupID: disk.PlacementGroupID, - baseDiskID: disk.BaseDiskID, - baseDiskCheckpointID: disk.BaseDiskCheckpointID, - createRequest: createRequest, createTaskID: disk.CreateTaskID, creatingAt: disk.CreatingAt, @@ -471,9 +465,6 @@ func (s *storageYDB) diskCreated( } state.status = diskStatusReady - // Base disk id and checkpoint id become known after disk creation. - state.baseDiskID = disk.BaseDiskID - state.baseDiskCheckpointID = disk.BaseDiskCheckpointID state.createdAt = disk.CreatedAt err = s.updateDiskState(ctx, tx, state) diff --git a/cloud/disk_manager/internal/pkg/resources/storage.go b/cloud/disk_manager/internal/pkg/resources/storage.go index 12d7134f155..866038178de 100644 --- a/cloud/disk_manager/internal/pkg/resources/storage.go +++ b/cloud/disk_manager/internal/pkg/resources/storage.go @@ -24,9 +24,6 @@ type DiskMeta struct { PlacementGroupID string `json:"placement_group_id"` PlacementPartitionIndex string `json:"placement_partition_index"` - BaseDiskID string `json:"base_disk_id"` - BaseDiskCheckpointID string `json:"base_disk_checkpoint_id"` - CreateRequest proto.Message `json:"create_request"` CreateTaskID string `json:"create_task_id"` CreatingAt time.Time `json:"creating_at"` diff --git a/cloud/disk_manager/internal/pkg/services/disks/alter_disk_task.go b/cloud/disk_manager/internal/pkg/services/disks/alter_disk_task.go index 34a7d7fbff0..567c492dec5 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/alter_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/disks/alter_disk_task.go @@ -66,7 +66,6 @@ func (t *alterDiskTask) Cancel( func (t *alterDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/disks/assign_disk_task.go b/cloud/disk_manager/internal/pkg/services/disks/assign_disk_task.go index 6a7d57d4c4c..09e296a44b3 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/assign_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/disks/assign_disk_task.go @@ -63,7 +63,6 @@ func (t *assignDiskTask) Cancel( func (t *assignDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/disks/clear_deleted_disks_task.go b/cloud/disk_manager/internal/pkg/services/disks/clear_deleted_disks_task.go index 17e3a91a823..090f2f5f1d0 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/clear_deleted_disks_task.go +++ b/cloud/disk_manager/internal/pkg/services/disks/clear_deleted_disks_task.go @@ -45,7 +45,6 @@ func (t *clearDeletedDisksTask) Cancel( func (t *clearDeletedDisksTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/disks/create_disk_from_image_task.go b/cloud/disk_manager/internal/pkg/services/disks/create_disk_from_image_task.go index c2704004883..1bb03325251 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/create_disk_from_image_task.go +++ b/cloud/disk_manager/internal/pkg/services/disks/create_disk_from_image_task.go @@ -241,7 +241,6 @@ func (t *createDiskFromImageTask) Cancel( func (t *createDiskFromImageTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { metadata := &disk_manager.CreateDiskMetadata{} diff --git a/cloud/disk_manager/internal/pkg/services/disks/create_disk_from_snapshot_task.go b/cloud/disk_manager/internal/pkg/services/disks/create_disk_from_snapshot_task.go index 448a519c2ce..fae5bbc4e46 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/create_disk_from_snapshot_task.go +++ b/cloud/disk_manager/internal/pkg/services/disks/create_disk_from_snapshot_task.go @@ -241,7 +241,6 @@ func (t *createDiskFromSnapshotTask) Cancel( func (t *createDiskFromSnapshotTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { metadata := &disk_manager.CreateDiskMetadata{} diff --git a/cloud/disk_manager/internal/pkg/services/disks/create_empty_disk_task.go b/cloud/disk_manager/internal/pkg/services/disks/create_empty_disk_task.go index 2529c3b73c6..ab5c0887c14 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/create_empty_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/disks/create_empty_disk_task.go @@ -137,7 +137,6 @@ func (t *createEmptyDiskTask) Cancel( func (t *createEmptyDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &disk_manager.CreateDiskMetadata{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/disks/create_overlay_disk_task.go b/cloud/disk_manager/internal/pkg/services/disks/create_overlay_disk_task.go index a07e43e69e5..ad394465052 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/create_overlay_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/disks/create_overlay_disk_task.go @@ -129,8 +129,6 @@ func (t *createOverlayDiskTask) Run( return err } - diskMeta.BaseDiskID = baseDiskID - diskMeta.BaseDiskCheckpointID = baseDiskCheckpointID diskMeta.CreatedAt = time.Now() return t.storage.DiskCreated(ctx, *diskMeta) @@ -193,7 +191,6 @@ func (t *createOverlayDiskTask) Cancel( func (t *createOverlayDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &disk_manager.CreateDiskMetadata{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/disks/delete_disk_task.go b/cloud/disk_manager/internal/pkg/services/disks/delete_disk_task.go index 327172482b6..a32d30e1862 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/delete_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/disks/delete_disk_task.go @@ -141,7 +141,6 @@ func (t *deleteDiskTask) Cancel( func (t *deleteDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &disk_manager.DeleteDiskMetadata{ diff --git a/cloud/disk_manager/internal/pkg/services/disks/migrate_disk_task.go b/cloud/disk_manager/internal/pkg/services/disks/migrate_disk_task.go index 1c74aac63f6..0c0011090c7 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/migrate_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/disks/migrate_disk_task.go @@ -151,7 +151,6 @@ func (t *migrateDiskTask) Cancel( func (t *migrateDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { status, err := t.getStatusForAPI(ctx) @@ -334,17 +333,6 @@ func (t *migrateDiskTask) scheduleReplicateTask( execCtx tasks.ExecutionContext, ) error { - diskMeta, err := t.resourceStorage.GetDiskMeta(ctx, t.request.Disk.DiskId) - if err != nil { - return err - } - - useLightCheckpoint := - diskMeta.Kind == diskKindToString(types.DiskKind_DISK_KIND_SSD_NONREPLICATED) || - diskMeta.Kind == diskKindToString(types.DiskKind_DISK_KIND_SSD_MIRROR2) || - diskMeta.Kind == diskKindToString(types.DiskKind_DISK_KIND_SSD_MIRROR3) || - diskMeta.Kind == diskKindToString(types.DiskKind_DISK_KIND_HDD_NONREPLICATED) - replicateTaskID, err := t.scheduler.ScheduleZonalTask( headers.SetIncomingIdempotencyKey(ctx, execCtx.GetTaskID()), "dataplane.ReplicateDisk", @@ -356,8 +344,7 @@ func (t *migrateDiskTask) scheduleReplicateTask( DiskId: t.request.Disk.DiskId, ZoneId: t.request.DstZoneId, }, - FillGeneration: t.state.FillGeneration, - UseLightCheckpoint: useLightCheckpoint, + FillGeneration: t.state.FillGeneration, // Performs full copy of base disk if |IgnoreBaseDisk == false|. IgnoreBaseDisk: len(t.state.RelocateInfo.TargetBaseDiskID) != 0, UseProxyOverlayDisk: t.disksConfig.GetUseProxyOverlayDiskInReplicateDiskTask(), diff --git a/cloud/disk_manager/internal/pkg/services/disks/migrate_disk_task_test.go b/cloud/disk_manager/internal/pkg/services/disks/migrate_disk_task_test.go index 87afa95535e..808fd04ac10 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/migrate_disk_task_test.go +++ b/cloud/disk_manager/internal/pkg/services/disks/migrate_disk_task_test.go @@ -51,7 +51,7 @@ func TestMigrateDiskTaskProgress(t *testing.T) { state: &protos.MigrateDiskTaskState{}, } - metadataRaw, err := task.GetMetadata(ctx, "migrate_task_id") + metadataRaw, err := task.GetMetadata(ctx) require.NoError(t, err) metadata, ok := metadataRaw.(*disk_manager.MigrateDiskMetadata) @@ -84,7 +84,7 @@ func TestMigrateDiskTaskProgress(t *testing.T) { UpdatedAt: replicateDiskTaskUpdatedAt, }, nil) - metadataRaw, err = task.GetMetadata(ctx, "migrate_task_id") + metadataRaw, err = task.GetMetadata(ctx) require.NoError(t, err) metadata, ok = metadataRaw.(*disk_manager.MigrateDiskMetadata) diff --git a/cloud/disk_manager/internal/pkg/services/disks/resize_disk_task.go b/cloud/disk_manager/internal/pkg/services/disks/resize_disk_task.go index f3e3202d9a1..0c8b34accdc 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/resize_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/disks/resize_disk_task.go @@ -65,7 +65,6 @@ func (t *resizeDiskTask) Cancel( func (t *resizeDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/disks/unassign_disk_task.go b/cloud/disk_manager/internal/pkg/services/disks/unassign_disk_task.go index 96b64935118..b53e0954390 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/unassign_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/disks/unassign_disk_task.go @@ -69,7 +69,6 @@ func (t *unassignDiskTask) Cancel( func (t *unassignDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/filesystem/clear_deleted_filesystems_task.go b/cloud/disk_manager/internal/pkg/services/filesystem/clear_deleted_filesystems_task.go index ad190b474ed..0309e6b8ce1 100644 --- a/cloud/disk_manager/internal/pkg/services/filesystem/clear_deleted_filesystems_task.go +++ b/cloud/disk_manager/internal/pkg/services/filesystem/clear_deleted_filesystems_task.go @@ -45,7 +45,6 @@ func (t *clearDeletedFilesystemsTask) Cancel( func (t *clearDeletedFilesystemsTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/filesystem/create_filesystem_task.go b/cloud/disk_manager/internal/pkg/services/filesystem/create_filesystem_task.go index 351d4b9de9c..ef9dfa4f688 100644 --- a/cloud/disk_manager/internal/pkg/services/filesystem/create_filesystem_task.go +++ b/cloud/disk_manager/internal/pkg/services/filesystem/create_filesystem_task.go @@ -134,7 +134,6 @@ func (t *createFilesystemTask) Cancel( func (t *createFilesystemTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/filesystem/delete_filesystem_task.go b/cloud/disk_manager/internal/pkg/services/filesystem/delete_filesystem_task.go index 4bd3e5337e6..4cebf2038ae 100644 --- a/cloud/disk_manager/internal/pkg/services/filesystem/delete_filesystem_task.go +++ b/cloud/disk_manager/internal/pkg/services/filesystem/delete_filesystem_task.go @@ -103,7 +103,6 @@ func (t *deleteFilesystemTask) Cancel( func (t *deleteFilesystemTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &disk_manager.DeleteFilesystemMetadata{ diff --git a/cloud/disk_manager/internal/pkg/services/filesystem/resize_filesystem_task.go b/cloud/disk_manager/internal/pkg/services/filesystem/resize_filesystem_task.go index c84c2498a21..e6b034e8571 100644 --- a/cloud/disk_manager/internal/pkg/services/filesystem/resize_filesystem_task.go +++ b/cloud/disk_manager/internal/pkg/services/filesystem/resize_filesystem_task.go @@ -57,7 +57,6 @@ func (t *resizeFilesystemTask) Cancel( func (t *resizeFilesystemTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/images/clear_deleted_images_task.go b/cloud/disk_manager/internal/pkg/services/images/clear_deleted_images_task.go index 3c68aaa3c9e..6eb419b0986 100644 --- a/cloud/disk_manager/internal/pkg/services/images/clear_deleted_images_task.go +++ b/cloud/disk_manager/internal/pkg/services/images/clear_deleted_images_task.go @@ -45,7 +45,6 @@ func (t *clearDeletedImagesTask) Cancel( func (t *clearDeletedImagesTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/images/create_image_from_disk_task.go b/cloud/disk_manager/internal/pkg/services/images/create_image_from_disk_task.go index aad1aaf7ea8..b9fc66f1f93 100644 --- a/cloud/disk_manager/internal/pkg/services/images/create_image_from_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/images/create_image_from_disk_task.go @@ -222,7 +222,6 @@ func (t *createImageFromDiskTask) Cancel( func (t *createImageFromDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { metadata := &disk_manager.CreateImageMetadata{} diff --git a/cloud/disk_manager/internal/pkg/services/images/create_image_from_image_task.go b/cloud/disk_manager/internal/pkg/services/images/create_image_from_image_task.go index ce7af26a571..9d60b780476 100644 --- a/cloud/disk_manager/internal/pkg/services/images/create_image_from_image_task.go +++ b/cloud/disk_manager/internal/pkg/services/images/create_image_from_image_task.go @@ -205,7 +205,6 @@ func (t *createImageFromImageTask) Cancel( func (t *createImageFromImageTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { metadata := &disk_manager.CreateImageMetadata{} diff --git a/cloud/disk_manager/internal/pkg/services/images/create_image_from_snapshot_task.go b/cloud/disk_manager/internal/pkg/services/images/create_image_from_snapshot_task.go index 007802778c8..51699c55ebe 100644 --- a/cloud/disk_manager/internal/pkg/services/images/create_image_from_snapshot_task.go +++ b/cloud/disk_manager/internal/pkg/services/images/create_image_from_snapshot_task.go @@ -205,7 +205,6 @@ func (t *createImageFromSnapshotTask) Cancel( func (t *createImageFromSnapshotTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { metadata := &disk_manager.CreateImageMetadata{} diff --git a/cloud/disk_manager/internal/pkg/services/images/create_image_from_url_task.go b/cloud/disk_manager/internal/pkg/services/images/create_image_from_url_task.go index 4eb6489ccf0..266362abef9 100644 --- a/cloud/disk_manager/internal/pkg/services/images/create_image_from_url_task.go +++ b/cloud/disk_manager/internal/pkg/services/images/create_image_from_url_task.go @@ -167,7 +167,6 @@ func (t *createImageFromURLTask) Cancel( func (t *createImageFromURLTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { metadata := &disk_manager.CreateImageMetadata{} diff --git a/cloud/disk_manager/internal/pkg/services/images/delete_image_task.go b/cloud/disk_manager/internal/pkg/services/images/delete_image_task.go index 1bf1ec61f9e..1b822b9103b 100644 --- a/cloud/disk_manager/internal/pkg/services/images/delete_image_task.go +++ b/cloud/disk_manager/internal/pkg/services/images/delete_image_task.go @@ -78,7 +78,6 @@ func (t *deleteImageTask) Cancel( func (t *deleteImageTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &disk_manager.DeleteImageMetadata{ diff --git a/cloud/disk_manager/internal/pkg/services/placementgroup/alter_placement_group_membership_task.go b/cloud/disk_manager/internal/pkg/services/placementgroup/alter_placement_group_membership_task.go index e2703cddeab..9e8c09f5980 100644 --- a/cloud/disk_manager/internal/pkg/services/placementgroup/alter_placement_group_membership_task.go +++ b/cloud/disk_manager/internal/pkg/services/placementgroup/alter_placement_group_membership_task.go @@ -67,7 +67,6 @@ func (t *alterPlacementGroupMembershipTask) Cancel( func (t *alterPlacementGroupMembershipTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/placementgroup/clear_deleted_placement_groups_task.go b/cloud/disk_manager/internal/pkg/services/placementgroup/clear_deleted_placement_groups_task.go index aa3cf0abb5a..11aab8eecdb 100644 --- a/cloud/disk_manager/internal/pkg/services/placementgroup/clear_deleted_placement_groups_task.go +++ b/cloud/disk_manager/internal/pkg/services/placementgroup/clear_deleted_placement_groups_task.go @@ -49,7 +49,6 @@ func (t *clearDeletedPlacementGroupsTask) Cancel( func (t *clearDeletedPlacementGroupsTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/placementgroup/create_placement_group_task.go b/cloud/disk_manager/internal/pkg/services/placementgroup/create_placement_group_task.go index 3babfff544e..0f4f031cbc1 100644 --- a/cloud/disk_manager/internal/pkg/services/placementgroup/create_placement_group_task.go +++ b/cloud/disk_manager/internal/pkg/services/placementgroup/create_placement_group_task.go @@ -124,7 +124,6 @@ func (t *createPlacementGroupTask) Cancel( func (t *createPlacementGroupTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/placementgroup/delete_placement_group_task.go b/cloud/disk_manager/internal/pkg/services/placementgroup/delete_placement_group_task.go index 1016c354581..e3f329935f9 100644 --- a/cloud/disk_manager/internal/pkg/services/placementgroup/delete_placement_group_task.go +++ b/cloud/disk_manager/internal/pkg/services/placementgroup/delete_placement_group_task.go @@ -98,7 +98,6 @@ func (t *deletePlacementGroupTask) Cancel( func (t *deletePlacementGroupTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &disk_manager.DeletePlacementGroupMetadata{ diff --git a/cloud/disk_manager/internal/pkg/services/pools/acquire_base_disk_task.go b/cloud/disk_manager/internal/pkg/services/pools/acquire_base_disk_task.go index 854987a22fe..4ab38f884a8 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/acquire_base_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/acquire_base_disk_task.go @@ -124,7 +124,6 @@ func (t *acquireBaseDiskTask) Cancel( func (t *acquireBaseDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/pools/clear_deleted_base_disks_task.go b/cloud/disk_manager/internal/pkg/services/pools/clear_deleted_base_disks_task.go index 3e5533416ef..62eec94f100 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/clear_deleted_base_disks_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/clear_deleted_base_disks_task.go @@ -45,7 +45,6 @@ func (t *clearDeletedBaseDisksTask) Cancel( func (t *clearDeletedBaseDisksTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/pools/clear_released_slots_task.go b/cloud/disk_manager/internal/pkg/services/pools/clear_released_slots_task.go index 37bb6078ce3..6d43f7a7caf 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/clear_released_slots_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/clear_released_slots_task.go @@ -45,7 +45,6 @@ func (t *clearReleasedSlotsTask) Cancel( func (t *clearReleasedSlotsTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/pools/configure_pool_task.go b/cloud/disk_manager/internal/pkg/services/pools/configure_pool_task.go index eb82e1ce370..7673c28d1cf 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/configure_pool_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/configure_pool_task.go @@ -93,7 +93,6 @@ func (t *configurePoolTask) Cancel( func (t *configurePoolTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil } diff --git a/cloud/disk_manager/internal/pkg/services/pools/create_base_disk_task.go b/cloud/disk_manager/internal/pkg/services/pools/create_base_disk_task.go index 4174103fe8f..aea03066492 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/create_base_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/create_base_disk_task.go @@ -205,7 +205,6 @@ func (t *createBaseDiskTask) Cancel( func (t *createBaseDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/pools/delete_base_disks_task.go b/cloud/disk_manager/internal/pkg/services/pools/delete_base_disks_task.go index 0c944dad65a..546dd5cedf0 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/delete_base_disks_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/delete_base_disks_task.go @@ -69,7 +69,6 @@ func (t *deleteBaseDisksTask) Cancel( func (t *deleteBaseDisksTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/pools/delete_pool_task.go b/cloud/disk_manager/internal/pkg/services/pools/delete_pool_task.go index 8cbe229097a..e935f1251e4 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/delete_pool_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/delete_pool_task.go @@ -65,7 +65,6 @@ func (t *deletePoolTask) Cancel( func (t *deletePoolTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/pools/image_deleting_task.go b/cloud/disk_manager/internal/pkg/services/pools/image_deleting_task.go index 61f89a78248..6c6b7fa7ce8 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/image_deleting_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/image_deleting_task.go @@ -65,7 +65,6 @@ func (t *imageDeletingTask) Cancel( func (t *imageDeletingTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/pools/optimize_base_disks_task.go b/cloud/disk_manager/internal/pkg/services/pools/optimize_base_disks_task.go index 277686031b3..9be015fcc54 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/optimize_base_disks_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/optimize_base_disks_task.go @@ -164,7 +164,6 @@ func (t *optimizeBaseDisksTask) Cancel( func (t *optimizeBaseDisksTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/pools/rebase_overlay_disk_task.go b/cloud/disk_manager/internal/pkg/services/pools/rebase_overlay_disk_task.go index ea429c20050..792be588acf 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/rebase_overlay_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/rebase_overlay_disk_task.go @@ -112,7 +112,6 @@ func (t *rebaseOverlayDiskTask) Cancel( func (t *rebaseOverlayDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/pools/release_base_disk_task.go b/cloud/disk_manager/internal/pkg/services/pools/release_base_disk_task.go index 2c723b1ee6d..861c8eebc58 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/release_base_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/release_base_disk_task.go @@ -74,7 +74,6 @@ func (t *releaseBaseDiskTask) Cancel( func (t *releaseBaseDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/pools/retire_base_disk_task.go b/cloud/disk_manager/internal/pkg/services/pools/retire_base_disk_task.go index 3d2861d2be2..b9e8b6b7296 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/retire_base_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/retire_base_disk_task.go @@ -118,7 +118,6 @@ func (t *retireBaseDiskTask) Cancel( func (t *retireBaseDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/pools/retire_base_disks_task.go b/cloud/disk_manager/internal/pkg/services/pools/retire_base_disks_task.go index 7e086d656ea..732986c266d 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/retire_base_disks_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/retire_base_disks_task.go @@ -148,7 +148,6 @@ func (t *retireBaseDisksTask) Cancel( func (t *retireBaseDisksTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/pools/schedule_base_disks_task.go b/cloud/disk_manager/internal/pkg/services/pools/schedule_base_disks_task.go index 5132a41e070..fbec2559c4a 100644 --- a/cloud/disk_manager/internal/pkg/services/pools/schedule_base_disks_task.go +++ b/cloud/disk_manager/internal/pkg/services/pools/schedule_base_disks_task.go @@ -80,7 +80,6 @@ func (t *scheduleBaseDisksTask) Cancel( func (t *scheduleBaseDisksTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/snapshots/clear_deleted_snapshots_task.go b/cloud/disk_manager/internal/pkg/services/snapshots/clear_deleted_snapshots_task.go index 0e5fd75f657..4e728870b86 100644 --- a/cloud/disk_manager/internal/pkg/services/snapshots/clear_deleted_snapshots_task.go +++ b/cloud/disk_manager/internal/pkg/services/snapshots/clear_deleted_snapshots_task.go @@ -45,7 +45,6 @@ func (t *clearDeletedSnapshotsTask) Cancel( func (t *clearDeletedSnapshotsTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/disk_manager/internal/pkg/services/snapshots/create_snapshot_from_disk_task.go b/cloud/disk_manager/internal/pkg/services/snapshots/create_snapshot_from_disk_task.go index 9702e5bf0fa..aad841cdc73 100644 --- a/cloud/disk_manager/internal/pkg/services/snapshots/create_snapshot_from_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/snapshots/create_snapshot_from_disk_task.go @@ -342,7 +342,6 @@ func (t *createSnapshotFromDiskTask) Cancel( func (t *createSnapshotFromDiskTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { metadata := &disk_manager.CreateSnapshotMetadata{} diff --git a/cloud/disk_manager/internal/pkg/services/snapshots/delete_snapshot_task.go b/cloud/disk_manager/internal/pkg/services/snapshots/delete_snapshot_task.go index 183cb937969..7379a2e2a06 100644 --- a/cloud/disk_manager/internal/pkg/services/snapshots/delete_snapshot_task.go +++ b/cloud/disk_manager/internal/pkg/services/snapshots/delete_snapshot_task.go @@ -149,7 +149,6 @@ func (t *deleteSnapshotTask) Cancel( func (t *deleteSnapshotTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &disk_manager.DeleteSnapshotMetadata{ diff --git a/cloud/disk_manager/test/acceptance/test_runner/lib/helpers.py b/cloud/disk_manager/test/acceptance/test_runner/lib/helpers.py index 305d95a67aa..394a18f5811 100644 --- a/cloud/disk_manager/test/acceptance/test_runner/lib/helpers.py +++ b/cloud/disk_manager/test/acceptance/test_runner/lib/helpers.py @@ -58,7 +58,7 @@ def _execute_command(self, args: list[str]) -> str: output = '' for line in iter(lambda: stdout.readline(2048), ''): output += line - _logger.info("stdout: %s", line) + _logger.info("stdout: %s", line.rstrip()) if stderr.channel.recv_exit_status(): stderr_lines = stderr.readlines() stderr_str = ''.join(stderr_lines) @@ -81,7 +81,7 @@ def get_virtual_disks_mapping(self) -> dict[str, str]: ) return self.get_disk_id_to_device_path_mapping(udevadm_output) - def wait_for_disk_to_appear(self, disk_id: str, timeout_sec: int = 60) -> str: + def wait_for_disk_to_appear(self, disk_id: str, timeout_sec: int = 120) -> str: _logger.info( "Started waiting for disk %s to appear in the guest system, waiting for %d", disk_id, diff --git a/cloud/disk_manager/test/images/recipe/__main__.py b/cloud/disk_manager/test/images/recipe/__main__.py index cb278c7cc35..e8cba1d48e8 100644 --- a/cloud/disk_manager/test/images/recipe/__main__.py +++ b/cloud/disk_manager/test/images/recipe/__main__.py @@ -49,6 +49,9 @@ def start(argv): vhd_image_file_server = ImageFileServerLauncher(vhd_image_file_path) vhd_image_file_server.start() set_env("DISK_MANAGER_RECIPE_VHD_IMAGE_FILE_SERVER_PORT", str(vhd_image_file_server.port)) + # size and crc32 after converting to raw image + set_env("DISK_MANAGER_RECIPE_VHD_IMAGE_SIZE", "117469184") + set_env("DISK_MANAGER_RECIPE_VHD_IMAGE_CRC32", "4215190084") image_map_file_path = yatest_common.source_path("cloud/disk_manager/test/images/recipe/data/vhd_image_map.json") set_env("DISK_MANAGER_RECIPE_VHD_IMAGE_MAP_FILE", image_map_file_path) @@ -106,17 +109,6 @@ def start(argv): image_map_file_path = yatest_common.source_path("cloud/disk_manager/test/images/recipe/data/windows_vmdk_stream_optimised_multiple_grains_image_map.json") set_env("DISK_MANAGER_RECIPE_VMDK_WINDOWS_IMAGE_MAP_FILE", image_map_file_path) - # reproduces panic issue (NBS-4635) - qcow2_panic_image_file_path = yatest_common.build_path("cloud/disk_manager/test/images/resources/qcow2_images/panic.img") - if os.path.exists(qcow2_panic_image_file_path): - qcow2_panic_image_file_server = ImageFileServerLauncher(qcow2_panic_image_file_path) - qcow2_panic_image_file_server.start() - set_env("DISK_MANAGER_RECIPE_QCOW2_PANIC_IMAGE_FILE_SERVER_PORT", str(qcow2_panic_image_file_server.port)) - set_env("DISK_MANAGER_RECIPE_QCOW2_PANIC_IMAGE_FILE_SIZE", "7348420608") - # size and crc32 after converting to raw image - set_env("DISK_MANAGER_RECIPE_QCOW2_PANIC_IMAGE_SIZE", "21474836480") - set_env("DISK_MANAGER_RECIPE_QCOW2_PANIC_IMAGE_CRC32", "3101932729") - working_dir = get_unique_path_for_current_test( output_path=yatest_common.output_path(), sub_folder="" diff --git a/cloud/disk_manager/test/images/recipe/image_file_server_launcher.py b/cloud/disk_manager/test/images/recipe/image_file_server_launcher.py index 2ebfaaed8e9..73f65a56a11 100644 --- a/cloud/disk_manager/test/images/recipe/image_file_server_launcher.py +++ b/cloud/disk_manager/test/images/recipe/image_file_server_launcher.py @@ -21,7 +21,8 @@ def __init__(self, port, working_dir, image_file_path, other_image_file_path): super(ImageFileServer, self).__init__( commands=[command], - cwd=working_dir) + cwd=working_dir, + service_name=SERVICE_NAME) class ImageFileServerLauncher: diff --git a/cloud/disk_manager/test/images/resources/ya.make b/cloud/disk_manager/test/images/resources/ya.make index b6cd6ca840d..115ad2968a2 100644 --- a/cloud/disk_manager/test/images/resources/ya.make +++ b/cloud/disk_manager/test/images/resources/ya.make @@ -14,13 +14,6 @@ FROM_SANDBOX( OUT_NOAUTO qcow2_images/ubuntu1604-ci-stable ) -FROM_SANDBOX( - FILE - 5274078903 - RENAME RESOURCE - OUT_NOAUTO qcow2_images/panic.img -) - FROM_SANDBOX( FILE 4709742882 diff --git a/cloud/filestore/README.md b/cloud/filestore/README.md index 2f3d2c3a28f..bb492626e93 100644 --- a/cloud/filestore/README.md +++ b/cloud/filestore/README.md @@ -1,6 +1,6 @@ # Prerequisites -### 1. qemu-kvm installed: +### 1. qemu-kvm installed: ``` apt-get install qemu-kvm ``` @@ -66,7 +66,7 @@ Thanks for flying NFS - Initial instruction is [here](https://virtio-fs.gitlab.io/howto-windows.html) -Please note that upon installing virtio drivers you should select _only_ virtiofs +Please note that upon installing virtio drivers you should select _only_ virtiofs or you will render vm unusable after restart ### 2. Preparing environment @@ -96,18 +96,18 @@ sudo socat TCP6-LISTEN:80,fork TCP4:127.0.0.1:3389 ### 4. Accessing filestore - By design there should be service mounting filestore starting from drive Z: -- Or you can manually run +- Or you can manually run ```promt C:\Program Files\Virtio-Win\VioFS>virtiofs.exe -D - -d -1 ``` ### 5. Bugs and limitations - So far it supports only one virtiofs drive with no way to specify specific tag -- It doesn't work: [NBS-3031](https://st.yandex-team.ru/NBS-3031) +- It doesn't work: NBS-3031 # initctl help -**NOTE**: you can chain commands but they are order sensitive. +**NOTE**: you can chain commands but they are order sensitive. ```bash ./initctl.sh stop # stop all running services @@ -117,7 +117,7 @@ C:\Program Files\Virtio-Win\VioFS>virtiofs.exe -D - -d -1 ./initctl.sh start # launches kikimr itself, kikimr storage service & vhost ./initctl.sh initialize # same as above but initializes kikimr, required at first run and after format cmd ./initctl.sh startlocal # launches passthrough to local fs storage service & vhost -./initctl.sh startnull # launches dummy storage service & vhost +./initctl.sh startnull # launches dummy storage service & vhost ./initctl.sh create # creates filestore, required for local/kikimr services at first run or after format ./initctl.sh mount # mounts filestore via fuse driver usually at ~/nfs unless configuration changed diff --git a/cloud/filestore/config/storage.proto b/cloud/filestore/config/storage.proto index 95a4f771c7b..804926b8a36 100644 --- a/cloud/filestore/config/storage.proto +++ b/cloud/filestore/config/storage.proto @@ -216,4 +216,28 @@ message TStorageConfig optional uint32 EntryTimeout = 335; optional uint32 NegativeEntryTimeout = 336; optional uint32 AttrTimeout = 337; + + // Threshold for the number of garbage blocks in a compaction range that + // triggers automatic compaction. + optional uint32 GarbageCompactionThreshold = 338; + // Threshold for average CompactionScore for the whole FS. + optional uint32 CompactionThresholdAverage = 339; + // Threshold for average GarbageCompactionScore for the whole FS. + optional uint32 GarbageCompactionThresholdAverage = 340; + // Enables 3 aforementioned thresholds. + optional bool NewCompactionEnabled = 341; + + // Threshold for average CleanupScore for the whole FS. + optional uint32 CleanupThresholdAverage = 342; + // Enables the aforementioned threshold. + optional bool NewCleanupEnabled = 343; + + // Enables GenerateBlobIds + WriteBlob + AddData instead of WriteBlob + // for writing. + optional bool ThreeStageWriteEnabled = 344; + + // When issuing blob ids, the tablet acquires a collect barrier. In order + // to release it in case of a client disconnect, this timeout is used. + optional uint32 GenerateBlobIdsReleaseCollectBarrierTimeout = 345; + } diff --git a/cloud/filestore/libs/client/tsan.supp b/cloud/filestore/libs/client/tsan.supp new file mode 100644 index 00000000000..0ad5266fd0b --- /dev/null +++ b/cloud/filestore/libs/client/tsan.supp @@ -0,0 +1,3 @@ +# There is a race between tc_on_alarm & on_writable NBS-3614 +race:tc_on_alarm +race:on_writable diff --git a/cloud/filestore/libs/daemon/common/bootstrap.cpp b/cloud/filestore/libs/daemon/common/bootstrap.cpp index 94d595b9723..e189d0c9be9 100644 --- a/cloud/filestore/libs/daemon/common/bootstrap.cpp +++ b/cloud/filestore/libs/daemon/common/bootstrap.cpp @@ -11,13 +11,13 @@ #include #include #include - #include #include #include #include #include #include +#include #include #include #include @@ -99,6 +99,7 @@ TBootstrapCommon::TBootstrapCommon( BootstrapLogging->Start(); Log = BootstrapLogging->CreateLog(logComponent); + SetCriticalEventsLog(Log); } TBootstrapCommon::~TBootstrapCommon() @@ -217,6 +218,7 @@ void TBootstrapCommon::InitCommonConfigs() Configs->InitKikimrConfig(); Configs->InitStorageConfig(); Configs->InitDiagnosticsConfig(); + Configs->InitFeaturesConfig(); } void TBootstrapCommon::InitDiagnostics() diff --git a/cloud/filestore/libs/daemon/common/config_initializer.cpp b/cloud/filestore/libs/daemon/common/config_initializer.cpp index 63f92c7fcba..668812f79fc 100644 --- a/cloud/filestore/libs/daemon/common/config_initializer.cpp +++ b/cloud/filestore/libs/daemon/common/config_initializer.cpp @@ -45,4 +45,15 @@ void TConfigInitializerCommon::InitStorageConfig() storageConfig); } +void TConfigInitializerCommon::InitFeaturesConfig() +{ + NCloud::NProto::TFeaturesConfig featuresConfig; + if (Options->FeaturesConfig) { + ParseProtoTextFromFileRobust(Options->FeaturesConfig, featuresConfig); + } + + FeaturesConfig = std::make_shared( + std::move(featuresConfig)); +} + } // namespace NCloud::NFileStore::NDaemon diff --git a/cloud/filestore/libs/daemon/common/config_initializer.h b/cloud/filestore/libs/daemon/common/config_initializer.h index 549f45f0086..0fa2850817b 100644 --- a/cloud/filestore/libs/daemon/common/config_initializer.h +++ b/cloud/filestore/libs/daemon/common/config_initializer.h @@ -5,6 +5,7 @@ #include #include +#include #include namespace NCloud::NFileStore::NDaemon { @@ -18,11 +19,13 @@ struct TConfigInitializerCommon TDiagnosticsConfigPtr DiagnosticsConfig; NStorage::TStorageConfigPtr StorageConfig; + NFeatures::TFeaturesConfigPtr FeaturesConfig; TConfigInitializerCommon(TOptionsCommonPtr options); void InitDiagnosticsConfig(); void InitStorageConfig(); + void InitFeaturesConfig(); }; } // namespace NCloud::NFileStore::NDaemon diff --git a/cloud/filestore/libs/daemon/common/options.cpp b/cloud/filestore/libs/daemon/common/options.cpp index e46d1b42ff9..780f2cea3b8 100644 --- a/cloud/filestore/libs/daemon/common/options.cpp +++ b/cloud/filestore/libs/daemon/common/options.cpp @@ -22,6 +22,10 @@ TOptionsCommon::TOptionsCommon() .RequiredArgument("FILE") .StoreResult(&StorageConfig); + Opts.AddLongOption("features-file") + .RequiredArgument("FILE") + .StoreResult(&FeaturesConfig); + Opts.AddLongOption("node-registration-attempts") .RequiredArgument("NUM") .DefaultValue(NodeRegistrationMaxAttempts) diff --git a/cloud/filestore/libs/daemon/common/options.h b/cloud/filestore/libs/daemon/common/options.h index c768d8d3a67..6c01002b130 100644 --- a/cloud/filestore/libs/daemon/common/options.h +++ b/cloud/filestore/libs/daemon/common/options.h @@ -25,6 +25,7 @@ struct TOptionsCommon { TString AppConfig; TString StorageConfig; + TString FeaturesConfig; ui32 NodeRegistrationMaxAttempts = 10; TDuration NodeRegistrationTimeout = TDuration::Seconds(10); diff --git a/cloud/filestore/libs/daemon/common/ya.make b/cloud/filestore/libs/daemon/common/ya.make index e86b1314f4f..0e06882965f 100644 --- a/cloud/filestore/libs/daemon/common/ya.make +++ b/cloud/filestore/libs/daemon/common/ya.make @@ -19,6 +19,7 @@ PEERDIR( cloud/storage/core/libs/common cloud/storage/core/libs/daemon cloud/storage/core/libs/diagnostics + cloud/storage/core/libs/features cloud/storage/core/libs/kikimr cloud/storage/core/libs/version diff --git a/cloud/filestore/libs/diagnostics/config.cpp b/cloud/filestore/libs/diagnostics/config.cpp index f987d60653c..13009fd25e2 100644 --- a/cloud/filestore/libs/diagnostics/config.cpp +++ b/cloud/filestore/libs/diagnostics/config.cpp @@ -12,7 +12,7 @@ namespace { //////////////////////////////////////////////////////////////////////////////// #define FILESTORE_DIAGNOSTICS_CONFIG(xxx) \ - xxx(BastionNameSuffix, TString, "ydb.bastion.cloud.yandex-team.ru" )\ + xxx(BastionNameSuffix, TString, "" )\ xxx(FilestoreMonPort, ui32, 8767 )\ \ xxx(SamplingRate, ui32, 0 )\ diff --git a/cloud/filestore/libs/diagnostics/critical_events.cpp b/cloud/filestore/libs/diagnostics/critical_events.cpp index 2c2284605fc..ede8c868ace 100644 --- a/cloud/filestore/libs/diagnostics/critical_events.cpp +++ b/cloud/filestore/libs/diagnostics/critical_events.cpp @@ -19,19 +19,46 @@ void InitCriticalEventsCounter(NMonitoring::TDynamicCountersPtr counters) // FILESTORE_INIT_CRITICAL_EVENT_COUNTER FILESTORE_CRITICAL_EVENTS(FILESTORE_INIT_CRITICAL_EVENT_COUNTER) + FILESTORE_IMPOSSIBLE_EVENTS(FILESTORE_INIT_CRITICAL_EVENT_COUNTER) #undef FILESTORE_INIT_CRITICAL_EVENT_COUNTER NCloud::InitCriticalEventsCounter(std::move(counters)); } #define FILESTORE_DEFINE_CRITICAL_EVENT_ROUTINE(name) \ - void Report##name() \ + TString Report##name(const TString& message) \ { \ - ReportCriticalEvent("AppCriticalEvents/"#name, "", false); \ + return ReportCriticalEvent( \ + GetCriticalEventFor##name(), \ + message, \ + false); \ } \ + \ + const TString GetCriticalEventFor##name() \ + { \ + return "AppCriticalEvents/"#name; \ + } \ // FILESTORE_DEFINE_CRITICAL_EVENT_ROUTINE FILESTORE_CRITICAL_EVENTS(FILESTORE_DEFINE_CRITICAL_EVENT_ROUTINE) #undef FILESTORE_DEFINE_CRITICAL_EVENT_ROUTINE +#define FILESTORE_DEFINE_IMPOSSIBLE_EVENT_ROUTINE(name) \ + TString Report##name(const TString& message) \ + { \ + return ReportCriticalEvent( \ + GetCriticalEventFor##name(), \ + message, \ + true); \ + } \ + \ + const TString GetCriticalEventFor##name() \ + { \ + return "AppCriticalEvents/"#name; \ + } \ +// FILESTORE_DEFINE_IMPOSSIBLE_EVENT_ROUTINE + + FILESTORE_IMPOSSIBLE_EVENTS(FILESTORE_DEFINE_IMPOSSIBLE_EVENT_ROUTINE) +#undef FILESTORE_DEFINE_IMPOSSIBLE_EVENT_ROUTINE + } // namespace NCloud::NFileStore diff --git a/cloud/filestore/libs/diagnostics/critical_events.h b/cloud/filestore/libs/diagnostics/critical_events.h index 381d79fe017..eba654aba21 100644 --- a/cloud/filestore/libs/diagnostics/critical_events.h +++ b/cloud/filestore/libs/diagnostics/critical_events.h @@ -13,20 +13,34 @@ namespace NCloud::NFileStore{ xxx(TabletBSFailure) \ xxx(TabletCommitIdOverflow) \ xxx(VfsQueueRunningError) \ + xxx(EndpointStartingError) \ xxx(MissingSessionId) \ xxx(CreateSessionError) \ xxx(DescribeFileStoreError) \ // FILESTORE_CRITICAL_EVENTS +#define FILESTORE_IMPOSSIBLE_EVENTS(xxx) \ + xxx(CancelRoutineIsNotSet) \ +// FILESTORE_IMPOSSIBLE_EVENTS + //////////////////////////////////////////////////////////////////////////////// void InitCriticalEventsCounter(NMonitoring::TDynamicCountersPtr counters); #define FILESTORE_DECLARE_CRITICAL_EVENT_ROUTINE(name) \ - void Report##name(); \ + TString Report##name(const TString& message = ""); \ + const TString GetCriticalEventFor##name(); \ // FILESTORE_DECLARE_CRITICAL_EVENT_ROUTINE FILESTORE_CRITICAL_EVENTS(FILESTORE_DECLARE_CRITICAL_EVENT_ROUTINE) #undef FILESTORE_DECLARE_CRITICAL_EVENT_ROUTINE +#define FILESTORE_DECLARE_IMPOSSIBLE_EVENT_ROUTINE(name) \ + TString Report##name(const TString& message = ""); \ + const TString GetCriticalEventFor##name(); \ +// FILESTORE_DECLARE_IMPOSSIBLE_EVENT_ROUTINE + + FILESTORE_IMPOSSIBLE_EVENTS(FILESTORE_DECLARE_IMPOSSIBLE_EVENT_ROUTINE) +#undef FILESTORE_DECLARE_IMPOSSIBLE_EVENT_ROUTINE + } // namespace NCloud::NFileStore diff --git a/cloud/filestore/libs/endpoint/service.cpp b/cloud/filestore/libs/endpoint/service.cpp index 9e9a3751cba..2b8adf1e836 100644 --- a/cloud/filestore/libs/endpoint/service.cpp +++ b/cloud/filestore/libs/endpoint/service.cpp @@ -331,10 +331,7 @@ NProto::TStopEndpointResponse TEndpointManager::DoStopEndpoint( const auto& response = future.GetValue(); if (SUCCEEDED(response.GetError().GetCode())) { - auto config = it->second.Config; - if (config.GetPersistent()) { - Storage->RemoveEndpoint(request.GetSocketPath()); - } + Storage->RemoveEndpoint(request.GetSocketPath()); } Endpoints.erase(it); diff --git a/cloud/filestore/libs/server/server.cpp b/cloud/filestore/libs/server/server.cpp index 96124733714..47647a97b76 100644 --- a/cloud/filestore/libs/server/server.cpp +++ b/cloud/filestore/libs/server/server.cpp @@ -3,6 +3,7 @@ #include "config.h" #include "probes.h" +#include #include #include #include @@ -23,6 +24,9 @@ #include #include #include +#include +#include +#include #include #include @@ -176,37 +180,6 @@ const NCloud::TRequestSourceKinds RequestSourceKinds = { { "SECURE_CONTROL_CHANNEL", NProto::SOURCE_SECURE_CONTROL_CHANNEL }, }; -void ValidateRequest( - const grpc::ServerContext& context, - NProto::THeaders& headers) -{ - auto authContext = context.auth_context(); - Y_ABORT_UNLESS(authContext); - - auto source = GetRequestSource( - *authContext, - RequestSourceKinds); - - if (source.Empty()) { - *source = NProto::SOURCE_FD_DATA_CHANNEL; - } - - if (headers.HasInternal()) { - ythrow TServiceError(E_ARGUMENT) - << "internal field should not be set by client"; - } - - auto& internal = *headers.MutableInternal(); - - internal.Clear(); - internal.SetRequestSource(*source); - - // we will only get token from secure control channel - if (source == NProto::SOURCE_SECURE_CONTROL_CHANNEL) { - internal.SetAuthToken(GetAuthToken(context.client_metadata())); - } -} - //////////////////////////////////////////////////////////////////////////////// class TServerRequestHandlerBase @@ -240,26 +213,277 @@ class TServerRequestHandlerBase //////////////////////////////////////////////////////////////////////////////// +class TSessionStorage; + struct TAppContext { TLog Log; TAtomic ShouldStop = 0; IRequestStatsPtr Stats; IProfileLogPtr ProfileLog; + std::unique_ptr Server; + std::shared_ptr SessionStorage; + + TAppContext() + : SessionStorage(std::make_shared(*this)) + {} }; -struct TFileStoreContext : TAppContext +struct TFileStoreContext: TAppContext { NProto::TFileStoreService::AsyncService Service; IFileStoreServicePtr ServiceImpl; + + void ValidateRequest( + const grpc::ServerContext& context, + NProto::THeaders& headers) + { + auto authContext = context.auth_context(); + Y_ABORT_UNLESS(authContext); + + auto source = GetRequestSource( + *authContext, + RequestSourceKinds); + + if (!source) { + source = NProto::SOURCE_SECURE_CONTROL_CHANNEL; + } + + if (headers.HasInternal()) { + ythrow TServiceError(E_ARGUMENT) + << "internal field should not be set by client"; + } + + auto& internal = *headers.MutableInternal(); + + internal.Clear(); + internal.SetRequestSource(*source); + + // we will only get token from secure control channel + if (source == NProto::SOURCE_SECURE_CONTROL_CHANNEL) { + internal.SetAuthToken(GetAuthToken(context.client_metadata())); + } + } }; struct TEndpointManagerContext : TAppContext { NProto::TEndpointManagerService::AsyncService Service; IEndpointManagerPtr ServiceImpl; + + void ValidateRequest( + const grpc::ServerContext& context, + NProto::THeaders& headers); +}; + +//////////////////////////////////////////////////////////////////////////////// + +using NCloud::NStorage::NServer::IClientStorage; +using NCloud::NStorage::NServer::IClientStoragePtr; + +//////////////////////////////////////////////////////////////////////////////// + +class TSessionStorage final + : public std::enable_shared_from_this +{ +private: + struct TClientInfo + { + ui32 Fd = 0; + NProto::ERequestSource Source = NProto::SOURCE_FD_DATA_CHANNEL; + }; + +private: + TAppContext& AppCtx; + + TMutex Lock; + THashMap ClientInfos; + +public: + TSessionStorage(TAppContext& appCtx) + : AppCtx(appCtx) + {} + + void AddClient( + const TSocketHolder& socket, + NProto::ERequestSource source) + { + if (AtomicGet(AppCtx.ShouldStop)) { + return; + } + + TSocketHolder dupSocket; + + with_lock (Lock) { + auto it = FindClient(socket); + Y_ABORT_UNLESS(it == ClientInfos.end()); + + // create duplicate socket. we poll socket to know when + // client disconnects and dupSocket is passed to GRPC to read + // messages. + dupSocket = SafeCreateDuplicate(socket); + + TClientInfo client {static_cast(socket), source}; + auto res = ClientInfos.emplace((ui32)dupSocket, std::move(client)); + Y_ABORT_UNLESS(res.second); + } + + TLog& Log = AppCtx.Log; + STORAGE_DEBUG( + "Accept client. Unix socket fd = " << + static_cast(dupSocket)); + grpc::AddInsecureChannelFromFd(AppCtx.Server.get(), dupSocket.Release()); + } + + void RemoveClient(const TSocketHolder& socket) + { + if (AtomicGet(AppCtx.ShouldStop)) { + return; + } + + with_lock (Lock) { + auto it = FindClient(socket); + Y_ABORT_UNLESS(it != ClientInfos.end()); + ClientInfos.erase(it); + } + } + + std::optional FindSourceByFd(ui32 fd) + { + with_lock (Lock) { + auto it = ClientInfos.find(fd); + if (it != ClientInfos.end()) { + const auto& clientInfo = it->second; + return clientInfo.Source; + } + } + return {}; + } + + IClientStoragePtr CreateClientStorage(); + +private: + static TSocketHolder CreateDuplicate(const TSocketHolder& socket) + { + auto fileHandle = TFileHandle(socket); + auto duplicateFd = fileHandle.Duplicate(); + fileHandle.Release(); + return TSocketHolder(duplicateFd); + } + + // Grpc can release socket before we call RemoveClient. So it is possible + // to observe the situation when CreateDuplicate returns fd which was not + // yet removed from ClientInfo's. So completed RemoveClient will close fd + // related to another connection. + TSocketHolder SafeCreateDuplicate(const TSocketHolder& socket) + { + TList holders; + + while (true) { + TSocketHolder holder = CreateDuplicate(socket); + if (ClientInfos.find(holder) == ClientInfos.end()) { + return holder; + } + + holders.push_back(std::move(holder)); + } + } + + THashMap::iterator FindClient(const TSocketHolder& socket) + { + ui32 fd = socket; + for (auto it = ClientInfos.begin(); it != ClientInfos.end(); ++it) { + const auto& clientInfo = it->second; + if (clientInfo.Fd == fd) { + return it; + } + } + return ClientInfos.end(); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TClientStorage final + : public IClientStorage +{ + std::shared_ptr Storage; + +public: + TClientStorage( + std::shared_ptr storage) + : Storage(std::move(storage)) + {} + + void AddClient( + const TSocketHolder& clientSocket, + NCloud::NProto::ERequestSource source) override + { + Storage->AddClient(clientSocket, source); + } + + void RemoveClient(const TSocketHolder& clientSocket) override + { + Storage->RemoveClient(clientSocket); + } }; +IClientStoragePtr TSessionStorage::CreateClientStorage() +{ + return std::make_shared( + this->shared_from_this()); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TEndpointManagerContext::ValidateRequest( + const grpc::ServerContext& context, + NProto::THeaders& headers) +{ + auto authContext = context.auth_context(); + Y_ABORT_UNLESS(authContext); + + auto source = GetRequestSource( + *authContext, + RequestSourceKinds); + + if (!source) { + ui32 fd = 0; + auto peer = context.peer(); + bool result = TryParseSourceFd(peer, &fd); + + if (!result) { + ythrow TServiceError(E_FAIL) + << "failed to parse request source fd: " << peer; + } + + // requests coming from unix sockets do not need to be authorized + // so pretend they are coming from data channel. + auto src = SessionStorage->FindSourceByFd(fd); + if (!src) { + ythrow TServiceError(E_GRPC_UNAVAILABLE) + << "endpoint has been stopped (fd = " << fd << ")."; + } + + source = *src; + } + + if (headers.HasInternal()) { + ythrow TServiceError(E_ARGUMENT) + << "internal field should not be set by client"; + } + + auto& internal = *headers.MutableInternal(); + + internal.Clear(); + internal.SetRequestSource(*source); + + // we will only get token from secure control channel + if (source == NProto::SOURCE_SECURE_CONTROL_CHANNEL) { + internal.SetAuthToken(GetAuthToken(context.client_metadata())); + } +} + //////////////////////////////////////////////////////////////////////////////// using TRequestsInFlight = @@ -473,7 +697,7 @@ class TRequestHandler final Started = true; try { - ValidateRequest( + AppCtx.ValidateRequest( *Context, *Request->MutableHeaders()); @@ -803,7 +1027,7 @@ class TStreamRequestHandler final // TODO report stats try { - ValidateRequest( + AppCtx.ValidateRequest( *Context, *Request->MutableHeaders()); @@ -926,6 +1150,8 @@ class TServer final TAdaptiveLock ExecutorsLock; TVector> Executors; + std::unique_ptr EndpointPoller; + public: template TServer( @@ -1015,18 +1241,24 @@ class TServer final auto executor = std::make_unique( TStringBuilder() << "SRV" << i, builder.AddCompletionQueue(), - AppCtx.Log); + Log); executor->Start(); Executors.push_back(std::move(executor)); } } - Server = builder.BuildAndStart(); - if (!Server) { + AppCtx.Server = builder.BuildAndStart(); + if (!AppCtx.Server) { ythrow TServiceError(E_FAIL) << "could not start gRPC server"; } + auto unixSocketPath = Config->GetUnixSocketPath(); + if (unixSocketPath) { + ui32 backlog = Config->GetUnixSocketBacklog(); + StartListenUnixSocket(unixSocketPath, backlog); + } + with_lock (ExecutorsLock) { for (auto& executor: Executors) { for (ui32 i = 0; i < Config->GetPreparedRequestsCount(); ++i) { @@ -1046,11 +1278,17 @@ class TServer final STORAGE_INFO("Shutting down"); + // UDS endpoints are allowed at nfs-vhost only + if constexpr (std::is_same::value) + { + StopListenUnixSocket(); + } + AppCtx.Stats->Reset(); auto deadline = Config->GetShutdownTimeout().ToDeadLine(); - if (Server) { - Server->Shutdown(deadline); + if (AppCtx.Server) { + AppCtx.Server->Shutdown(deadline); } for (;;) { @@ -1123,6 +1361,37 @@ class TServer final return sslOptions; } + + void StartListenUnixSocket(const TString& unixSocketPath, ui32 backlog) + { + auto& Log = AppCtx.Log; + + STORAGE_INFO("Listen on (control) " << unixSocketPath.Quote()); + + EndpointPoller = std::make_unique(); + EndpointPoller->Start(); + + auto error = EndpointPoller->StartListenEndpoint( + unixSocketPath, + backlog, + true, // multiClient + NProto::SOURCE_FD_CONTROL_CHANNEL, + AppCtx.SessionStorage->CreateClientStorage()); + + if (HasError(error)) { + ReportEndpointStartingError(); + STORAGE_ERROR("Failed to start (control) endpoint: " << FormatError(error)); + StopListenUnixSocket(); + } + } + + void StopListenUnixSocket() + { + if (EndpointPoller) { + EndpointPoller->Stop(); + EndpointPoller.reset(); + } + } }; using TFileStoreServer = TServer; diff --git a/cloud/filestore/libs/server/server_ut.cpp b/cloud/filestore/libs/server/server_ut.cpp index c9573f8b584..bf807e23534 100644 --- a/cloud/filestore/libs/server/server_ut.cpp +++ b/cloud/filestore/libs/server/server_ut.cpp @@ -5,9 +5,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -19,6 +21,7 @@ #include #include +#include namespace NCloud::NFileStore::NServer { @@ -165,6 +168,11 @@ class TTestServerBuilder final { return ServerConfig; } + + void SetUnixSocketPath(const TString& path) + { + ServerConfig.SetUnixSocketPath(path); + } }; //////////////////////////////////////////////////////////////////////////////// @@ -197,18 +205,83 @@ class TTestClientBuilder final { return ClientConfig; } + + void SetUnixSocketPath(const TString& path) + { + ClientConfig.SetUnixSocketPath(path); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TServerSetup +{ + using TService = TFileStoreTest; + using TClientIntf = IFileStoreServicePtr; + + static TClientIntf CreateClient( + TClientConfigPtr config, + ILoggingServicePtr logging) + { + return CreateFileStoreClient(std::move(config), std::move(logging)); + } + + static IServerPtr CreateTestServer( + TServerConfigPtr config, + ILoggingServicePtr logging, + IRequestStatsPtr requestStats, + IFileStoreServicePtr service) + { + return CreateServer( + std::move(config), + std::move(logging), + std::move(requestStats), + CreateProfileLogStub(), + std::move(service)); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TVHostSetup +{ + using TService = TEndpointManagerTest; + using TClientIntf = IEndpointManagerPtr; + + static TClientIntf CreateClient( + TClientConfigPtr config, + ILoggingServicePtr logging) + { + return CreateEndpointManagerClient( + std::move(config), + std::move(logging)); + } + + static IServerPtr CreateTestServer( + TServerConfigPtr config, + ILoggingServicePtr logging, + IRequestStatsPtr requestStats, + IEndpointManagerPtr service) + { + return CreateServer( + std::move(config), + std::move(logging), + std::move(requestStats), + std::move(service)); + } }; //////////////////////////////////////////////////////////////////////////////// +template struct TBootstrap { NMonitoring::TDynamicCountersPtr Counters; IServerPtr Server; - TVector Clients; + TVector Clients; ILoggingServicePtr Logging; - std::shared_ptr Service; + std::shared_ptr Service; TServerConfigPtr ServerConfig; bool Stopped = false; @@ -218,7 +291,7 @@ struct TBootstrap std::shared_ptr logBackend = nullptr) : Counters{MakeIntrusive()} , Logging{logBackend ? CreateLoggingService(logBackend) : CreateLoggingService("console")} - , Service{std::make_shared()} + , Service{std::make_shared()} { auto registry = CreateRequestStatsRegistry( "server_ut", @@ -229,11 +302,10 @@ struct TBootstrap ServerConfig = CreateConfig(serverConfig); - Server = CreateServer( + Server = TSetup::CreateTestServer( ServerConfig, Logging, registry->GetRequestStats(), - CreateProfileLogStub(), Service); CreateClient(); @@ -263,7 +335,7 @@ struct TBootstrap } } - IFileStoreServicePtr CreateClient(NProto::TClientConfig config = {}) + typename TSetup::TClientIntf CreateClient(NProto::TClientConfig config = {}) { auto clientConfig = config; if (!clientConfig.GetSecurePort() && clientConfig.GetRootCertsFile()) { @@ -273,7 +345,7 @@ struct TBootstrap clientConfig.SetPort(ServerConfig->GetPort()); } Clients.push_back( - CreateFileStoreClient( + TSetup::CreateClient( std::make_shared(clientConfig), Logging)); return Clients.back(); @@ -301,12 +373,13 @@ Y_UNIT_TEST_SUITE(TServerTest) { Y_UNIT_TEST(ShouldHandleRequests) { - TBootstrap bootstrap; + TBootstrap bootstrap; bootstrap.Service->PingHandler = [] (auto, auto) { return MakeFuture(); }; + bootstrap.CreateClient(); bootstrap.Start(); auto context = MakeIntrusive("fs"); @@ -323,7 +396,7 @@ Y_UNIT_TEST_SUITE(TServerTest) Y_UNIT_TEST(CheckLoggingPriority) { auto logBackend = std::make_shared(); - TBootstrap bootstrap({}, logBackend); + TBootstrap bootstrap({}, logBackend); bootstrap.Service->PingHandler = [] (auto, auto) { return MakeFuture(); @@ -333,6 +406,7 @@ Y_UNIT_TEST_SUITE(TServerTest) return MakeFuture(); }; + bootstrap.CreateClient(); bootstrap.Start(); auto context = MakeIntrusive("fs"); @@ -363,13 +437,14 @@ Y_UNIT_TEST_SUITE(TServerTest) { TAtomicStorage> storage; - TBootstrap bootstrap; + TBootstrap bootstrap; bootstrap.Service->GetSessionEventsStreamHandler = [&] (auto callContext, auto request, auto responseHandler) { Y_UNUSED(callContext, request); storage.Set(responseHandler); }; + bootstrap.CreateClient(); bootstrap.Start(); auto context = MakeIntrusive("fs"); @@ -397,11 +472,12 @@ Y_UNIT_TEST_SUITE(TServerTest) Y_UNIT_TEST(ShouldHitErrorMetricOnFailure) { - TBootstrap bootstrap; + TBootstrap bootstrap; bootstrap.Service->CreateNodeHandler = [] (auto, auto) { return MakeFuture(TErrorResponse(E_IO, "")); }; + bootstrap.CreateClient(); bootstrap.Start(); auto context = MakeIntrusive("fs"); @@ -429,7 +505,7 @@ Y_UNIT_TEST_SUITE(TServerTest) "certs/server.crt", {{"certs/server.crt", "certs/server.key"}}); - TBootstrap bootstrap(serverConfigBuilder.BuildServerConfig()); + TBootstrap bootstrap(serverConfigBuilder.BuildServerConfig()); bootstrap.Service->PingHandler = [&] (auto callContext, auto request) { Y_UNUSED(callContext); @@ -471,7 +547,7 @@ Y_UNIT_TEST_SUITE(TServerTest) "certs/server.crt", "certs/server.key"); - TBootstrap bootstrap(serverConfigBuilder.BuildServerConfig()); + TBootstrap bootstrap(serverConfigBuilder.BuildServerConfig()); bootstrap.Service->PingHandler = [&] (auto callContext, auto request) { Y_UNUSED(callContext); @@ -507,7 +583,7 @@ Y_UNIT_TEST_SUITE(TServerTest) "certs/server.crt", {{"certs/server.crt", "certs/server.key"}}); - TBootstrap bootstrap(serverConfigBuilder.BuildServerConfig()); + TBootstrap bootstrap(serverConfigBuilder.BuildServerConfig()); bootstrap.Service->PingHandler = [&] (auto callContext, auto request) { Y_UNUSED(callContext); @@ -515,6 +591,8 @@ Y_UNIT_TEST_SUITE(TServerTest) return MakeFuture(); }; + bootstrap.CreateClient(); + TTestClientBuilder clientConfigBuilder; clientConfigBuilder.SetSecureEndpoint( "certs/server.crt", @@ -547,7 +625,7 @@ Y_UNIT_TEST_SUITE(TServerTest) Y_UNIT_TEST(ShouldFailRequestWithNonEmptyInternalHeaders) { - TBootstrap bootstrap; + TBootstrap bootstrap; bootstrap.Service->PingHandler = [&] (auto callContext, auto request) { Y_UNUSED(callContext); @@ -570,6 +648,88 @@ Y_UNIT_TEST_SUITE(TServerTest) const auto& response = future.GetValue(TDuration::Seconds(5)); UNIT_ASSERT_VALUES_EQUAL(E_ARGUMENT, response.GetError().GetCode()); } + + Y_UNIT_TEST(ShouldIdentifyFdControlChannelSource) + { + TFsPath unixSocket(CreateGuidAsString() + ".sock"); + + TTestServerBuilder serverConfigBuilder; + serverConfigBuilder.SetUnixSocketPath(unixSocket.GetPath()); + + TBootstrap bootstrap( + serverConfigBuilder.BuildServerConfig()); + bootstrap.Service->PingHandler = + [&] (auto request) { + UNIT_ASSERT_VALUES_EQUAL( + int(NProto::SOURCE_FD_CONTROL_CHANNEL), + int(request->GetHeaders().GetInternal().GetRequestSource()) + ); + return MakeFuture(); + }; + + bootstrap.Start(); + + TTestClientBuilder clientConfigBuilder; + clientConfigBuilder.SetUnixSocketPath(unixSocket.GetPath()); + auto client = bootstrap.CreateClient(clientConfigBuilder.BuildClientConfig()); + client->Start(); + + auto request = std::make_shared(); + auto future = client->Ping( + MakeIntrusive(), + std::move(request) + ); + + const auto& response = future.GetValue(TDuration::Seconds(5)); + UNIT_ASSERT_C(!HasError(response), response.GetError()); + } + + Y_UNIT_TEST(ShouldReportCriticalEventIfFailedToStartUnixSocketEndpoint) + { + TFsPath unixSocket("./invalid/path/test_socket"); + + TTestServerBuilder serverConfigBuilder; + serverConfigBuilder.SetUnixSocketPath(unixSocket.GetPath()); + + TBootstrap bootstrap( + serverConfigBuilder.BuildServerConfig()); + auto errorCounter = + bootstrap.Counters-> + GetSubgroup("component", "server_ut")-> + GetCounter("AppCriticalEvents/EndpointStartingError", true); + + UNIT_ASSERT_VALUES_EQUAL(0, static_cast(*errorCounter)); + bootstrap.Start(); + UNIT_ASSERT_VALUES_EQUAL(1, static_cast(*errorCounter)); + } + + Y_UNIT_TEST(ShouldNotStartUnixSocketEndpointForNonVHostServer) + { + TFsPath unixSocket("./invalid/path/test_socket"); + + TTestServerBuilder serverConfigBuilder; + serverConfigBuilder.SetUnixSocketPath(unixSocket.GetPath()); + + TBootstrap bootstrap( + serverConfigBuilder.BuildServerConfig()); + + bootstrap.Start(); + + TTestClientBuilder clientConfigBuilder; + clientConfigBuilder.SetUnixSocketPath(unixSocket.GetPath()); + auto client = bootstrap.CreateClient(clientConfigBuilder.BuildClientConfig()); + + bool gotException = false; + try { + client->Start(); + } catch (const TServiceError& ex) { + gotException = true; + } + + UNIT_ASSERT_C( + gotException, + "Client connected to socket that should not be listened"); + } } } // namespace NCloud::NFileStore::NServer diff --git a/cloud/filestore/libs/server/tsan.supp b/cloud/filestore/libs/server/tsan.supp new file mode 100644 index 00000000000..7d1039d2522 --- /dev/null +++ b/cloud/filestore/libs/server/tsan.supp @@ -0,0 +1,2 @@ +# There is race between c_call() & set_call() +race:server.cpp diff --git a/cloud/filestore/libs/server/ya.make b/cloud/filestore/libs/server/ya.make index 5e286a30a93..feeb78565a2 100644 --- a/cloud/filestore/libs/server/ya.make +++ b/cloud/filestore/libs/server/ya.make @@ -17,6 +17,7 @@ PEERDIR( cloud/storage/core/libs/common cloud/storage/core/libs/diagnostics cloud/storage/core/libs/grpc + cloud/storage/core/libs/uds cloud/storage/core/protos library/cpp/deprecated/atomic diff --git a/cloud/filestore/libs/storage/api/components.h b/cloud/filestore/libs/storage/api/components.h index ce63470a057..d8de97ce3f8 100644 --- a/cloud/filestore/libs/storage/api/components.h +++ b/cloud/filestore/libs/storage/api/components.h @@ -2,6 +2,8 @@ #include "public.h" +#include + #include #include @@ -33,6 +35,7 @@ namespace NCloud::NFileStore::NStorage { xxx(FUSE) \ xxx(CLIENT) \ xxx(AUTH) \ + xxx(USER_STATS) \ // FILESTORE_COMPONENTS //////////////////////////////////////////////////////////////////////////////// @@ -41,7 +44,7 @@ struct TFileStoreComponents { enum { - START = 2048, // TODO + START = TComponentsStart::FileStoreComponentsStart, #define FILESTORE_DECLARE_COMPONENT(component) \ component, \ @@ -59,22 +62,6 @@ const TString& GetComponentName(int component); //////////////////////////////////////////////////////////////////////////////// -struct TFileStoreActivities -{ - enum - { -#define FILESTORE_DECLARE_COMPONENT(component) \ - component = NKikimrServices::TActivity::FILESTORE_##component, \ -// FILESTORE_DECLARE_COMPONENT - - FILESTORE_ACTORS(FILESTORE_DECLARE_COMPONENT) - -#undef FILESTORE_DECLARE_COMPONENT - }; -}; - -//////////////////////////////////////////////////////////////////////////////// - struct TFileStoreEvents { enum diff --git a/cloud/filestore/libs/storage/api/tablet.h b/cloud/filestore/libs/storage/api/tablet.h index 4c5cd91a5e5..d88758b18fd 100644 --- a/cloud/filestore/libs/storage/api/tablet.h +++ b/cloud/filestore/libs/storage/api/tablet.h @@ -24,6 +24,8 @@ namespace NCloud::NFileStore::NStorage { xxx(ChangeStorageConfig, __VA_ARGS__) \ xxx(DescribeData, __VA_ARGS__) \ xxx(DescribeSessions, __VA_ARGS__) \ + xxx(GenerateBlobIds, __VA_ARGS__) \ + xxx(AddData, __VA_ARGS__) \ // FILESTORE_TABLET_REQUESTS //////////////////////////////////////////////////////////////////////////////// @@ -65,6 +67,12 @@ struct TEvIndexTablet EvDescribeSessionsRequest = EvBegin + 17, EvDescribeSessionsResponse, + EvGenerateBlobIdsRequest = EvBegin + 19, + EvGenerateBlobIdsResponse, + + EvAddDataRequest = EvBegin + 21, + EvAddDataResponse, + EvEnd }; diff --git a/cloud/filestore/libs/storage/api/ya.make b/cloud/filestore/libs/storage/api/ya.make index aff26d0ef2e..1aea131ec05 100644 --- a/cloud/filestore/libs/storage/api/ya.make +++ b/cloud/filestore/libs/storage/api/ya.make @@ -15,6 +15,7 @@ PEERDIR( cloud/filestore/private/api/protos cloud/filestore/public/api/protos cloud/storage/core/libs/common + cloud/storage/core/libs/kikimr contrib/ydb/library/actors/core contrib/ydb/core/base contrib/ydb/core/protos diff --git a/cloud/filestore/libs/storage/core/config.cpp b/cloud/filestore/libs/storage/core/config.cpp index eaf9b22961c..d30c966af53 100644 --- a/cloud/filestore/libs/storage/core/config.cpp +++ b/cloud/filestore/libs/storage/core/config.cpp @@ -26,17 +26,23 @@ namespace { xxx(WriteBatchTimeout, TDuration, TDuration::MilliSeconds(0) )\ xxx(WriteBlobThreshold, ui32, 128_KB )\ \ - xxx(MaxBlobSize, ui32, 4_MB )\ - xxx(FlushThreshold, ui32, 4_MB )\ - xxx(CleanupThreshold, ui32, 512 )\ - xxx(CompactionThreshold, ui32, 20 )\ - xxx(CollectGarbageThreshold, ui32, 4_MB )\ - xxx(FlushBytesThreshold, ui32, 4_MB )\ - xxx(MaxDeleteGarbageBlobsPerTx, ui32, 16384 )\ - xxx(LoadedCompactionRangesPerTx, ui32, 10 * 1024 * 1024 )\ - xxx(MaxBlocksPerTruncateTx, ui32, 0 /*TODO: 8388608 32gb/4kb*/)\ - xxx(MaxTruncateTxInflight, ui32, 10 )\ - xxx(CompactionRetryTimeout, TDuration, TDuration::Seconds(1) )\ + xxx(MaxBlobSize, ui32, 4_MB )\ + xxx(FlushThreshold, ui32, 4_MB )\ + xxx(CleanupThreshold, ui32, 512 )\ + xxx(CleanupThresholdAverage, ui32, 64 )\ + xxx(NewCleanupEnabled, bool, false )\ + xxx(CompactionThreshold, ui32, 20 )\ + xxx(GarbageCompactionThreshold, ui32, 100 )\ + xxx(CompactionThresholdAverage, ui32, 4 )\ + xxx(GarbageCompactionThresholdAverage, ui32, 20 )\ + xxx(NewCompactionEnabled, bool, false )\ + xxx(CollectGarbageThreshold, ui32, 4_MB )\ + xxx(FlushBytesThreshold, ui32, 4_MB )\ + xxx(MaxDeleteGarbageBlobsPerTx, ui32, 16384 )\ + xxx(LoadedCompactionRangesPerTx, ui32, 10 * 1024 * 1024 )\ + xxx(MaxBlocksPerTruncateTx, ui32, 0 /*TODO: 32GiB/4KiB*/ )\ + xxx(MaxTruncateTxInflight, ui32, 10 )\ + xxx(CompactionRetryTimeout, TDuration, TDuration::Seconds(1) )\ \ xxx(FlushThresholdForBackpressure, ui32, 128_MB )\ xxx(CleanupThresholdForBackpressure, ui32, 32768 )\ @@ -133,11 +139,16 @@ namespace { NCloud::NProto::AUTHORIZATION_IGNORE )\ \ xxx(TwoStageReadEnabled, bool, false )\ + xxx(ThreeStageWriteEnabled, bool, false )\ xxx(EntryTimeout, TDuration, TDuration::Zero() )\ xxx(NegativeEntryTimeout, TDuration, TDuration::Zero() )\ xxx(AttrTimeout, TDuration, TDuration::Zero() )\ xxx(MaxOutOfOrderCompactionMapLoadRequestsInQueue, ui32, 5 )\ xxx(MaxBackpressureErrorsBeforeSuicide, ui32, 1000 )\ + \ + xxx(GenerateBlobIdsReleaseCollectBarrierTimeout, \ + TDuration, \ + TDuration::Seconds(10) )\ // FILESTORE_STORAGE_CONFIG #define FILESTORE_DECLARE_CONFIG(name, type, value) \ diff --git a/cloud/filestore/libs/storage/core/config.h b/cloud/filestore/libs/storage/core/config.h index 4b262a05b7d..015ca29137f 100644 --- a/cloud/filestore/libs/storage/core/config.h +++ b/cloud/filestore/libs/storage/core/config.h @@ -66,7 +66,13 @@ class TStorageConfig ui32 GetFlushThreshold() const; ui32 GetCleanupThreshold() const; + ui32 GetCleanupThresholdAverage() const; + bool GetNewCleanupEnabled() const; ui32 GetCompactionThreshold() const; + ui32 GetGarbageCompactionThreshold() const; + ui32 GetCompactionThresholdAverage() const; + ui32 GetGarbageCompactionThresholdAverage() const; + bool GetNewCompactionEnabled() const; ui32 GetCollectGarbageThreshold() const; ui32 GetFlushBytesThreshold() const; ui32 GetMaxDeleteGarbageBlobsPerTx() const; @@ -175,6 +181,7 @@ class TStorageConfig NCloud::NProto::EAuthorizationMode GetAuthorizationMode() const; bool GetTwoStageReadEnabled() const; + bool GetThreeStageWriteEnabled() const; TDuration GetEntryTimeout() const; TDuration GetNegativeEntryTimeout() const; TDuration GetAttrTimeout() const; @@ -185,6 +192,8 @@ class TStorageConfig ui32 GetMaxBackpressureErrorsBeforeSuicide() const; + TDuration GetGenerateBlobIdsReleaseCollectBarrierTimeout() const; + void Dump(IOutputStream& out) const; void DumpHtml(IOutputStream& out) const; void DumpOverridesHtml(IOutputStream& out) const; diff --git a/cloud/filestore/libs/storage/core/model_ut.cpp b/cloud/filestore/libs/storage/core/model_ut.cpp index 685a7a30228..3e26dc82337 100644 --- a/cloud/filestore/libs/storage/core/model_ut.cpp +++ b/cloud/filestore/libs/storage/core/model_ut.cpp @@ -193,10 +193,8 @@ Y_UNIT_TEST_SUITE(TModel) // In this case answer will be STORAGE_MEDIA_HYBRID, because we cannot // initialize proto value with 0 - // https://a.yandex-team.ru/arcadia/cloud/filestore/libs/storage/core/config.cpp?rev=r10218064#L139. // // In this case default value will be used. - // https://a.yandex-team.ru/arcadia/cloud/filestore/libs/storage/core/config.cpp?rev=r10218064#L80 DO_TEST( STORAGE_MEDIA_HDD, STORAGE_MEDIA_DEFAULT, diff --git a/cloud/filestore/libs/storage/core/request_info.h b/cloud/filestore/libs/storage/core/request_info.h index 6df414a608c..d45cbd5b58d 100644 --- a/cloud/filestore/libs/storage/core/request_info.h +++ b/cloud/filestore/libs/storage/core/request_info.h @@ -2,6 +2,7 @@ #include "public.h" +#include #include #include @@ -48,6 +49,15 @@ struct TRequestInfo , CallContext(std::move(callContext)) {} + void CancelRequest(const NActors::TActorContext& ctx) + { + if (!CancelRoutine) { + ReportCancelRoutineIsNotSet(); + return; + }; + CancelRoutine(ctx, *this); + } + void AddExecCycles(ui64 cycles) { AtomicAdd(ExecCycles, cycles); @@ -121,7 +131,7 @@ TRequestInfoPtr CreateRequestInfo( TRequestInfo& requestInfo) { auto response = std::make_unique( - MakeError(E_REJECTED, "tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); NCloud::Reply(ctx, requestInfo, std::move(response)); }; diff --git a/cloud/filestore/libs/storage/init/actorsystem.cpp b/cloud/filestore/libs/storage/init/actorsystem.cpp index e0afee081e2..f1a5468252a 100644 --- a/cloud/filestore/libs/storage/init/actorsystem.cpp +++ b/cloud/filestore/libs/storage/init/actorsystem.cpp @@ -144,6 +144,7 @@ class TStorageServicesInitializer final // auto storageUserStats = NUserStats::CreateStorageUserStats( + TFileStoreComponents::USER_STATS, "filestore", "FileStore", {Args.UserCounters}); diff --git a/cloud/filestore/libs/storage/init/ya.make b/cloud/filestore/libs/storage/init/ya.make index 32f2644ed50..9d8ff892b6c 100644 --- a/cloud/filestore/libs/storage/init/ya.make +++ b/cloud/filestore/libs/storage/init/ya.make @@ -18,6 +18,7 @@ PEERDIR( cloud/storage/core/libs/hive_proxy cloud/storage/core/libs/kikimr cloud/storage/core/libs/user_stats + cloud/storage/core/libs/version_ydb contrib/ydb/library/actors/core diff --git a/cloud/filestore/libs/storage/service/service_actor.h b/cloud/filestore/libs/storage/service/service_actor.h index cafa3b69286..844753e73e1 100644 --- a/cloud/filestore/libs/storage/service/service_actor.h +++ b/cloud/filestore/libs/storage/service/service_actor.h @@ -150,6 +150,10 @@ class TStorageServiceActor final TRequestInfoPtr requestInfo, TString input); + NActors::IActorPtr CreateReassignTabletActionActor( + TRequestInfoPtr requestInfo, + TString input); + private: void RenderSessions(IOutputStream& out); void RenderLocalFileStores(IOutputStream& out); diff --git a/cloud/filestore/libs/storage/service/service_actor_actions.cpp b/cloud/filestore/libs/storage/service/service_actor_actions.cpp index 3786aa6bdc7..61207b99505 100644 --- a/cloud/filestore/libs/storage/service/service_actor_actions.cpp +++ b/cloud/filestore/libs/storage/service/service_actor_actions.cpp @@ -48,6 +48,10 @@ void TStorageServiceActor::HandleExecuteAction( "describesessions", &TStorageServiceActor::CreateDescribeSessionsActionActor }, + { + "reassigntablet", + &TStorageServiceActor::CreateReassignTabletActionActor + }, }; auto it = actions.find(action); diff --git a/cloud/filestore/libs/storage/service/service_actor_actions_reassign_tablet.cpp b/cloud/filestore/libs/storage/service/service_actor_actions_reassign_tablet.cpp new file mode 100644 index 00000000000..4725e6b4237 --- /dev/null +++ b/cloud/filestore/libs/storage/service/service_actor_actions_reassign_tablet.cpp @@ -0,0 +1,119 @@ +#include "service_actor.h" + +#include + +#include +#include + +#include + +namespace NCloud::NFileStore::NStorage { + +using namespace NActors; +using namespace NCloud::NStorage; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +struct TReassignTabletActionActor final + : public TActorBootstrapped +{ + TRequestInfoPtr RequestInfo; + TString Input; + + TReassignTabletActionActor(TRequestInfoPtr requestInfo, TString input) + : RequestInfo(std::move(requestInfo)) + , Input(std::move(input)) + {} + + void Bootstrap(const TActorContext& ctx) + { + NProtoPrivate::TReassignTabletRequest request; + if (!google::protobuf::util::JsonStringToMessage(Input, &request).ok()) { + ReplyWithError(ctx, MakeError(E_ARGUMENT, "Failed to parse input")); + return; + } + ReassignTablet( + ctx, + request.GetTabletId(), + {request.GetChannels().begin(), request.GetChannels().end()}); + Become(&TThis::StateWork); + } + + STFUNC(StateWork) + { + switch (ev->GetTypeRewrite()) { + HFunc(TEvHiveProxy::TEvReassignTabletResponse, HandleReassignResponse); + + default: + HandleUnexpectedEvent(ev, TFileStoreComponents::SERVICE); + break; + } + } + + void ReassignTablet( + const TActorContext& ctx, + ui64 tabletId, + TVector channels) const + { + auto request = std::make_unique( + tabletId, + std::move(channels)); + NCloud::Send( + ctx, + MakeHiveProxyServiceId(), + std::move(request), + RequestInfo->Cookie); + } + + void HandleReassignResponse( + TEvHiveProxy::TEvReassignTabletResponse::TPtr& ev, + const TActorContext& ctx) + { + if (const auto& error = ev->Get()->GetError(); FAILED(error.GetCode())) { + ReplyWithError(ctx, error); + return; + } + ReplyWithSuccess(ctx); + } + + void ReplyWithError(const TActorContext& ctx, const NProto::TError& error) + { + auto response = + std::make_unique(error); + ReplyAndDie(ctx, std::move(response)); + } + + void ReplyWithSuccess(const TActorContext& ctx) + { + auto response = std::make_unique(); + google::protobuf::util::MessageToJsonString( + NProtoPrivate::TReassignTabletResponse(), + response->Record.MutableOutput()); + ReplyAndDie(ctx, std::move(response)); + } + + void ReplyAndDie( + const TActorContext& ctx, + std::unique_ptr response) + { + NCloud::Reply(ctx, *RequestInfo, std::move(response)); + Die(ctx); + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +IActorPtr TStorageServiceActor::CreateReassignTabletActionActor( + TRequestInfoPtr requestInfo, + TString input) +{ + return std::make_unique( + std::move(requestInfo), + std::move(input)); +} + +} // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/service/service_ut.cpp b/cloud/filestore/libs/storage/service/service_ut.cpp index b4e2dd71815..1b8ba09cbb3 100644 --- a/cloud/filestore/libs/storage/service/service_ut.cpp +++ b/cloud/filestore/libs/storage/service/service_ut.cpp @@ -6,11 +6,14 @@ #include #include #include +#include #include #include #include +#include + namespace NCloud::NFileStore::NStorage { using namespace NActors; diff --git a/cloud/filestore/libs/storage/service/ya.make b/cloud/filestore/libs/storage/service/ya.make index fbca807c47e..82b4abda035 100644 --- a/cloud/filestore/libs/storage/service/ya.make +++ b/cloud/filestore/libs/storage/service/ya.make @@ -8,6 +8,7 @@ SRCS( service_actor_actions_describe_sessions.cpp service_actor_actions_drain_tablets.cpp service_actor_actions_get_storage_config_fields.cpp + service_actor_actions_reassign_tablet.cpp service_actor_actions.cpp service_actor_alterfs.cpp service_actor_complete.cpp diff --git a/cloud/filestore/libs/storage/tablet/actors/tablet_adddata.cpp b/cloud/filestore/libs/storage/tablet/actors/tablet_adddata.cpp new file mode 100644 index 00000000000..d7a08cda887 --- /dev/null +++ b/cloud/filestore/libs/storage/tablet/actors/tablet_adddata.cpp @@ -0,0 +1,140 @@ +#include "tablet_adddata.h" + +#include +#include +#include +#include +#include + +#include + +namespace NCloud::NFileStore::NStorage { + +using namespace NActors; + +//////////////////////////////////////////////////////////////////////////////// + +LWTRACE_USING(FILESTORE_STORAGE_PROVIDER); + +//////////////////////////////////////////////////////////////////////////////// + +TAddDataActor::TAddDataActor( + ITraceSerializerPtr traceSerializer, + TString logTag, + TActorId tablet, + TRequestInfoPtr requestInfo, + ui64 commitId, + TVector blobs, + TWriteRange writeRange) + : TraceSerializer(std::move(traceSerializer)) + , LogTag(std::move(logTag)) + , Tablet(tablet) + , RequestInfo(std::move(requestInfo)) + , CommitId(commitId) + , Blobs(std::move(blobs)) + , WriteRange(writeRange) +{} + +void TAddDataActor::Bootstrap(const TActorContext& ctx) +{ + FILESTORE_TRACK( + RequestReceived_TabletWorker, + RequestInfo->CallContext, + "AddData"); + + AddBlob(ctx); + Become(&TThis::StateWork); +} + +void TAddDataActor::AddBlob(const TActorContext& ctx) +{ + auto request = std::make_unique( + RequestInfo->CallContext); + request->Mode = EAddBlobMode::Write; + request->WriteRanges.push_back(WriteRange); + + for (const auto& blob: Blobs) { + request->MergedBlobs.emplace_back( + blob.BlobId, + blob.Block, + blob.BlocksCount); + } + + NCloud::Send(ctx, Tablet, std::move(request)); +} + +void TAddDataActor::HandleAddBlobResponse( + const TEvIndexTabletPrivate::TEvAddBlobResponse::TPtr& ev, + const TActorContext& ctx) +{ + const auto* msg = ev->Get(); + ReplyAndDie(ctx, msg->GetError()); +} + +void TAddDataActor::HandlePoisonPill( + const TEvents::TEvPoisonPill::TPtr& ev, + const TActorContext& ctx) +{ + Y_UNUSED(ev); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); +} + +void TAddDataActor::ReplyAndDie( + const TActorContext& ctx, + const NProto::TError& error) +{ + // notify tablet + NCloud::Send( + ctx, + // We try to release commit barrier twice: once for the lock + // acquired by the GenerateBlob request and once for the lock + // acquired by the AddData request. Though, the first lock is + // scheduled to be released, it is better to release it as early + // as possible. + Tablet, + std::make_unique( + CommitId, + 2)); + + FILESTORE_TRACK( + ResponseSent_TabletWorker, + RequestInfo->CallContext, + "AddData"); + + if (RequestInfo->Sender != Tablet) { + auto response = + std::make_unique(error); + LOG_DEBUG( + ctx, + TFileStoreComponents::TABLET_WORKER, + "%s AddData: #%lu completed (%s)", + LogTag.c_str(), + RequestInfo->CallContext->RequestId, + FormatError(response->Record.GetError()).c_str()); + + BuildTraceInfo( + TraceSerializer, + RequestInfo->CallContext, + response->Record); + BuildThrottlerInfo(*RequestInfo->CallContext, response->Record); + + NCloud::Reply(ctx, *RequestInfo, std::move(response)); + } + + Die(ctx); +} + +STFUNC(TAddDataActor::StateWork) +{ + switch (ev->GetTypeRewrite()) { + HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); + + HFunc(TEvIndexTabletPrivate::TEvAddBlobResponse, HandleAddBlobResponse); + + default: + HandleUnexpectedEvent(ev, TFileStoreComponents::TABLET_WORKER); + break; + } +} + +} // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/actors/tablet_adddata.h b/cloud/filestore/libs/storage/tablet/actors/tablet_adddata.h new file mode 100644 index 00000000000..35645e5a8bc --- /dev/null +++ b/cloud/filestore/libs/storage/tablet/actors/tablet_adddata.h @@ -0,0 +1,59 @@ +#include +#include +#include +#include + +#include +#include + +#include + +namespace NCloud::NFileStore::NStorage { + +using namespace NActors; +using namespace NKikimr; + +//////////////////////////////////////////////////////////////////////////////// + +class TAddDataActor final: public TActorBootstrapped +{ +private: + const ITraceSerializerPtr TraceSerializer; + + const TString LogTag; + const TActorId Tablet; + const TRequestInfoPtr RequestInfo; + + const ui64 CommitId; + /*const*/ TVector Blobs; + const TWriteRange WriteRange; + +public: + TAddDataActor( + ITraceSerializerPtr traceSerializer, + TString logTag, + TActorId tablet, + TRequestInfoPtr requestInfo, + ui64 commitId, + TVector blobs, + TWriteRange writeRange); + + void Bootstrap(const TActorContext& ctx); + +private: + STFUNC(StateWork); + + void AddBlob(const TActorContext& ctx); + void HandleAddBlobResponse( + const TEvIndexTabletPrivate::TEvAddBlobResponse::TPtr& ev, + const TActorContext& ctx); + + void HandlePoisonPill( + const TEvents::TEvPoisonPill::TPtr& ev, + const TActorContext& ctx); + + void + ReplyAndDie(const TActorContext& ctx, const NProto::TError& error = {}); +}; + +} // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/actors/tablet_writedata.cpp b/cloud/filestore/libs/storage/tablet/actors/tablet_writedata.cpp new file mode 100644 index 00000000000..1afaa1756bf --- /dev/null +++ b/cloud/filestore/libs/storage/tablet/actors/tablet_writedata.cpp @@ -0,0 +1,167 @@ +#include "tablet_writedata.h" + +#include +#include +#include +#include + +#include + +namespace NCloud::NFileStore::NStorage { + +using namespace NActors; + +//////////////////////////////////////////////////////////////////////////////// + +LWTRACE_USING(FILESTORE_STORAGE_PROVIDER); + +//////////////////////////////////////////////////////////////////////////////// + +TWriteDataActor::TWriteDataActor( + ITraceSerializerPtr traceSerializer, + TString logTag, + TActorId tablet, + TRequestInfoPtr requestInfo, + ui64 commitId, + TVector blobs, + TWriteRange writeRange) + : TraceSerializer(std::move(traceSerializer)) + , LogTag(std::move(logTag)) + , Tablet(tablet) + , RequestInfo(std::move(requestInfo)) + , CommitId(commitId) + , Blobs(std::move(blobs)) + , WriteRange(writeRange) +{ + for (const auto& blob: Blobs) { + BlobsSize += blob.BlobContent.Size(); + } +} + +void TWriteDataActor::Bootstrap(const TActorContext& ctx) +{ + FILESTORE_TRACK( + RequestReceived_TabletWorker, + RequestInfo->CallContext, + "WriteData"); + + WriteBlob(ctx); + Become(&TThis::StateWork); +} + +void TWriteDataActor::WriteBlob(const TActorContext& ctx) +{ + auto request = std::make_unique( + RequestInfo->CallContext + ); + + for (auto& blob: Blobs) { + request->Blobs.emplace_back(blob.BlobId, std::move(blob.BlobContent)); + } + + NCloud::Send(ctx, Tablet, std::move(request)); +} + +void TWriteDataActor::HandleWriteBlobResponse( + const TEvIndexTabletPrivate::TEvWriteBlobResponse::TPtr& ev, + const TActorContext& ctx) +{ + const auto* msg = ev->Get(); + + if (FAILED(msg->GetStatus())) { + ReplyAndDie(ctx, msg->GetError()); + return; + } + + AddBlob(ctx); +} + +void TWriteDataActor::AddBlob(const TActorContext& ctx) +{ + auto request = std::make_unique( + RequestInfo->CallContext + ); + request->Mode = EAddBlobMode::Write; + request->WriteRanges.push_back(WriteRange); + + for (const auto& blob: Blobs) { + request->MergedBlobs.emplace_back( + blob.BlobId, + blob.Block, + blob.BlocksCount); + } + + NCloud::Send(ctx, Tablet, std::move(request)); +} + +void TWriteDataActor::HandleAddBlobResponse( + const TEvIndexTabletPrivate::TEvAddBlobResponse::TPtr& ev, + const TActorContext& ctx) +{ + const auto* msg = ev->Get(); + ReplyAndDie(ctx, msg->GetError()); +} + +void TWriteDataActor::HandlePoisonPill( + const TEvents::TEvPoisonPill::TPtr& ev, + const TActorContext& ctx) +{ + Y_UNUSED(ev); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); +} + +void TWriteDataActor::ReplyAndDie( + const TActorContext& ctx, + const NProto::TError& error) +{ + { + // notify tablet + using TCompletion = TEvIndexTabletPrivate::TEvWriteDataCompleted; + auto response = std::make_unique(error); + response->CommitId = CommitId; + response->Count = 1; + response->Size = BlobsSize; + response->Time = ctx.Now() - RequestInfo->StartedTs; + NCloud::Send(ctx, Tablet, std::move(response)); + } + + FILESTORE_TRACK( + ResponseSent_TabletWorker, + RequestInfo->CallContext, + "WriteData"); + + if (RequestInfo->Sender != Tablet) { + auto response = std::make_unique(error); + LOG_DEBUG(ctx, TFileStoreComponents::TABLET_WORKER, + "%s WriteData: #%lu completed (%s)", + LogTag.c_str(), + RequestInfo->CallContext->RequestId, + FormatError(response->Record.GetError()).c_str()); + + BuildTraceInfo( + TraceSerializer, + RequestInfo->CallContext, + response->Record); + BuildThrottlerInfo(*RequestInfo->CallContext, response->Record); + + NCloud::Reply(ctx, *RequestInfo, std::move(response)); + } + + Die(ctx); +} + +STFUNC(TWriteDataActor::StateWork) +{ + switch (ev->GetTypeRewrite()) { + HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); + + HFunc(TEvIndexTabletPrivate::TEvWriteBlobResponse, HandleWriteBlobResponse); + HFunc(TEvIndexTabletPrivate::TEvAddBlobResponse, HandleAddBlobResponse); + + default: + HandleUnexpectedEvent(ev, TFileStoreComponents::TABLET_WORKER); + break; + } +} + +} // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/actors/tablet_writedata.h b/cloud/filestore/libs/storage/tablet/actors/tablet_writedata.h new file mode 100644 index 00000000000..a6bb81ef43e --- /dev/null +++ b/cloud/filestore/libs/storage/tablet/actors/tablet_writedata.h @@ -0,0 +1,65 @@ +#include +#include +#include +#include + +#include +#include + +#include + +namespace NCloud::NFileStore::NStorage { + +using namespace NActors; +using namespace NKikimr; + +//////////////////////////////////////////////////////////////////////////////// + +class TWriteDataActor final: public TActorBootstrapped +{ +private: + const ITraceSerializerPtr TraceSerializer; + + const TString LogTag; + const TActorId Tablet; + const TRequestInfoPtr RequestInfo; + + const ui64 CommitId; + /*const*/ TVector Blobs; + const TWriteRange WriteRange; + ui32 BlobsSize = 0; + +public: + TWriteDataActor( + ITraceSerializerPtr traceSerializer, + TString logTag, + TActorId tablet, + TRequestInfoPtr requestInfo, + ui64 commitId, + TVector blobs, + TWriteRange writeRange); + + void Bootstrap(const TActorContext& ctx); + +private: + STFUNC(StateWork); + + void WriteBlob(const TActorContext& ctx); + void HandleWriteBlobResponse( + const TEvIndexTabletPrivate::TEvWriteBlobResponse::TPtr& ev, + const TActorContext& ctx); + + void AddBlob(const TActorContext& ctx); + void HandleAddBlobResponse( + const TEvIndexTabletPrivate::TEvAddBlobResponse::TPtr& ev, + const TActorContext& ctx); + + void HandlePoisonPill( + const TEvents::TEvPoisonPill::TPtr& ev, + const TActorContext& ctx); + + void + ReplyAndDie(const TActorContext& ctx, const NProto::TError& error = {}); +}; + +} // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/actors/ya.make b/cloud/filestore/libs/storage/tablet/actors/ya.make new file mode 100644 index 00000000000..389d926dc02 --- /dev/null +++ b/cloud/filestore/libs/storage/tablet/actors/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +SRCS( + tablet_writedata.cpp + tablet_adddata.cpp +) + +PEERDIR( + cloud/filestore/libs/storage/tablet/model +) + +END() diff --git a/cloud/filestore/libs/storage/tablet/model/channels.cpp b/cloud/filestore/libs/storage/tablet/model/channels.cpp index 4aed7db606b..6ef94d1301b 100644 --- a/cloud/filestore/libs/storage/tablet/model/channels.cpp +++ b/cloud/filestore/libs/storage/tablet/model/channels.cpp @@ -83,6 +83,10 @@ struct TChannels::TImpl TVector GetUnwritableChannels() const; TVector GetChannelsToMove(ui32 percentageThreshold) const; + TVector MakeChannelMonInfos() const; + + TChannelsStats CalculateChannelsStats() const; + ui32 Size() const; bool Empty() const; }; @@ -117,6 +121,11 @@ void TChannels::TImpl::RegisterChannelToMove(ui32 channel) AllChannels[channel].ToMove = true; } +TVector TChannels::TImpl::GetChannels(EChannelDataKind dataKind) const +{ + return ByDataKind[static_cast(dataKind)].GetChannels(); +} + TVector TChannels::TImpl::GetUnwritableChannels() const { TVector result; @@ -149,6 +158,36 @@ TVector TChannels::TImpl::GetChannelsToMove(ui32 percentageThreshold) cons return result; } +TVector +TChannels::TImpl::MakeChannelMonInfos() const +{ + TVector result; + + for (const auto& meta: AllChannels) { + result.push_back({ + meta.PoolKind, + TStringBuilder() << meta.DataKind, + meta.Writable, + meta.Writable, // TODO: SystemWritable + }); + } + + return result; +} + +TChannelsStats TChannels::TImpl::CalculateChannelsStats() const +{ + TChannelsStats stats; + + for (const auto& meta: AllChannels) { + stats.WritableChannelCount += meta.Writable; + stats.UnwritableChannelCount += !meta.Writable; + stats.ChannelsToMoveCount += meta.ToMove; + } + + return stats; +} + TMaybe TChannels::TImpl::SelectChannel(EChannelDataKind dataKind) { auto& byDataKind = ByDataKind[static_cast(dataKind)]; @@ -159,11 +198,6 @@ TMaybe TChannels::TImpl::SelectChannel(EChannelDataKind dataKind) return Nothing(); } -TVector TChannels::TImpl::GetChannels(EChannelDataKind dataKind) const -{ - return ByDataKind[static_cast(dataKind)].GetChannels(); -} - ui32 TChannels::TImpl::Size() const { return AllChannels.size(); @@ -202,6 +236,11 @@ void TChannels::RegisterChannelToMove(ui32 channel) GetImpl().RegisterChannelToMove(channel); } +TVector TChannels::GetChannels(EChannelDataKind dataKind) const +{ + return GetImpl().GetChannels(dataKind); +} + TVector TChannels::GetUnwritableChannels() const { return GetImpl().GetUnwritableChannels(); @@ -212,14 +251,19 @@ TVector TChannels::GetChannelsToMove(ui32 percentageThreshold) const return GetImpl().GetChannelsToMove(percentageThreshold); } -TMaybe TChannels::SelectChannel(EChannelDataKind dataKind) +TVector TChannels::MakeChannelMonInfos() const { - return GetImpl().SelectChannel(dataKind); + return GetImpl().MakeChannelMonInfos(); } -TVector TChannels::GetChannels(EChannelDataKind dataKind) const +TChannelsStats TChannels::CalculateChannelsStats() const { - return GetImpl().GetChannels(dataKind); + return GetImpl().CalculateChannelsStats(); +} + +TMaybe TChannels::SelectChannel(EChannelDataKind dataKind) +{ + return GetImpl().SelectChannel(dataKind); } ui32 TChannels::Size() const diff --git a/cloud/filestore/libs/storage/tablet/model/channels.h b/cloud/filestore/libs/storage/tablet/model/channels.h index 75bd6758621..b1db2c5ad5b 100644 --- a/cloud/filestore/libs/storage/tablet/model/channels.h +++ b/cloud/filestore/libs/storage/tablet/model/channels.h @@ -4,6 +4,8 @@ #include +#include + #include #include @@ -11,6 +13,15 @@ namespace NCloud::NFileStore::NStorage { //////////////////////////////////////////////////////////////////////////////// +struct TChannelsStats +{ + ui32 WritableChannelCount = 0; + ui32 UnwritableChannelCount = 0; + ui32 ChannelsToMoveCount = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + class TChannels { private: @@ -31,6 +42,10 @@ class TChannels TVector GetUnwritableChannels() const; TVector GetChannelsToMove(ui32 percentageThreshold) const; + TVector MakeChannelMonInfos() const; + + TChannelsStats CalculateChannelsStats() const; + ui32 Size() const; bool Empty() const; diff --git a/cloud/filestore/libs/storage/tablet/model/compaction_map.cpp b/cloud/filestore/libs/storage/tablet/model/compaction_map.cpp index e413d8ecf06..862d51f6420 100644 --- a/cloud/filestore/libs/storage/tablet/model/compaction_map.cpp +++ b/cloud/filestore/libs/storage/tablet/model/compaction_map.cpp @@ -449,10 +449,15 @@ TCompactionMapStats TCompactionMap::GetStats(ui32 topSize) const TCompactionMapStats stats = { .UsedRangesCount = Impl->UsedRangesCount, .AllocatedRangesCount = Impl->Groups.Size() * TCompactionMap::GroupSize, - .TopRangesByCleanupScore = GetTopRangesByCleanupScore(topSize), - .TopRangesByCompactionScore = GetTopRangesByCompactionScore(topSize), }; + if (!topSize) { + return stats; + } + + stats.TopRangesByCleanupScore = GetTopRangesByCleanupScore(topSize); + stats.TopRangesByCompactionScore = GetTopRangesByCompactionScore(topSize); + return stats; } diff --git a/cloud/filestore/libs/storage/tablet/model/garbage_queue.cpp b/cloud/filestore/libs/storage/tablet/model/garbage_queue.cpp index b489439553a..1c47512b29d 100644 --- a/cloud/filestore/libs/storage/tablet/model/garbage_queue.cpp +++ b/cloud/filestore/libs/storage/tablet/model/garbage_queue.cpp @@ -182,15 +182,19 @@ void TGarbageQueue::AcquireCollectBarrier(ui64 commitId) } } -void TGarbageQueue::ReleaseCollectBarrier(ui64 commitId) +bool TGarbageQueue::TryReleaseCollectBarrier(ui64 commitId) { { auto it = Impl->Barriers.find(commitId); - Y_ABORT_UNLESS(it != Impl->Barriers.end()); + if (it == Impl->Barriers.end()) { + return false; + } auto& barrier = const_cast(*it); - Y_ABORT_UNLESS(barrier.RefCount > 0); + if (barrier.RefCount == 0) { + return false; + } --barrier.RefCount; } @@ -208,6 +212,13 @@ void TGarbageQueue::ReleaseCollectBarrier(ui64 commitId) it = Impl->Barriers.erase(it); } + return true; +} + +bool TGarbageQueue::IsCollectBarrierAcquired(ui64 commitId) const +{ + auto it = Impl->Barriers.find(commitId); + return it != Impl->Barriers.end() && it->RefCount > 0; } ui64 TGarbageQueue::GetCollectCommitId() const diff --git a/cloud/filestore/libs/storage/tablet/model/garbage_queue.h b/cloud/filestore/libs/storage/tablet/model/garbage_queue.h index 493a2edf59b..bbdb12da8ab 100644 --- a/cloud/filestore/libs/storage/tablet/model/garbage_queue.h +++ b/cloud/filestore/libs/storage/tablet/model/garbage_queue.h @@ -46,7 +46,8 @@ class TGarbageQueue // void AcquireCollectBarrier(ui64 commitId); - void ReleaseCollectBarrier(ui64 commitId); + [[ nodiscard ]] bool TryReleaseCollectBarrier(ui64 commitId); + bool IsCollectBarrierAcquired(ui64 commitId) const; ui64 GetCollectCommitId() const; }; diff --git a/cloud/filestore/libs/storage/tablet/rebase_logic.cpp b/cloud/filestore/libs/storage/tablet/rebase_logic.cpp index b844aef7bb7..3288a5f4b84 100644 --- a/cloud/filestore/libs/storage/tablet/rebase_logic.cpp +++ b/cloud/filestore/libs/storage/tablet/rebase_logic.cpp @@ -4,11 +4,11 @@ namespace NCloud::NFileStore::NStorage { //////////////////////////////////////////////////////////////////////////////// -TRebaseResult RebaseMixedBlocks( +TRebaseResult RebaseBlocks( TVector& blocks, ui64 lastCommitId, - TFindCheckpoint findCheckpoint, - TFindBlock findBlock) + const TFindCheckpoint& findCheckpoint, + const TFindBlock& findBlock) { TRebaseResult result; diff --git a/cloud/filestore/libs/storage/tablet/rebase_logic.h b/cloud/filestore/libs/storage/tablet/rebase_logic.h index 184345579d8..3308c2cc890 100644 --- a/cloud/filestore/libs/storage/tablet/rebase_logic.h +++ b/cloud/filestore/libs/storage/tablet/rebase_logic.h @@ -29,10 +29,10 @@ using TFindBlock = std::function; //////////////////////////////////////////////////////////////////////////////// -TRebaseResult RebaseMixedBlocks( +TRebaseResult RebaseBlocks( TVector& blocks, ui64 lastCommitId, - TFindCheckpoint findCheckpoint, - TFindBlock findBlock); + const TFindCheckpoint& findCheckpoint, + const TFindBlock& findBlock); } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/rebase_logic_ut.cpp b/cloud/filestore/libs/storage/tablet/rebase_logic_ut.cpp index 5cb593cfd2d..da44b1a1ae1 100644 --- a/cloud/filestore/libs/storage/tablet/rebase_logic_ut.cpp +++ b/cloud/filestore/libs/storage/tablet/rebase_logic_ut.cpp @@ -8,9 +8,96 @@ namespace NCloud::NFileStore::NStorage { Y_UNIT_TEST_SUITE(TRebaseLogicTest) { - Y_UNIT_TEST(Test) + Y_UNIT_TEST(TestRebaseBlocks) { - // TODO + const ui64 defaultNodeId = 111; + const auto b = [=] (ui32 blockIndex, ui64 minCommitId, ui64 maxCommitId) + { + return TBlock(defaultNodeId, blockIndex, minCommitId, maxCommitId); + }; + TVector blocks = { + b(10, 5, 9), + b(11, 7, 20), + b(11, 20, InvalidCommitId), + b(12, 20, 31), + b(13, 31, InvalidCommitId), + b(14, 32, InvalidCommitId), + }; + + const ui64 c1 = 3; + const ui64 c2 = 6; + const ui64 c3 = 10; + const ui64 lastCommitId = 40; + + const auto findCheckpoint = [&] (ui64 nodeId, ui64 commitId) { + UNIT_ASSERT_VALUES_EQUAL(defaultNodeId, nodeId); + + if (commitId < c1) { + return c1; + } + + if (commitId < c2) { + return c2; + } + + if (commitId < c3) { + return c3; + } + + return InvalidCommitId; + }; + + const THashSet freshBlocks = {11, 14}; + const auto findBlock = [&] (ui64 nodeId, ui32 blockIndex) { + UNIT_ASSERT_VALUES_EQUAL(defaultNodeId, nodeId); + + return freshBlocks.contains(blockIndex); + }; + + const auto rebaseResult = RebaseBlocks( + blocks, + lastCommitId, + findCheckpoint, + findBlock); + + UNIT_ASSERT_VALUES_EQUAL(5, rebaseResult.LiveBlocks); + UNIT_ASSERT_VALUES_EQUAL(1, rebaseResult.GarbageBlocks); + UNIT_ASSERT_VALUES_EQUAL(2, rebaseResult.CheckpointBlocks); + UNIT_ASSERT_VALUES_EQUAL(2, rebaseResult.UsedCheckpoints.size()); + UNIT_ASSERT(rebaseResult.UsedCheckpoints.contains(c2)); + UNIT_ASSERT(rebaseResult.UsedCheckpoints.contains(c3)); + + UNIT_ASSERT_VALUES_EQUAL(6, blocks.size()); + + UNIT_ASSERT_VALUES_EQUAL(defaultNodeId, blocks[0].NodeId); + UNIT_ASSERT_VALUES_EQUAL(10, blocks[0].BlockIndex); + UNIT_ASSERT_VALUES_EQUAL(c2, blocks[0].MinCommitId); + UNIT_ASSERT_VALUES_EQUAL(c3, blocks[0].MaxCommitId); + + UNIT_ASSERT_VALUES_EQUAL(defaultNodeId, blocks[1].NodeId); + UNIT_ASSERT_VALUES_EQUAL(11, blocks[1].BlockIndex); + UNIT_ASSERT_VALUES_EQUAL(7, blocks[1].MinCommitId); + UNIT_ASSERT_VALUES_EQUAL(20, blocks[1].MaxCommitId); + + UNIT_ASSERT_VALUES_EQUAL(defaultNodeId, blocks[2].NodeId); + UNIT_ASSERT_VALUES_EQUAL(11, blocks[2].BlockIndex); + UNIT_ASSERT_VALUES_EQUAL(20, blocks[2].MinCommitId); + UNIT_ASSERT_VALUES_EQUAL(InvalidCommitId, blocks[2].MaxCommitId); + + UNIT_ASSERT_VALUES_EQUAL(defaultNodeId, blocks[3].NodeId); + UNIT_ASSERT_VALUES_EQUAL(12, blocks[3].BlockIndex); + UNIT_ASSERT_VALUES_EQUAL(0, blocks[3].MinCommitId); + UNIT_ASSERT_VALUES_EQUAL(0, blocks[3].MaxCommitId); + + UNIT_ASSERT_VALUES_EQUAL(defaultNodeId, blocks[4].NodeId); + UNIT_ASSERT_VALUES_EQUAL(13, blocks[4].BlockIndex); + UNIT_ASSERT_VALUES_EQUAL(lastCommitId, blocks[4].MinCommitId); + UNIT_ASSERT_VALUES_EQUAL(InvalidCommitId, blocks[4].MaxCommitId); + + UNIT_ASSERT_VALUES_EQUAL(defaultNodeId, blocks[5].NodeId); + UNIT_ASSERT_VALUES_EQUAL(14, blocks[5].BlockIndex); + UNIT_ASSERT_VALUES_EQUAL(32, blocks[5].MinCommitId); + UNIT_ASSERT_VALUES_EQUAL(InvalidCommitId, blocks[5].MaxCommitId); } } diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor.cpp index da846555e90..eebb95eb427 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.cpp @@ -181,6 +181,10 @@ void TIndexTabletActor::ReassignDataChannelsIfNeeded( sb.c_str()); } + Metrics.ReassignCount.fetch_add( + channels.size(), + std::memory_order_relaxed); + NCloud::Send( ctx, MakeHiveProxyServiceId(), @@ -271,7 +275,7 @@ void TIndexTabletActor::TerminateTransactions(const TActorContext& ctx) TRequestInfo* requestInfo = ActiveTransactions.PopFront(); TABLET_VERIFY(requestInfo->RefCount() >= 1); - requestInfo->CancelRoutine(ctx, *requestInfo); + requestInfo->CancelRequest(ctx); requestInfo->UnRef(); } } @@ -315,6 +319,67 @@ void TIndexTabletActor::ResetThrottlingPolicy() //////////////////////////////////////////////////////////////////////////////// +template +NProto::TError TIndexTabletActor::ValidateWriteRequest( + const TActorContext& ctx, + const TRequest& request, + const TByteRange& range) +{ + if (auto error = ValidateRange(range); HasError(error)) { + return error; + } + + auto* handle = FindHandle(request.GetHandle()); + if (!handle || handle->GetSessionId() != GetSessionId(request)) { + return ErrorInvalidHandle(request.GetHandle()); + } + + if (!IsWriteAllowed(BuildBackpressureThresholds())) { + if (CompactionStateLoadStatus.Finished && + ++BackpressureErrorCount >= + Config->GetMaxBackpressureErrorsBeforeSuicide()) + { + LOG_WARN( + ctx, + TFileStoreComponents::TABLET_WORKER, + "%s Suiciding after %u backpressure errors", + LogTag.c_str(), + BackpressureErrorCount); + + Suicide(ctx); + } + + return MakeError(E_REJECTED, "rejected due to backpressure"); + } + + return NProto::TError{}; +} + +template NProto::TError +TIndexTabletActor::ValidateWriteRequest( + const TActorContext& ctx, + const NProto::TWriteDataRequest& request, + const TByteRange& range); + +template NProto::TError +TIndexTabletActor::ValidateWriteRequest( + const TActorContext& ctx, + const NProtoPrivate::TAddDataRequest& request, + const TByteRange& range); + +//////////////////////////////////////////////////////////////////////////////// + +NProto::TError TIndexTabletActor::IsDataOperationAllowed() const +{ + if (!CompactionStateLoadStatus.Finished) { + return MakeError(E_REJECTED, "compaction state not loaded yet"); + } + + return {}; +} + +//////////////////////////////////////////////////////////////////////////////// + void TIndexTabletActor::HandleWakeup( const TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx) @@ -492,6 +557,7 @@ STFUNC(TIndexTabletActor::StateBoot) IgnoreFunc(TEvLocal::TEvTabletMetrics); IgnoreFunc(TEvIndexTabletPrivate::TEvUpdateCounters); IgnoreFunc(TEvIndexTabletPrivate::TEvUpdateLeakyBucketCounters); + IgnoreFunc(TEvIndexTabletPrivate::TEvReleaseCollectBarrier); IgnoreFunc(TEvHiveProxy::TEvReassignTabletResponse); @@ -515,6 +581,7 @@ STFUNC(TIndexTabletActor::StateInit) HFunc(TEvFileStore::TEvUpdateConfig, HandleUpdateConfig); HFunc(TEvIndexTabletPrivate::TEvUpdateCounters, HandleUpdateCounters); HFunc(TEvIndexTabletPrivate::TEvUpdateLeakyBucketCounters, HandleUpdateLeakyBucketCounters); + HFunc(TEvIndexTabletPrivate::TEvReleaseCollectBarrier, HandleReleaseCollectBarrier); FILESTORE_HANDLE_REQUEST(WaitReady, TEvIndexTablet) @@ -542,6 +609,8 @@ STFUNC(TIndexTabletActor::StateWork) HFunc(TEvIndexTabletPrivate::TEvUpdateCounters, HandleUpdateCounters); HFunc(TEvIndexTabletPrivate::TEvUpdateLeakyBucketCounters, HandleUpdateLeakyBucketCounters); + HFunc(TEvIndexTabletPrivate::TEvReleaseCollectBarrier, HandleReleaseCollectBarrier); + HFunc(TEvents::TEvWakeup, HandleWakeup); HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); @@ -579,6 +648,8 @@ STFUNC(TIndexTabletActor::StateZombie) IgnoreFunc(TEvIndexTabletPrivate::TEvUpdateCounters); IgnoreFunc(TEvIndexTabletPrivate::TEvUpdateLeakyBucketCounters); + IgnoreFunc(TEvIndexTabletPrivate::TEvReleaseCollectBarrier); + IgnoreFunc(TEvIndexTabletPrivate::TEvReadDataCompleted); IgnoreFunc(TEvIndexTabletPrivate::TEvWriteDataCompleted); @@ -604,6 +675,7 @@ STFUNC(TIndexTabletActor::StateBroken) switch (ev->GetTypeRewrite()) { IgnoreFunc(TEvIndexTabletPrivate::TEvUpdateCounters); IgnoreFunc(TEvIndexTabletPrivate::TEvUpdateLeakyBucketCounters); + IgnoreFunc(TEvIndexTabletPrivate::TEvReleaseCollectBarrier); HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); HFunc(TEvTablet::TEvTabletDead, HandleTabletDead); @@ -629,7 +701,7 @@ void TIndexTabletActor::RebootTabletOnCommitOverflow( const TActorContext& ctx, const TString& request) { - LOG_ERROR(ctx, TFileStoreActivities::TABLET, + LOG_ERROR(ctx, TFileStoreComponents::TABLET, "%s CommitId overflow in %s. Restarting", LogTag.c_str(), request.c_str()); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.h b/cloud/filestore/libs/storage/tablet/tablet_actor.h index e0b52ee95f2..5e8cf1e48b7 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.h +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.h @@ -89,6 +89,11 @@ class TIndexTabletActor final std::atomic AllocatedCompactionRangesCount{0}; std::atomic UsedCompactionRangesCount{0}; + std::atomic ReassignCount{0}; + std::atomic WritableChannelCount{0}; + std::atomic UnwritableChannelCount{0}; + std::atomic ChannelsToMoveCount{0}; + // Data stats std::atomic FreshBytesCount{0}; std::atomic MixedBytesCount{0}; @@ -150,7 +155,8 @@ class TIndexTabletActor final const NProto::TFileSystemStats& stats, const NProto::TFileStorePerformanceProfile& performanceProfile, const TCompactionMapStats& compactionStats, - const TSessionsStats& sessionsStats); + const TSessionsStats& sessionsStats, + const TChannelsStats& channelsStats); } Metrics; const IProfileLogPtr ProfileLog; @@ -248,7 +254,7 @@ class TIndexTabletActor final TRequestInfo& requestInfo) { auto response = std::make_unique( - MakeError(E_REJECTED, "tablet is dead")); + MakeError(E_REJECTED, "tablet is shutting down")); NCloud::Reply(ctx, requestInfo, std::move(response)); }; @@ -281,7 +287,7 @@ class TIndexTabletActor final TRequestInfo& requestInfo) { auto response = std::make_unique( - MakeError(E_REJECTED, "request cancelled")); + MakeError(E_REJECTED, "tablet is shutting down")); NCloud::Reply(ctx, requestInfo, std::move(response)); }; @@ -347,6 +353,14 @@ class TIndexTabletActor final const typename TMethod::TRequest::TPtr& ev, const NActors::TActorContext& ctx); + template + NProto::TError ValidateWriteRequest( + const NActors::TActorContext& ctx, + const TRequest& request, + const TByteRange& range); + + NProto::TError IsDataOperationAllowed() const; + void HandleWakeup( const NActors::TEvents::TEvWakeup::TPtr& ev, const NActors::TActorContext& ctx); @@ -391,6 +405,10 @@ class TIndexTabletActor final const TEvIndexTabletPrivate::TEvUpdateLeakyBucketCounters::TPtr& ev, const NActors::TActorContext& ctx); + void HandleReleaseCollectBarrier( + const TEvIndexTabletPrivate::TEvReleaseCollectBarrier::TPtr& ev, + const NActors::TActorContext& ctx); + void HandleReadDataCompleted( const TEvIndexTabletPrivate::TEvReadDataCompleted::TPtr& ev, const NActors::TActorContext& ctx); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_adddata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_adddata.cpp new file mode 100644 index 00000000000..e6a5b2c5dce --- /dev/null +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_adddata.cpp @@ -0,0 +1,372 @@ +#include "tablet_actor.h" + +#include +#include + +#include + +#include + +#include +#include +#include + +namespace NCloud::NFileStore::NStorage { + +using namespace NActors; + +using namespace NKikimr; +using namespace NKikimr::NTabletFlatExecutor; + + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +/** + * @param range Aligned byte range + * @returns The vector of sizes of the blobs that the data should be split into. + */ +TVector SplitData(ui32 blockSize, TByteRange range) +{ + TVector blobSizes; + blobSizes.reserve(range.AlignedBlockCount() / BlockGroupSize + 1); + + SplitRange( + range.FirstAlignedBlock(), + range.AlignedBlockCount(), + BlockGroupSize, + [&](ui32 /*blockOffset*/, ui64 blocksCount) + { blobSizes.push_back(blocksCount * blockSize); }); + + return blobSizes; +} + +} // namespace + +class TWriteDataActor; + +//////////////////////////////////////////////////////////////////////////////// + +bool TIndexTabletActor::PrepareTx_AddData( + const TActorContext& ctx, + TTransactionContext& tx, + TTxIndexTablet::TAddData& args) +{ + auto* session = + FindSession(args.ClientId, args.SessionId, args.SessionSeqNo); + if (!session) { + args.Error = ErrorInvalidSession( + args.ClientId, + args.SessionId, + args.SessionSeqNo); + return true; + } + + auto* handle = FindHandle(args.Handle); + if (!handle || handle->Session != session) { + args.Error = ErrorInvalidHandle(args.Handle); + return true; + } + + if (!HasFlag(handle->GetFlags(), NProto::TCreateHandleRequest::E_WRITE)) { + args.Error = ErrorInvalidHandle(args.Handle); + return true; + } + + args.NodeId = handle->GetNodeId(); + ui64 commitId = GetCurrentCommitId(); + + LOG_TRACE( + ctx, + TFileStoreComponents::TABLET, + "%s AddData tx %lu @%lu %s", + LogTag.c_str(), + args.CommitId, + args.NodeId, + args.ByteRange.Describe().c_str()); + + TIndexTabletDatabase db(tx.DB); + + if (!ReadNode(db, args.NodeId, commitId, args.Node)) { + return false; + } + + // TODO: access check + TABLET_VERIFY(args.Node); + if (!HasSpaceLeft(args.Node->Attrs, args.ByteRange.End())) { + args.Error = ErrorNoSpaceLeft(); + return true; + } + + return true; +} + +void TIndexTabletActor::ExecuteTx_AddData( + const TActorContext& ctx, + TTransactionContext& tx, + TTxIndexTablet::TAddData& args) +{ + Y_UNUSED(ctx, tx, args); +} + +void TIndexTabletActor::CompleteTx_AddData( + const TActorContext& ctx, + TTxIndexTablet::TAddData& args) +{ + RemoveTransaction(*args.RequestInfo); + + auto reply = [&](const TActorContext& ctx, TTxIndexTablet::TAddData& args) + { + TABLET_VERIFY(TryReleaseCollectBarrier(args.CommitId)); + TryReleaseCollectBarrier(args.CommitId); + + auto response = + std::make_unique(args.Error); + CompleteResponse( + response->Record, + args.RequestInfo->CallContext, + ctx); + + NCloud::Reply(ctx, *args.RequestInfo, std::move(response)); + }; + + if (HasError(args.Error)) { + reply(ctx, args); + return; + } + + TVector blobs; + SplitRange( + args.ByteRange.FirstAlignedBlock(), + args.ByteRange.AlignedBlockCount(), + BlockGroupSize, + [&](ui32 blockOffset, ui32 blocksCount) + { + TBlock block{ + args.NodeId, + IntegerCast( + args.ByteRange.FirstAlignedBlock() + blockOffset), + // correct CommitId will be assigned later in AddBlobs + InvalidCommitId, + InvalidCommitId}; + blobs.emplace_back( + TPartialBlobId(), // need to generate BlobId later + block, + blocksCount, + /* data buffer */ ""); + }); + + if (blobs.empty() || blobs.size() != args.BlobIds.size()) { + args.Error = MakeError( + E_ARGUMENT, + TStringBuilder() << "blobs count mismatch: expected" << blobs.size() + << " got " << args.BlobIds.size()); + reply(ctx, args); + return; + } + + for (size_t i = 0; i < blobs.size(); ++i) { + auto& targetBlob = blobs[i]; + auto& srcBlob = args.BlobIds[i]; + targetBlob.BlobId = TPartialBlobId( + srcBlob.Generation(), + srcBlob.Step(), + srcBlob.Channel(), + srcBlob.BlobSize(), + srcBlob.Cookie(), + srcBlob.PartId()); + } + auto actor = std::make_unique( + TraceSerializer, + LogTag, + ctx.SelfID, + args.RequestInfo, + args.CommitId, + std::move(blobs), + TWriteRange{args.NodeId, args.ByteRange.End()}); + + auto actorId = NCloud::Register(ctx, std::move(actor)); + WorkerActors.insert(actorId); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TIndexTabletActor::HandleGenerateBlobIds( + const TEvIndexTablet::TEvGenerateBlobIdsRequest::TPtr& ev, + const TActorContext& ctx) +{ + auto* msg = ev->Get(); + + LOG_DEBUG( + ctx, + TFileStoreComponents::TABLET, + "%s %s: %s", + LogTag.c_str(), + "GenerateBlobIds", + DumpMessage(msg->Record).c_str()); + + // It is up to the client to provide an aligned range, but we still verify + // it and reject the request if it is not aligned. + const ui32 blockSize = GetBlockSize(); + if (msg->Record.GetLength() % blockSize != 0 || + msg->Record.GetOffset() % blockSize != 0) + { + auto response = + std::make_unique( + MakeError(E_ARGUMENT, "unaligned range")); + NCloud::Reply(ctx, *ev, std::move(response)); + return; + } + + ui64 commitId = GenerateCommitId(); + if (commitId == InvalidCommitId) { + return RebootTabletOnCommitOverflow(ctx, "GenerateBlobIds"); + } + + // We schedule this event for the case if the client does not call AddData. + // Thus we ensure that the collect barrier will be released eventually. + ctx.Schedule( + Config->GetGenerateBlobIdsReleaseCollectBarrierTimeout(), + new TEvIndexTabletPrivate::TEvReleaseCollectBarrier(commitId, 1)); + AcquireCollectBarrier(commitId); + + TByteRange range( + msg->Record.GetOffset(), + msg->Record.GetLength(), + blockSize); + + auto response = + std::make_unique(); + ui64 offset = range.Offset; + for (auto [blobIndex, length]: Enumerate(SplitData(blockSize, range))) { + TPartialBlobId partialBlobId; + // TODO(debnatkh): better selection of channel + + const auto ok = + GenerateBlobId(commitId, length, blobIndex, &partialBlobId); + if (!ok) { + ReassignDataChannelsIfNeeded(ctx); + + auto response = + std::make_unique( + MakeError(E_FS_OUT_OF_SPACE, "failed to generate blobId")); + + NCloud::Reply(ctx, *ev, std::move(response)); + return; + } + auto generatedBlob = response->Record.MutableBlobs()->Add(); + LogoBlobIDFromLogoBlobID( + MakeBlobId(TabletID(), partialBlobId), + generatedBlob->MutableBlobId()); + generatedBlob->SetOffset(offset); + generatedBlob->SetBSGroupId(Info()->GroupFor( + partialBlobId.Channel(), + partialBlobId.Generation())); + offset += length; + } + + // TODO(debnatkh): Throttling + + response->Record.SetCommitId(commitId); + + NCloud::Reply(ctx, *ev, std::move(response)); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TIndexTabletActor::HandleAddData( + const TEvIndexTablet::TEvAddDataRequest::TPtr& ev, + const TActorContext& ctx) +{ + auto* msg = ev->Get(); + + if (auto error = IsDataOperationAllowed(); HasError(error)) { + NCloud::Reply( + ctx, + *ev, + std::make_unique( + std::move(error))); + return; + } + + auto commitId = msg->Record.GetCommitId(); + if (!IsCollectBarrierAcquired(commitId)) { + // The client has sent the AddData request too late, after + // the lease has expired. + auto response = std::make_unique( + MakeError(E_REJECTED, "collect barrier expired")); + NCloud::Reply(ctx, *ev, std::move(response)); + return; + } + // We acquire the collect barrier for the second time in order to prolong + // the already acquired lease + AcquireCollectBarrier(commitId); + bool txStarted = false; + Y_DEFER + { + // Until the tx is started, it is this method's responsibility to + // release the collect barrier + if (!txStarted) { + TABLET_VERIFY(TryReleaseCollectBarrier(commitId)); + // The second one is used to release the barrier, acquired in + // GenerateBlobIds method. Though it will be eventually released + // upon lease expiration, it is better to release it as soon as + // possible. + TryReleaseCollectBarrier(commitId); + } + }; + + const TByteRange range( + msg->Record.GetOffset(), + msg->Record.GetLength(), + GetBlockSize()); + + auto validator = [&](const NProtoPrivate::TAddDataRequest& request) + { + return ValidateWriteRequest(ctx, request, range); + }; + + if (!AcceptRequest(ev, ctx, validator)) { + return; + } + + auto requestInfo = + CreateRequestInfo(ev->Sender, ev->Cookie, msg->CallContext); + requestInfo->StartedTs = ctx.Now(); + + TVector blobIds; + for (const auto& blobId: msg->Record.GetBlobIds()) { + blobIds.push_back(LogoBlobIDFromLogoBlobID(blobId)); + } + + if (blobIds.empty()) { + auto response = + std::make_unique(MakeError( + E_ARGUMENT, + "empty list of blobs given in AddData request")); + NCloud::Reply(ctx, *ev, std::move(response)); + } + + LOG_DEBUG( + ctx, + TFileStoreComponents::TABLET, + "%s %s: blobId: %s,... (total: %lu)", + LogTag.c_str(), + "AddData", + blobIds[0].ToString().c_str(), + blobIds.size()); + + AddTransaction(*requestInfo); + + ExecuteTx( + ctx, + std::move(requestInfo), + msg->Record, + range, + std::move(blobIds), + msg->Record.GetCommitId()); + txStarted = true; +} + +} // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_allocatedata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_allocatedata.cpp index 7c03737b749..aa910f0e856 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_allocatedata.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_allocatedata.cpp @@ -40,7 +40,7 @@ NProto::TError ValidateRequest( "FALLOC_FL_PUNCH_HOLE flag must be ORed with FALLOC_FL_KEEP_SIZE"); } - // TODO: Support later after https://st.yandex-team.ru/NBS-3095 and + // TODO: Support later after NBS-3095 and // add errors from https://man7.org/linux/man-pages/man2/fallocate.2.html if (HasFlag(request.GetFlags(), NProto::TAllocateDataRequest::F_INSERT_RANGE) || HasFlag(request.GetFlags(), NProto::TAllocateDataRequest::F_COLLAPSE_RANGE)) @@ -60,6 +60,16 @@ void TIndexTabletActor::HandleAllocateData( const TEvService::TEvAllocateDataRequest::TPtr& ev, const TActorContext& ctx) { + if (auto error = IsDataOperationAllowed(); HasError(error)) { + NCloud::Reply( + ctx, + *ev, + std::make_unique( + std::move(error))); + + return; + } + auto validator = [&] (const NProto::TAllocateDataRequest& request) { return ValidateRequest(request, GetBlockSize()); }; @@ -187,7 +197,7 @@ void TIndexTabletActor::ExecuteTx_AllocateData( } // TODO: We should not use this range request because tx size // is limited. Need some generic process range mechanism after - // https://st.yandex-team.ru/NBS-2979 + // NBS-2979 ZeroRange( db, args.NodeId, @@ -198,7 +208,7 @@ void TIndexTabletActor::ExecuteTx_AllocateData( if (!needExtend) { // TODO: Right now we cannot preallocate blocks, but in the future we // would do it when F_KEEP_SIZE is set - // (probably after: https://st.yandex-team.ru/NBS-3095) + // (probably after: NBS-3095) return; } diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp index 30dad1a6304..da9d7983211 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_cleanup.cpp @@ -122,7 +122,7 @@ void TIndexTabletActor::CompleteTx_Cleanup( BlobIndexOpState.Complete(); ReleaseMixedBlocks(args.RangeId); - ReleaseCollectBarrier(args.CollectBarrier); + TABLET_VERIFY(TryReleaseCollectBarrier(args.CollectBarrier)); FILESTORE_TRACK( ResponseSent_Tablet, diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_cleanupsessions.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_cleanupsessions.cpp index 9ad342c8765..07f7104f007 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_cleanupsessions.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_cleanupsessions.cpp @@ -114,7 +114,7 @@ void TCleanupSessionsActor::HandlePoisonPill( const TActorContext& ctx) { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "request cancelled")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } void TCleanupSessionsActor::ReplyAndDie( diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_collectgarbage.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_collectgarbage.cpp index 83c16d27ee1..58c8b1042a7 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_collectgarbage.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_collectgarbage.cpp @@ -249,7 +249,7 @@ void TCollectGarbageActor::HandlePoisonPill( const TActorContext& ctx) { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "request cancelled")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } void TCollectGarbageActor::HandleError(NProto::TError error) @@ -473,4 +473,19 @@ void TIndexTabletActor::HandleCollectGarbageCompleted( EnqueueCollectGarbageIfNeeded(ctx); } +//////////////////////////////////////////////////////////////////////////////// + +void TIndexTabletActor::HandleReleaseCollectBarrier( + const TEvIndexTabletPrivate::TEvReleaseCollectBarrier::TPtr& ev, + const TActorContext& ctx) +{ + Y_UNUSED(ctx); + auto commitId = ev->Get()->CommitId; + for (ui32 i = 0; i < ev->Get()->Count; ++i) { + // We do not check if the barrier was acquired, because the barrier may + // have already been released by a completed three-stage write operation + TryReleaseCollectBarrier(commitId); + } +} + } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp index 31869692e6a..16ac5699995 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_compaction.cpp @@ -251,7 +251,7 @@ void TCompactionActor::HandlePoisonPill( const TActorContext& ctx) { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "request cancelled")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } void TCompactionActor::ReplyAndDie( @@ -315,10 +315,47 @@ void TIndexTabletActor::EnqueueBlobIndexOpIfNeeded(const TActorContext& ctx) if (BlobIndexOps.Empty()) { if (compactionScore >= Config->GetCompactionThreshold()) { BlobIndexOps.Push(EBlobIndexOp::Compaction); + } else if (Config->GetNewCompactionEnabled()) { + const auto& stats = GetFileSystemStats(); + const auto compactionStats = GetCompactionMapStats(0); + const auto used = stats.GetUsedBlocksCount(); + auto alive = stats.GetMixedBlocksCount(); + if (alive > stats.GetGarbageBlocksCount()) { + alive -= stats.GetGarbageBlocksCount(); + } else { + alive = 0; + } + const auto avgGarbagePercentage = used && alive > used + ? 100 * static_cast(alive - used) / used + : 0; + const auto rangeCount = compactionStats.UsedRangesCount; + const auto avgCompactionScore = rangeCount + ? static_cast(stats.GetMixedBlobsCount()) / rangeCount + : 0; + // TODO: use GarbageCompactionThreshold + + const bool shouldCompact = + avgGarbagePercentage >= Config->GetGarbageCompactionThresholdAverage() + || avgCompactionScore >= Config->GetCompactionThresholdAverage(); + if (compactionScore > 1 && shouldCompact) { + BlobIndexOps.Push(EBlobIndexOp::Compaction); + } } if (cleanupScore >= Config->GetCleanupThreshold()) { BlobIndexOps.Push(EBlobIndexOp::Cleanup); + } else if (Config->GetNewCleanupEnabled()) { + const auto& stats = GetFileSystemStats(); + const auto compactionStats = GetCompactionMapStats(0); + const auto rangeCount = compactionStats.UsedRangesCount; + const auto avgCleanupScore = rangeCount + ? static_cast(stats.GetDeletionMarkersCount()) / rangeCount + : 0; + const bool shouldCleanup = + avgCleanupScore >= Config->GetCleanupThresholdAverage(); + if (cleanupScore && shouldCleanup) { + BlobIndexOps.Push(EBlobIndexOp::Cleanup); + } } if (GetFreshBytesCount() >= Config->GetFlushBytesThreshold()) { @@ -627,7 +664,7 @@ void TIndexTabletActor::HandleCompactionCompleted( FormatError(msg->GetError()).c_str()); ReleaseMixedBlocks(msg->MixedBlocksRanges); - ReleaseCollectBarrier(msg->CommitId); + TABLET_VERIFY(TryReleaseCollectBarrier(msg->CommitId)); BlobIndexOpState.Complete(); EnqueueBlobIndexOpIfNeeded(ctx); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp index 31eb8ca68fe..69a94d222a7 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp @@ -96,6 +96,11 @@ void TIndexTabletActor::TMetrics::Register( REGISTER_AGGREGATABLE_SUM(StatelessSessionsCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM(SessionTimeouts, EMetricType::MT_DERIVATIVE); + REGISTER_AGGREGATABLE_SUM(ReassignCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_SUM(WritableChannelCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_SUM(UnwritableChannelCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_SUM(ChannelsToMoveCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_SUM(FreshBytesCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM(MixedBytesCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM(MixedBlobsCount, EMetricType::MT_ABSOLUTE); @@ -218,7 +223,8 @@ void TIndexTabletActor::TMetrics::Update( const NProto::TFileSystemStats& stats, const NProto::TFileStorePerformanceProfile& performanceProfile, const TCompactionMapStats& compactionStats, - const TSessionsStats& sessionsStats) + const TSessionsStats& sessionsStats, + const TChannelsStats& channelsStats) { const ui32 blockSize = fileSystem.GetBlockSize(); @@ -267,6 +273,9 @@ void TIndexTabletActor::TMetrics::Update( Store(StatefulSessionsCount, sessionsStats.StatefulSessionsCount); Store(StatelessSessionsCount, sessionsStats.StatelessSessionsCount); + Store(WritableChannelCount, channelsStats.WritableChannelCount); + Store(UnwritableChannelCount, channelsStats.UnwritableChannelCount); + Store(ChannelsToMoveCount, channelsStats.ChannelsToMoveCount); BusyIdleCalc.OnUpdateStats(); } @@ -312,7 +321,8 @@ void TIndexTabletActor::RegisterStatCounters() GetFileSystemStats(), GetPerformanceProfile(), GetCompactionMapStats(1), - CalculateSessionsStats()); + CalculateSessionsStats(), + CalculateChannelsStats()); Metrics.Register(fsId, storageMediaKind); } @@ -344,7 +354,8 @@ void TIndexTabletActor::HandleUpdateCounters( GetFileSystemStats(), GetPerformanceProfile(), GetCompactionMapStats(1), - CalculateSessionsStats()); + CalculateSessionsStats(), + CalculateChannelsStats()); UpdateCountersScheduled = false; ScheduleUpdateCounters(ctx); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_createsession.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_createsession.cpp index 9ded570e8a4..cc5cf820bad 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_createsession.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_createsession.cpp @@ -15,6 +15,7 @@ void FillFeatures(const TStorageConfig& config, NProto::TFileStore& fileStore) { auto* features = fileStore.MutableFeatures(); features->SetTwoStageReadEnabled(config.GetTwoStageReadEnabled()); + features->SetThreeStageWriteEnabled(config.GetThreeStageWriteEnabled()); features->SetEntryTimeout(config.GetEntryTimeout().MilliSeconds()); features->SetNegativeEntryTimeout( config.GetNegativeEntryTimeout().MilliSeconds()); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_deletecheckpoint.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_deletecheckpoint.cpp index bbaeebb34f1..4cfff2cbc7e 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_deletecheckpoint.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_deletecheckpoint.cpp @@ -18,6 +18,16 @@ void TIndexTabletActor::HandleDeleteCheckpoint( const TEvIndexTabletPrivate::TEvDeleteCheckpointRequest::TPtr& ev, const TActorContext& ctx) { + if (auto error = IsDataOperationAllowed(); HasError(error)) { + NCloud::Reply( + ctx, + *ev, + std::make_unique( + std::move(error))); + + return; + } + auto* msg = ev->Get(); LOG_DEBUG(ctx, TFileStoreComponents::TABLET, @@ -245,7 +255,7 @@ void TIndexTabletActor::CompleteTx_DeleteCheckpoint( FormatError(args.Error).c_str()); ReleaseMixedBlocks(args.MixedBlocksRanges); - ReleaseCollectBarrier(args.CollectBarrier); + TABLET_VERIFY(TryReleaseCollectBarrier(args.CollectBarrier)); auto response = std::make_unique(args.Error); NCloud::Reply(ctx, *args.RequestInfo, std::move(response)); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_destroycheckpoint.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_destroycheckpoint.cpp index 83186a77165..08ebcc282f1 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_destroycheckpoint.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_destroycheckpoint.cpp @@ -158,7 +158,7 @@ void TDestroyCheckpointActor::HandlePoisonPill( const TActorContext& ctx) { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "request cancelled")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } void TDestroyCheckpointActor::ReplyAndDie( @@ -252,6 +252,16 @@ void TIndexTabletActor::HandleDestroyCheckpoint( const TEvService::TEvDestroyCheckpointRequest::TPtr& ev, const TActorContext& ctx) { + if (auto error = IsDataOperationAllowed(); HasError(error)) { + NCloud::Reply( + ctx, + *ev, + std::make_unique( + std::move(error))); + + return; + } + auto* msg = ev->Get(); const auto& checkpointId = msg->Record.GetCheckpointId(); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_destroyhandle.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_destroyhandle.cpp index c8ad9ff7057..e939e6586bd 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_destroyhandle.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_destroyhandle.cpp @@ -13,6 +13,16 @@ void TIndexTabletActor::HandleDestroyHandle( const TEvService::TEvDestroyHandleRequest::TPtr& ev, const TActorContext& ctx) { + if (auto error = IsDataOperationAllowed(); HasError(error)) { + NCloud::Reply( + ctx, + *ev, + std::make_unique( + std::move(error))); + + return; + } + if (!AcceptRequest(ev, ctx)) { return; } diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_destroysession.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_destroysession.cpp index 6303fe031b4..751189ce3b3 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_destroysession.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_destroysession.cpp @@ -13,6 +13,16 @@ void TIndexTabletActor::HandleDestroySession( const TEvIndexTablet::TEvDestroySessionRequest::TPtr& ev, const TActorContext& ctx) { + if (auto error = IsDataOperationAllowed(); HasError(error)) { + NCloud::Reply( + ctx, + *ev, + std::make_unique( + std::move(error))); + + return; + } + auto* msg = ev->Get(); const auto& clientId = GetClientId(msg->Record); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_flush.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_flush.cpp index 45fcb5c558a..3aede2331a1 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_flush.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_flush.cpp @@ -157,7 +157,7 @@ void TFlushActor::HandlePoisonPill( const TActorContext& ctx) { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "request cancelled")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } void TFlushActor::ReplyAndDie( @@ -377,7 +377,7 @@ void TIndexTabletActor::HandleFlushCompleted( LogTag.c_str(), FormatError(msg->GetError()).c_str()); - ReleaseCollectBarrier(msg->CommitId); + TABLET_VERIFY(TryReleaseCollectBarrier(msg->CommitId)); FlushState.Complete(); WorkerActors.erase(ev->Sender); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_flush_bytes.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_flush_bytes.cpp index f94f16b7193..7ce84cb08b2 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_flush_bytes.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_flush_bytes.cpp @@ -395,7 +395,7 @@ void TFlushBytesActor::HandlePoisonPill( const TActorContext& ctx) { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "request cancelled")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } void TFlushBytesActor::ReplyAndDie( @@ -829,7 +829,7 @@ void TIndexTabletActor::HandleFlushBytesCompleted( FormatError(msg->GetError()).c_str()); ReleaseMixedBlocks(msg->MixedBlocksRanges); - ReleaseCollectBarrier(msg->CollectCommitId); + TABLET_VERIFY(TryReleaseCollectBarrier(msg->CollectCommitId)); WorkerActors.erase(ev->Sender); auto requestInfo = CreateRequestInfo( diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_monitoring.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_monitoring.cpp index 367c0c7236b..d003f4bf28f 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_monitoring.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_monitoring.cpp @@ -4,7 +4,9 @@ #include #include +#include #include +#include #include #include @@ -715,6 +717,34 @@ void DumpSessions(IOutputStream& out, const TVector& sessions) //////////////////////////////////////////////////////////////////////////////// +void DumpChannels( + IOutputStream& out, + const TVector& channelInfos, + const TTabletStorageInfo& storage, + ui64 hiveTabletId) +{ + NCloud::NStorage::DumpChannels( + out, + channelInfos, + storage, + [&] (ui32 groupId, const TString& storagePool) { + // TODO: group mon url + Y_UNUSED(groupId); + Y_UNUSED(storagePool); + return TString(); + }, + [&] (IOutputStream& out, ui64 hiveTabletId, ui64 tabletId, ui32 c) { + // TODO: reassign button + Y_UNUSED(out); + Y_UNUSED(hiveTabletId); + Y_UNUSED(tabletId); + Y_UNUSED(c); + }, + hiveTabletId); +} + +//////////////////////////////////////////////////////////////////////////////// + void GenerateActionsJS(IOutputStream& out) { out << R"___( @@ -828,7 +858,7 @@ struct TIndexTabletMonitoringActor const TActorContext& ctx) { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "request cancelled"), {}); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down"), {}); } void ReplyAndDie( @@ -988,7 +1018,22 @@ void TIndexTabletActor::HandleHttpInfo_Default( out << ""; } - TAG(TH3) { out << "Profilling allocator stats"; } + TAG(TH3) { + out << "Channels"; + } + + ui64 hiveTabletId = Config->GetTenantHiveTabletId(); + if (!hiveTabletId) { + hiveTabletId = NCloud::NStorage::GetHiveTabletId(ctx); + } + + DumpChannels( + out, + MakeChannelMonInfos(), + *Info(), + hiveTabletId); + + TAG(TH3) { out << "Profiling allocator stats"; } DumpProfillingAllocatorStats(GetFileStoreProfilingRegistry(), out); TAG(TH3) { out << "Blob index stats"; } diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_readblob.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_readblob.cpp index c94a528ddc5..fed0db44d76 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_readblob.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_readblob.cpp @@ -245,7 +245,7 @@ void TReadBlobActor::HandlePoisonPill( const TActorContext& ctx) { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "request cancelled")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } void TReadBlobActor::ReplyError( diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp index a227fc18995..d7619c3fae4 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp @@ -422,7 +422,7 @@ void TReadDataActor::HandlePoisonPill( const TActorContext& ctx) { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "request cancelled")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } void TReadDataActor::ReplyAndDie( @@ -565,7 +565,7 @@ void TIndexTabletActor::HandleReadDataCompleted( const auto* msg = ev->Get(); ReleaseMixedBlocks(msg->MixedBlocksRanges); - ReleaseCollectBarrier(msg->CommitId); + TABLET_VERIFY(TryReleaseCollectBarrier(msg->CommitId)); WorkerActors.erase(ev->Sender); Metrics.ReadData.Count.fetch_add(msg->Count, std::memory_order_relaxed); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_request.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_request.cpp index 4813d94c535..1b4cf0daa79 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_request.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_request.cpp @@ -87,6 +87,12 @@ void TIndexTabletActor::CompleteResponse( callContext->RequestId, FormatError(response.GetError()).c_str()); + if (HasError(response.GetError())) { + auto* e = response.MutableError(); + e->SetMessage(TStringBuilder() + << e->GetMessage() << ", request-id: " << callContext->RequestId); + } + FILESTORE_TRACK( ResponseSent_Tablet, callContext, @@ -114,6 +120,8 @@ template void TIndexTabletActor::CompleteResponse( FILESTORE_SERVICE(FILESTORE_GENERATE_IMPL, TEvService) FILESTORE_GENERATE_IMPL(DescribeData, TEvIndexTablet) FILESTORE_GENERATE_IMPL(DescribeSessions, TEvIndexTablet) +FILESTORE_GENERATE_IMPL(GenerateBlobIds, TEvIndexTablet) +FILESTORE_GENERATE_IMPL(AddData, TEvIndexTablet) #undef FILESTORE_GENERATE_IMPL diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_resetsession.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_resetsession.cpp index a328f57c0dc..0b96170b197 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_resetsession.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_resetsession.cpp @@ -127,8 +127,21 @@ void TIndexTabletActor::ExecuteTx_ResetSession( auto nodeId = handle->GetNodeId(); DestroyHandle(db, &*(handle++)); + LOG_INFO(ctx, TFileStoreComponents::TABLET, + "%s Removing handle upon session reset s:%s n:%lu", + LogTag.c_str(), + args.SessionId.c_str(), + nodeId); + auto it = args.Nodes.find(nodeId); if (it != args.Nodes.end() && !HasOpenHandles(nodeId)) { + LOG_INFO(ctx, TFileStoreComponents::TABLET, + "%s Removing node upon session reset s:%s n:%lu (size %lu)", + LogTag.c_str(), + args.SessionId.c_str(), + nodeId, + it->Attrs.GetSize()); + RemoveNode( db, *it, diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_setnodeattr.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_setnodeattr.cpp index 9fb5a938895..6ce8f86d03e 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_setnodeattr.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_setnodeattr.cpp @@ -39,6 +39,21 @@ void TIndexTabletActor::HandleSetNodeAttr( const TEvService::TEvSetNodeAttrRequest::TPtr& ev, const TActorContext& ctx) { + auto* msg = ev->Get(); + + const auto flags = msg->Record.GetFlags(); + if (HasFlag(flags, NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE)) { + if (auto error = IsDataOperationAllowed(); HasError(error)) { + NCloud::Reply( + ctx, + *ev, + std::make_unique( + std::move(error))); + + return; + } + } + auto validator = [&] (const NProto::TSetNodeAttrRequest& request) { return ValidateRequest(request, GetBlockSize()); }; @@ -46,7 +61,6 @@ void TIndexTabletActor::HandleSetNodeAttr( if (!AcceptRequest(ev, ctx, validator)) { return; } - auto* msg = ev->Get(); auto requestInfo = CreateRequestInfo( ev->Sender, ev->Cookie, diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_truncate.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_truncate.cpp index 2a573505213..3023a83eff3 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_truncate.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_truncate.cpp @@ -155,6 +155,16 @@ void TIndexTabletActor::HandleTruncate( const TEvIndexTabletPrivate::TEvTruncateRequest::TPtr& ev, const TActorContext& ctx) { + if (auto error = IsDataOperationAllowed(); HasError(error)) { + NCloud::Reply( + ctx, + *ev, + std::make_unique( + std::move(error))); + + return; + } + auto* msg = ev->Get(); LOG_DEBUG(ctx, TFileStoreComponents::TABLET, diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_unlinknode.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_unlinknode.cpp index 08ace989d4e..cd0e91c7bad 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_unlinknode.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_unlinknode.cpp @@ -32,6 +32,16 @@ void TIndexTabletActor::HandleUnlinkNode( const TEvService::TEvUnlinkNodeRequest::TPtr& ev, const TActorContext& ctx) { + if (auto error = IsDataOperationAllowed(); HasError(error)) { + NCloud::Reply( + ctx, + *ev, + std::make_unique( + std::move(error))); + + return; + } + auto* session = AcceptRequest(ev, ctx, ValidateRequest); if (!session) { return; @@ -90,7 +100,7 @@ bool TIndexTabletActor::PrepareTx_UnlinkNode( } if (!args.ChildRef) { - args.Error = ErrorInvalidTarget(args.ParentNodeId); + args.Error = ErrorInvalidTarget(args.ParentNodeId, args.Name); return true; } diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_writebatch.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_writebatch.cpp index ffe5dbb9b3e..5e7d5d22a81 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_writebatch.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_writebatch.cpp @@ -175,7 +175,7 @@ void TWriteBatchActor::HandlePoisonPill( const TActorContext& ctx) { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "request cancelled")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } void TWriteBatchActor::ReplyAndDie( @@ -286,7 +286,7 @@ void TIndexTabletActor::HandleWriteBatchCompleted( FormatError(msg->GetError()).c_str()); } - ReleaseCollectBarrier(msg->CommitId); + TABLET_VERIFY(TryReleaseCollectBarrier(msg->CommitId)); WorkerActors.erase(ev->Sender); EnqueueBlobIndexOpIfNeeded(ctx); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_writeblob.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_writeblob.cpp index ac38ca4dfd4..a3d57769cba 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_writeblob.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_writeblob.cpp @@ -180,7 +180,7 @@ void TWriteBlobActor::HandlePoisonPill( const TActorContext& ctx) { Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "request cancelled")); + ReplyAndDie(ctx, MakeError(E_REJECTED, "tablet is shutting down")); } void TWriteBlobActor::ReplyError( diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_writedata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_writedata.cpp index 09199a1e033..df669f0d8d5 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_writedata.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_writedata.cpp @@ -1,14 +1,11 @@ #include "tablet_actor.h" -#include "helpers.h" - #include #include +#include #include #include -#include - #include #include #include @@ -20,210 +17,6 @@ using namespace NActors; using namespace NKikimr; using namespace NKikimr::NTabletFlatExecutor; -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -class TWriteDataActor final - : public TActorBootstrapped -{ -private: - const ITraceSerializerPtr TraceSerializer; - - const TString LogTag; - const TActorId Tablet; - const TRequestInfoPtr RequestInfo; - - const ui64 CommitId; - /*const*/ TVector Blobs; - const TWriteRange WriteRange; - ui32 BlobsSize = 0; - -public: - TWriteDataActor( - ITraceSerializerPtr traceSerializer, - TString logTag, - TActorId tablet, - TRequestInfoPtr requestInfo, - ui64 commitId, - TVector blobs, - TWriteRange writeRange); - - void Bootstrap(const TActorContext& ctx); - -private: - STFUNC(StateWork); - - void WriteBlob(const TActorContext& ctx); - void HandleWriteBlobResponse( - const TEvIndexTabletPrivate::TEvWriteBlobResponse::TPtr& ev, - const TActorContext& ctx); - - void AddBlob(const TActorContext& ctx); - void HandleAddBlobResponse( - const TEvIndexTabletPrivate::TEvAddBlobResponse::TPtr& ev, - const TActorContext& ctx); - - void HandlePoisonPill( - const TEvents::TEvPoisonPill::TPtr& ev, - const TActorContext& ctx); - - void ReplyAndDie( - const TActorContext& ctx, - const NProto::TError& error = {}); -}; - -//////////////////////////////////////////////////////////////////////////////// - -TWriteDataActor::TWriteDataActor( - ITraceSerializerPtr traceSerializer, - TString logTag, - TActorId tablet, - TRequestInfoPtr requestInfo, - ui64 commitId, - TVector blobs, - TWriteRange writeRange) - : TraceSerializer(std::move(traceSerializer)) - , LogTag(std::move(logTag)) - , Tablet(tablet) - , RequestInfo(std::move(requestInfo)) - , CommitId(commitId) - , Blobs(std::move(blobs)) - , WriteRange(writeRange) -{ - for (const auto& blob: Blobs) { - BlobsSize += blob.BlobContent.Size(); - } -} - -void TWriteDataActor::Bootstrap(const TActorContext& ctx) -{ - FILESTORE_TRACK( - RequestReceived_TabletWorker, - RequestInfo->CallContext, - "WriteData"); - - WriteBlob(ctx); - Become(&TThis::StateWork); -} - -void TWriteDataActor::WriteBlob(const TActorContext& ctx) -{ - auto request = std::make_unique( - RequestInfo->CallContext - ); - - for (auto& blob: Blobs) { - request->Blobs.emplace_back(blob.BlobId, std::move(blob.BlobContent)); - } - - NCloud::Send(ctx, Tablet, std::move(request)); -} - -void TWriteDataActor::HandleWriteBlobResponse( - const TEvIndexTabletPrivate::TEvWriteBlobResponse::TPtr& ev, - const TActorContext& ctx) -{ - const auto* msg = ev->Get(); - - if (FAILED(msg->GetStatus())) { - ReplyAndDie(ctx, msg->GetError()); - return; - } - - AddBlob(ctx); -} - -void TWriteDataActor::AddBlob(const TActorContext& ctx) -{ - auto request = std::make_unique( - RequestInfo->CallContext - ); - request->Mode = EAddBlobMode::Write; - request->WriteRanges.push_back(WriteRange); - - for (const auto& blob: Blobs) { - request->MergedBlobs.emplace_back( - blob.BlobId, - blob.Block, - blob.BlocksCount); - } - - NCloud::Send(ctx, Tablet, std::move(request)); -} - -void TWriteDataActor::HandleAddBlobResponse( - const TEvIndexTabletPrivate::TEvAddBlobResponse::TPtr& ev, - const TActorContext& ctx) -{ - const auto* msg = ev->Get(); - ReplyAndDie(ctx, msg->GetError()); -} - -void TWriteDataActor::HandlePoisonPill( - const TEvents::TEvPoisonPill::TPtr& ev, - const TActorContext& ctx) -{ - Y_UNUSED(ev); - ReplyAndDie(ctx, MakeError(E_REJECTED, "request cancelled")); -} - -void TWriteDataActor::ReplyAndDie( - const TActorContext& ctx, - const NProto::TError& error) -{ - { - // notify tablet - using TCompletion = TEvIndexTabletPrivate::TEvWriteDataCompleted; - auto response = std::make_unique(error); - response->CommitId = CommitId; - response->Count = 1; - response->Size = BlobsSize; - response->Time = ctx.Now() - RequestInfo->StartedTs; - NCloud::Send(ctx, Tablet, std::move(response)); - } - - FILESTORE_TRACK( - ResponseSent_TabletWorker, - RequestInfo->CallContext, - "WriteData"); - - if (RequestInfo->Sender != Tablet) { - auto response = std::make_unique(error); - LOG_DEBUG(ctx, TFileStoreComponents::TABLET_WORKER, - "%s WriteData: #%lu completed (%s)", - LogTag.c_str(), - RequestInfo->CallContext->RequestId, - FormatError(response->Record.GetError()).c_str()); - - BuildTraceInfo( - TraceSerializer, - RequestInfo->CallContext, - response->Record); - BuildThrottlerInfo(*RequestInfo->CallContext, response->Record); - - NCloud::Reply(ctx, *RequestInfo, std::move(response)); - } - - Die(ctx); -} - -STFUNC(TWriteDataActor::StateWork) -{ - switch (ev->GetTypeRewrite()) { - HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); - - HFunc(TEvIndexTabletPrivate::TEvWriteBlobResponse, HandleWriteBlobResponse); - HFunc(TEvIndexTabletPrivate::TEvAddBlobResponse, HandleAddBlobResponse); - - default: - HandleUnexpectedEvent(ev, TFileStoreComponents::TABLET_WORKER); - break; - } -} - -} // namespace - //////////////////////////////////////////////////////////////////////////////// void TIndexTabletActor::HandleWriteData( @@ -303,33 +96,9 @@ void TIndexTabletActor::HandleWriteData( } } - auto validator = [&] (const NProto::TWriteDataRequest& request) { - if (auto error = ValidateRange(range); HasError(error)) { - return error; - } - - auto* handle = FindHandle(request.GetHandle()); - if (!handle || handle->GetSessionId() != GetSessionId(request)) { - return ErrorInvalidHandle(request.GetHandle()); - } - - if (!IsWriteAllowed(BuildBackpressureThresholds())) { - if (CompactionStateLoadStatus.Finished - && ++BackpressureErrorCount >= - Config->GetMaxBackpressureErrorsBeforeSuicide()) - { - LOG_WARN(ctx, TFileStoreComponents::TABLET_WORKER, - "%s Suiciding after %u backpressure errors", - LogTag.c_str(), - BackpressureErrorCount); - - Suicide(ctx); - } - - return MakeError(E_REJECTED, "rejected due to backpressure"); - } - - return NProto::TError{}; + auto validator = [&](const NProto::TWriteDataRequest& request) + { + return ValidateWriteRequest(ctx, request, range); }; if (!AcceptRequest(ev, ctx, validator)) { @@ -380,7 +149,7 @@ void TIndexTabletActor::HandleWriteDataCompleted( { const auto* msg = ev->Get(); - ReleaseCollectBarrier(msg->CommitId); + TABLET_VERIFY(TryReleaseCollectBarrier(msg->CommitId)); WorkerActors.erase(ev->Sender); EnqueueBlobIndexOpIfNeeded(ctx); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_zerorange.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_zerorange.cpp index 696aa0d38cc..f0a6baf1c87 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_zerorange.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_zerorange.cpp @@ -15,6 +15,16 @@ void TIndexTabletActor::HandleZeroRange( const TEvIndexTabletPrivate::TEvZeroRangeRequest::TPtr& ev, const TActorContext& ctx) { + if (auto error = IsDataOperationAllowed(); HasError(error)) { + NCloud::Reply( + ctx, + *ev, + std::make_unique( + std::move(error))); + + return; + } + auto* msg = ev->Get(); LOG_DEBUG(ctx, TFileStoreComponents::TABLET, diff --git a/cloud/filestore/libs/storage/tablet/tablet_counters.h b/cloud/filestore/libs/storage/tablet/tablet_counters.h index 8be4fa7d47e..389ea9d2001 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_counters.h +++ b/cloud/filestore/libs/storage/tablet/tablet_counters.h @@ -31,6 +31,7 @@ namespace NCloud::NFileStore::NStorage { xxx(CheckpointBlocksCount, __VA_ARGS__) \ xxx(CheckpointBlobsCount, __VA_ARGS__) \ xxx(FreshBytesCount, __VA_ARGS__) \ + xxx(LastCollectCommitId, __VA_ARGS__) \ // FILESTORE_TABLET_STATS #define FILESTORE_TABLET_SIMPLE_COUNTERS(xxx) \ diff --git a/cloud/filestore/libs/storage/tablet/tablet_private.h b/cloud/filestore/libs/storage/tablet/tablet_private.h index 006084e8fcb..15ec02c53c5 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_private.h +++ b/cloud/filestore/libs/storage/tablet/tablet_private.h @@ -580,6 +580,23 @@ struct TEvIndexTabletPrivate TSet Nodes; }; + // + // Release collect barrier + // + + struct TReleaseCollectBarrier + { + // Commit id to release + ui64 CommitId; + // Number of times to perform the release + ui32 Count; + + TReleaseCollectBarrier(ui64 commitId, ui32 count) + : CommitId(commitId) + , Count(count) + {} + }; + // // Events declaration // @@ -597,6 +614,8 @@ struct TEvIndexTabletPrivate EvReadDataCompleted, EvWriteDataCompleted, + EvReleaseCollectBarrier, + EvEnd }; @@ -609,6 +628,9 @@ struct TEvIndexTabletPrivate using TEvUpdateCounters = TRequestEvent; using TEvUpdateLeakyBucketCounters = TRequestEvent; + using TEvReleaseCollectBarrier = + TRequestEvent; + using TEvReadDataCompleted = TResponseEvent; using TEvWriteDataCompleted = TResponseEvent; }; diff --git a/cloud/filestore/libs/storage/tablet/tablet_state.h b/cloud/filestore/libs/storage/tablet/tablet_state.h index 05680f941f9..795c82e82b5 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state.h +++ b/cloud/filestore/libs/storage/tablet/tablet_state.h @@ -7,6 +7,7 @@ #include "checkpoint.h" #include "helpers.h" +#include "rebase_logic.h" #include "session.h" #include @@ -25,6 +26,7 @@ #include #include +#include #include @@ -233,6 +235,9 @@ FILESTORE_FILESYSTEM_STATS(FILESTORE_DECLARE_COUNTER) TVector GetChannels(EChannelDataKind kind) const; TVector GetUnwritableChannels() const; TVector GetChannelsToMove(ui32 percentageThreshold) const; + TVector MakeChannelMonInfos() const; + + TChannelsStats CalculateChannelsStats() const; void RegisterUnwritableChannel(ui32 channel); void RegisterChannelToMove(ui32 channel); @@ -737,6 +742,8 @@ FILESTORE_DUPCACHE_REQUESTS(FILESTORE_DECLARE_DUPCACHE) const TPartialBlobId& blobId, const TVector& blocks); + TRebaseResult RebaseMixedBlocks(TVector& blocks) const; + // // Garbage // @@ -758,7 +765,8 @@ FILESTORE_DUPCACHE_REQUESTS(FILESTORE_DECLARE_DUPCACHE) } void AcquireCollectBarrier(ui64 commitId); - void ReleaseCollectBarrier(ui64 commitId); + bool TryReleaseCollectBarrier(ui64 commitId); + bool IsCollectBarrierAcquired(ui64 commitId) const; ui64 GetCollectCommitId() const; diff --git a/cloud/filestore/libs/storage/tablet/tablet_state_channels.cpp b/cloud/filestore/libs/storage/tablet/tablet_state_channels.cpp index 7e35d275a81..d1ccedcdb62 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state_channels.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_state_channels.cpp @@ -39,6 +39,17 @@ TVector TIndexTabletState::GetChannelsToMove(ui32 percentageThreshold) con return Impl->Channels.GetChannelsToMove(percentageThreshold); } +TVector +TIndexTabletState::MakeChannelMonInfos() const +{ + return Impl->Channels.MakeChannelMonInfos(); +} + +TChannelsStats TIndexTabletState::CalculateChannelsStats() const +{ + return Impl->Channels.CalculateChannelsStats(); +} + void TIndexTabletState::LoadChannels() { Y_ABORT_UNLESS(Impl->Channels.Empty()); diff --git a/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp b/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp index 45099e0c897..e07258522f4 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_state_data.cpp @@ -1,7 +1,6 @@ #include "tablet_state_impl.h" #include "profile_log_events.h" -#include "rebase_logic.h" #include #include @@ -44,7 +43,7 @@ bool TIndexTabletState::EnqueueWriteBatch(std::unique_ptr request TWriteRequestList TIndexTabletState::DequeueWriteBatch() { - // TODO: deduplicate writes (https://st.yandex-team.ru/NBS-2161) + // TODO: deduplicate writes (NBS-2161) return std::move(Impl->WriteBatch); } @@ -580,21 +579,7 @@ bool TIndexTabletState::WriteMixedBlocks( Impl->MixedBlocks.ApplyDeletionMarkers(GetRangeIdHasher(), blocks); } - auto rebaseResult = RebaseMixedBlocks( - blocks, - GetCurrentCommitId(), - [=] (ui64 nodeId, ui64 commitId) { - return Impl->Checkpoints.FindCheckpoint(nodeId, commitId); - }, - [=] (ui64 nodeId, ui32 blockIndex) { - return IntersectsWithFresh( - Impl->FreshBytes, - Impl->FreshBlocks, - GetBlockSize(), - nodeId, - blockIndex - ); - }); + auto rebaseResult = RebaseMixedBlocks(blocks); if (!rebaseResult.LiveBlocks) { AddGarbageBlob(db, blobId); @@ -676,9 +661,32 @@ void TIndexTabletState::DeleteMixedBlocks( DecrementMixedBlocksCount(db, blocks.size()); } +TRebaseResult TIndexTabletState::RebaseMixedBlocks(TVector& blocks) const +{ + return RebaseBlocks( + blocks, + GetCurrentCommitId(), + [=] (ui64 nodeId, ui64 commitId) { + return Impl->Checkpoints.FindCheckpoint(nodeId, commitId); + }, + [=] (ui64 nodeId, ui32 blockIndex) { + return IntersectsWithFresh( + Impl->FreshBytes, + Impl->FreshBlocks, + GetBlockSize(), + nodeId, + blockIndex + ); + }); +} + TVector TIndexTabletState::GetBlobsForCompaction(ui32 rangeId) const { - return Impl->MixedBlocks.GetBlobsForCompaction(rangeId); + auto blobs = Impl->MixedBlocks.GetBlobsForCompaction(rangeId); + for (auto& blob: blobs) { + RebaseMixedBlocks(blob.Blocks); + } + return blobs; } TMixedBlobMeta TIndexTabletState::FindBlob(ui32 rangeId, TPartialBlobId blobId) const @@ -808,21 +816,7 @@ void TIndexTabletState::RewriteMixedBlocks( Impl->MixedBlocks.ApplyDeletionMarkers(GetRangeIdHasher(), blob.Blocks); - auto rebaseResult = RebaseMixedBlocks( - blob.Blocks, - GetCurrentCommitId(), - [=] (ui64 nodeId, ui64 commitId) { - return Impl->Checkpoints.FindCheckpoint(nodeId, commitId); - }, - [=] (ui64 nodeId, ui32 blockIndex) { - return IntersectsWithFresh( - Impl->FreshBytes, - Impl->FreshBlocks, - GetBlockSize(), - nodeId, - blockIndex - ); - }); + auto rebaseResult = RebaseMixedBlocks(blob.Blocks); if (!rebaseResult.LiveBlocks) { DeleteMixedBlocks(db, blob.BlobId, blob.Blocks); @@ -927,9 +921,15 @@ void TIndexTabletState::AcquireCollectBarrier(ui64 commitId) Impl->GarbageQueue.AcquireCollectBarrier(commitId); } -void TIndexTabletState::ReleaseCollectBarrier(ui64 commitId) +// returns true if the barrier was present +bool TIndexTabletState::TryReleaseCollectBarrier(ui64 commitId) +{ + return Impl->GarbageQueue.TryReleaseCollectBarrier(commitId); +} + +bool TIndexTabletState::IsCollectBarrierAcquired(ui64 commitId) const { - Impl->GarbageQueue.ReleaseCollectBarrier(commitId); + return Impl->GarbageQueue.IsCollectBarrierAcquired(commitId); } ui64 TIndexTabletState::GetCollectCommitId() const diff --git a/cloud/filestore/libs/storage/tablet/tablet_tx.h b/cloud/filestore/libs/storage/tablet/tablet_tx.h index af5e3e47196..4d51f412f38 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_tx.h +++ b/cloud/filestore/libs/storage/tablet/tablet_tx.h @@ -97,6 +97,7 @@ namespace NCloud::NFileStore::NStorage { \ xxx(ReadData, __VA_ARGS__) \ xxx(WriteData, __VA_ARGS__) \ + xxx(AddData, __VA_ARGS__) \ xxx(WriteBatch, __VA_ARGS__) \ xxx(AllocateData, __VA_ARGS__) \ \ @@ -1212,6 +1213,42 @@ struct TTxIndexTablet } }; + // + // AddData + // + + struct TAddData : TSessionAware + { + const TRequestInfoPtr RequestInfo; + const ui64 Handle; + const TByteRange ByteRange; + TVector BlobIds; + ui64 CommitId; + + ui64 NodeId = InvalidNodeId; + TMaybe Node; + + TAddData( + TRequestInfoPtr requestInfo, + const NProtoPrivate::TAddDataRequest& request, + TByteRange byteRange, + TVector blobIds, + ui64 commitId) + : TSessionAware(request) + , RequestInfo(std::move(requestInfo)) + , Handle(request.GetHandle()) + , ByteRange(byteRange) + , BlobIds(std::move(blobIds)) + , CommitId(commitId) + {} + + void Clear() + { + NodeId = InvalidNodeId; + Node.Clear(); + } + }; + // // WriteBatch // diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp index 0667631f847..aa4f50f8830 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp @@ -1,7 +1,5 @@ #include "tablet.h" -#include -#include #include #include @@ -17,8 +15,6 @@ namespace { //////////////////////////////////////////////////////////////////////////////// -using NCloud::NFileStore::NMetrics::TLabel; - TFileSystemConfig MakeThrottlerConfig( bool throttlingEnabled, ui32 maxReadIops, @@ -57,138 +53,6 @@ TFileSystemConfig MakeThrottlerConfig( //////////////////////////////////////////////////////////////////////////////// -class TTestRegistryVisitor - : public NMetrics::IRegistryVisitor -{ -private: - struct TMetricsEntry - { - TInstant Time; - NMetrics::EAggregationType AggrType; - NMetrics::EMetricType MetrType; - THashMap Labels; - i64 Value; - - bool Matches(const TVector& labels) const - { - for (auto& label: labels) { - auto it = Labels.find(label.GetName()); - if (it == Labels.end() || it->second != label.GetValue()) { - return false; - } - } - return true; - } - }; - - TVector MetricsEntries; - TMetricsEntry CurrentEntry; - -public: - void OnStreamBegin() override - { - CurrentEntry = TMetricsEntry(); - MetricsEntries.clear(); - } - - void OnStreamEnd() override - {} - - void OnMetricBegin( - TInstant time, - NMetrics::EAggregationType aggrType, - NMetrics::EMetricType metrType) override - { - CurrentEntry.Time = time; - CurrentEntry.AggrType = aggrType; - CurrentEntry.MetrType = metrType; - } - - void OnMetricEnd() override - { - MetricsEntries.emplace_back(std::move(CurrentEntry)); - } - - void OnLabelsBegin() override - {} - - void OnLabelsEnd() override - {} - - void OnLabel(TStringBuf name, TStringBuf value) override - { - CurrentEntry.Labels.emplace(TString(name), TString(value)); - } - - void OnValue(i64 value) override - { - CurrentEntry.Value = value; - } - -public: - const TVector& GetEntries() const - { - return MetricsEntries; - } - - void ValidateExpectedCounters( - const TVector, i64>>& expectedCounters) - { - for (const auto& [labels, value]: expectedCounters) { - const auto labelsStr = LabelsToString(labels); - - int matchingCountersCount = 0; - for (const auto& entry: MetricsEntries) { - if (entry.Matches(labels)) { - ++matchingCountersCount; - UNIT_ASSERT_VALUES_EQUAL_C(entry.Value, value, labelsStr); - } - } - UNIT_ASSERT_VALUES_EQUAL_C(matchingCountersCount, 1, labelsStr); - } - } - - void ValidateExpectedHistogram( - const TVector, i64>>& expectedCounters, - bool checkEqual) - { - for (const auto& [labels, value]: expectedCounters) { - const auto labelsStr = LabelsToString(labels); - i64 total = 0; - - int matchingCountersCount = 0; - for (const auto& entry: MetricsEntries) { - if (entry.Matches(labels)) { - ++matchingCountersCount; - total += entry.Value; - } - } - if (checkEqual) { - UNIT_ASSERT_VALUES_EQUAL_C(total, value, labelsStr); - } else { - UNIT_ASSERT_VALUES_UNEQUAL_C(total, value, labelsStr); - } - UNIT_ASSERT_VALUES_UNEQUAL_C(matchingCountersCount, 0, labelsStr); - } - } - -private: - static TString LabelsToString(const TVector& labels) - { - TStringBuilder labelsStr; - for (const auto& label: labels) { - if (labelsStr) { - labelsStr << ", "; - } - labelsStr << label.GetName() << "=" << label.GetValue(); - } - - return labelsStr; - } -}; - -//////////////////////////////////////////////////////////////////////////////// - struct TEnv : public NUnitTest::TBaseFixture { diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp index 2f7429f92cf..0d3e79f56ee 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp @@ -7,6 +7,7 @@ #include +#include #include #include @@ -27,12 +28,16 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) using namespace NCloud::NStorage; -#define TABLET_TEST_IMPL(name, largeBS) \ +#define TABLET_TEST_HEAD(name) \ void TestImpl##name(TFileSystemConfig tabletConfig); \ Y_UNIT_TEST(name) \ { \ TestImpl##name(TFileSystemConfig{.BlockSize = 4_KB}); \ } \ +// TABLET_TEST_HEAD + +#define TABLET_TEST_IMPL(name, largeBS) \ + TABLET_TEST_HEAD(name) \ Y_UNIT_TEST(name##largeBS) \ { \ TestImpl##name(TFileSystemConfig{.BlockSize = largeBS}); \ @@ -40,6 +45,11 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) void TestImpl##name(TFileSystemConfig tabletConfig) \ // TABLET_TEST_IMPL +#define TABLET_TEST_4K_ONLY(name) \ + TABLET_TEST_HEAD(name) \ + void TestImpl##name(TFileSystemConfig tabletConfig) \ +// TABLET_TEST_4K_ONLY + #define TABLET_TEST(name) \ TABLET_TEST_IMPL(name, 128_KB) \ // TABLET_TEST @@ -980,7 +990,9 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) const auto& stats = response->Record.GetStats(); UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlocksCount(), 0); UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlobsCount(), 0); - UNIT_ASSERT_VALUES_EQUAL(stats.GetGarbageQueueSize(), 2 * block); + // still 1 block - Compaction doesn't move unneeded blocks to its + // dst blobs + UNIT_ASSERT_VALUES_EQUAL(stats.GetGarbageQueueSize(), block); } id = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); @@ -1001,7 +1013,7 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) // GarbageQueueSize remains the same since FlushBytes doesn't // generate any new blobs after UnlinkNode (leads to Truncate to // zero size) - UNIT_ASSERT_VALUES_EQUAL(stats.GetGarbageQueueSize(), 2 * block); + UNIT_ASSERT_VALUES_EQUAL(stats.GetGarbageQueueSize(), block); } tablet.CollectGarbage(); @@ -1063,12 +1075,12 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) { auto response = tablet.GetStorageStats(); const auto& stats = response->Record.GetStats(); - UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlocksCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlocksCount(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlobsCount(), 1); UNIT_ASSERT_VALUES_EQUAL( stats.GetGarbageQueueSize(), - 2 * block + 2 * block - ); // new: 1 (x 2 blocks), garbage: 2 (x block) + 1 * block + 2 * block + ); // new: 1 (x block), garbage: 2 (x block) } tablet.CollectGarbage(); @@ -1076,7 +1088,7 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) { auto response = tablet.GetStorageStats(); const auto& stats = response->Record.GetStats(); - UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlocksCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlocksCount(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlobsCount(), 1); UNIT_ASSERT_VALUES_EQUAL(stats.GetGarbageQueueSize(), 0); // new: 0, garbage: 0 } @@ -1438,6 +1450,178 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) tablet.DestroyHandle(handle); } + TABLET_TEST_4K_ONLY(ShouldAutomaticallyRunCompactionDueToCompactionThresholdAverage) + { + const auto block = tabletConfig.BlockSize; + + NProto::TStorageConfig storageConfig; + storageConfig.SetNewCompactionEnabled(true); + storageConfig.SetCompactionThresholdAverage(2); + storageConfig.SetGarbageCompactionThresholdAverage(999'999); + storageConfig.SetCompactionThreshold(999'999); + storageConfig.SetCleanupThreshold(999'999); + storageConfig.SetWriteBlobThreshold(block); + + TTestEnv env({}, std::move(storageConfig)); + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + ui64 tabletId = env.BootIndexTablet(nodeIdx); + + TIndexTabletClient tablet( + env.GetRuntime(), + nodeIdx, + tabletId, + tabletConfig); + tablet.InitSession("client", "session"); + + auto id = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); + auto handle = CreateHandle(tablet, id); + + // allocating 4 compaction ranges + TSetNodeAttrArgs args(id); + args.SetFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE); + args.SetSize(1_MB); + tablet.SetNodeAttr(args); + + // 3 blobs in 3 different ranges + tablet.WriteData(handle, 0, block, 'a'); + tablet.WriteData(handle, 256_KB, block, 'a'); + tablet.WriteData(handle, 512_KB, block, 'a'); + + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL(3, stats.GetMixedBlobsCount()); + UNIT_ASSERT_VALUES_EQUAL(3, stats.GetMixedBlocksCount()); + UNIT_ASSERT_VALUES_EQUAL(1_MB / block, stats.GetUsedBlocksCount()); + UNIT_ASSERT_VALUES_EQUAL(3, stats.GetUsedCompactionRanges()); + // GroupSize (64) * ranges needed for our file (1_MB / 256_KB) + UNIT_ASSERT_VALUES_EQUAL(256, stats.GetAllocatedCompactionRanges()); + } + + // 3 more blobs in 3 different ranges + tablet.WriteData(handle, block, block, 'b'); + tablet.WriteData(handle, 256_KB + block, block, 'b'); + tablet.WriteData(handle, 512_KB + block, block, 'b'); + + // average CompactionScore per used range became 2 => Compaction + // should've been triggered for one of the ranges and should've + // decreased its blob count from 2 to 1 => we should have 5 blobs + // now + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL(5, stats.GetMixedBlobsCount()); + UNIT_ASSERT_VALUES_EQUAL(6, stats.GetMixedBlocksCount()); + } + + TString expected(1_MB, 0); + memset(expected.begin(), 'a', block); + memset(expected.begin() + block, 'b', block); + memset(expected.begin() + 256_KB, 'a', block); + memset(expected.begin() + 256_KB + block, 'b', block); + memset(expected.begin() + 512_KB, 'a', block); + memset(expected.begin() + 512_KB + block, 'b', block); + + { + auto response = tablet.ReadData(handle, 0, 1_MB); + const auto& buffer = response->Record.GetBuffer(); + UNIT_ASSERT_EQUAL(expected, buffer); + } + + tablet.DestroyHandle(handle); + } + + TABLET_TEST_4K_ONLY(ShouldAutomaticallyRunCompactionDueToGarbageCompactionThresholdAverage) + { + const auto block = tabletConfig.BlockSize; + + NProto::TStorageConfig storageConfig; + storageConfig.SetNewCompactionEnabled(true); + storageConfig.SetCompactionThresholdAverage(999'999); + storageConfig.SetGarbageCompactionThresholdAverage(20); + storageConfig.SetCompactionThreshold(999'999); + storageConfig.SetCleanupThreshold(999'999); + storageConfig.SetWriteBlobThreshold(block); + + TTestEnv env({}, std::move(storageConfig)); + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + ui64 tabletId = env.BootIndexTablet(nodeIdx); + + TIndexTabletClient tablet( + env.GetRuntime(), + nodeIdx, + tabletId, + tabletConfig); + tablet.InitSession("client", "session"); + + auto id = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); + auto handle = CreateHandle(tablet, id); + + // allocating 4 compaction ranges + TSetNodeAttrArgs args(id); + args.SetFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE); + args.SetSize(1_MB); + tablet.SetNodeAttr(args); + + // 768_KB in 3 different ranges + tablet.WriteData(handle, 0, 256_KB, 'a'); + tablet.WriteData(handle, 256_KB, 256_KB, 'a'); + tablet.WriteData(handle, 512_KB, 256_KB, 'a'); + + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL(3, stats.GetMixedBlobsCount()); + UNIT_ASSERT_VALUES_EQUAL(768_KB / block, stats.GetMixedBlocksCount()); + UNIT_ASSERT_VALUES_EQUAL(1_MB / block, stats.GetUsedBlocksCount()); + UNIT_ASSERT_VALUES_EQUAL(0, stats.GetGarbageBlocksCount()); + UNIT_ASSERT_VALUES_EQUAL(3, stats.GetUsedCompactionRanges()); + // GroupSize (64) * ranges needed for our file (1_MB / 256_KB) + UNIT_ASSERT_VALUES_EQUAL(256, stats.GetAllocatedCompactionRanges()); + } + + // 512_KB more + tablet.WriteData(handle, 0, 256_KB, 'b'); + tablet.WriteData(handle, 256_KB, 256_KB, 'b'); + + // garbage fraction became 1280_KB / 1024_KB > 1.2 => Compaction + // should've been triggered for one of the ranges and should've + // decreased its blob count from 2 to 1 => we should have 5 blobs + // byte count in that range should've been decreased from 512_KB to + // 256_KB + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL(4, stats.GetMixedBlobsCount()); + // now we have 1_MB of data + UNIT_ASSERT_VALUES_EQUAL( + 1_MB / block, + stats.GetMixedBlocksCount()); + // 0 garbage blocks + UNIT_ASSERT_VALUES_EQUAL(0, stats.GetGarbageBlocksCount()); + // 1280_KB new (written by user) + // 256_KB new (rewritten by Compaction) + // 512_KB garbage (marked after Compaction) + UNIT_ASSERT_VALUES_EQUAL(2_MB, stats.GetGarbageQueueSize()); + } + + TString expected(1_MB, 0); + memset(expected.begin(), 'b', 512_KB); + memset(expected.begin() + 512_KB, 'a', 256_KB); + + { + auto response = tablet.ReadData(handle, 0, 1_MB); + const auto& buffer = response->Record.GetBuffer(); + UNIT_ASSERT_EQUAL(expected, buffer); + } + + tablet.DestroyHandle(handle); + } + TABLET_TEST(ShouldAutomaticallyRunCleanup) { const auto block = tabletConfig.BlockSize; @@ -1508,6 +1692,93 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) tablet.DestroyHandle(handle); } + TABLET_TEST_4K_ONLY(ShouldAutomaticallyRunCleanupDueToCleanupThresholdAverage) + { + const auto block = tabletConfig.BlockSize; + + NProto::TStorageConfig storageConfig; + storageConfig.SetNewCleanupEnabled(true); + storageConfig.SetCompactionThreshold(999'999); + storageConfig.SetCleanupThreshold(999'999); + storageConfig.SetCleanupThresholdAverage(3); + storageConfig.SetWriteBlobThreshold(block); + + TTestEnv env({}, std::move(storageConfig)); + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + ui64 tabletId = env.BootIndexTablet(nodeIdx); + + TIndexTabletClient tablet( + env.GetRuntime(), + nodeIdx, + tabletId, + tabletConfig); + tablet.InitSession("client", "session"); + + auto id = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); + auto handle = CreateHandle(tablet, id); + + // allocating 4 compaction ranges + TSetNodeAttrArgs args(id); + args.SetFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE); + args.SetSize(1_MB); + tablet.SetNodeAttr(args); + + tablet.WriteData(handle, 0, 2 * block, 'a'); + tablet.WriteData(handle, 256_KB, 3 * block, 'b'); + tablet.WriteData(handle, 512_KB, 3 * block, 'c'); + tablet.WriteData(handle, 768_KB, 3 * block, 'd'); + + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlobsCount(), 4); + UNIT_ASSERT_VALUES_EQUAL(stats.GetDeletionMarkersCount(), 11); + } + + tablet.WriteData(handle, 2 * block, 2 * block, 'e'); + + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlobsCount(), 5); + UNIT_ASSERT_VALUES_EQUAL(stats.GetDeletionMarkersCount(), 9); + } + + { + auto response = tablet.ReadData(handle, 0, 2 * block); + const auto& buffer = response->Record.GetBuffer(); + UNIT_ASSERT(CompareBuffer(buffer, 2 * block, 'a')); + } + + { + auto response = tablet.ReadData(handle, 2 * block, 2 * block); + const auto& buffer = response->Record.GetBuffer(); + UNIT_ASSERT(CompareBuffer(buffer, 2 * block, 'e')); + } + + { + auto response = tablet.ReadData(handle, 256_KB, 3 * block); + const auto& buffer = response->Record.GetBuffer(); + UNIT_ASSERT(CompareBuffer(buffer, 3 * block, 'b')); + } + + { + auto response = tablet.ReadData(handle, 512_KB, 3 * block); + const auto& buffer = response->Record.GetBuffer(); + UNIT_ASSERT(CompareBuffer(buffer, 3 * block, 'c')); + } + + { + auto response = tablet.ReadData(handle, 768_KB, 3 * block); + const auto& buffer = response->Record.GetBuffer(); + UNIT_ASSERT(CompareBuffer(buffer, 3 * block, 'd')); + } + + tablet.DestroyHandle(handle); + } + TABLET_TEST(ShouldAutomaticallyRunCollectGarbage) { const auto block = tabletConfig.BlockSize; @@ -1746,6 +2017,7 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) storageConfig.SetWriteBlobThreshold(4 * block); TTestEnv env({}, std::move(storageConfig)); + auto registry = env.GetRegistry(); env.CreateSubDomain("nfs"); @@ -1887,6 +2159,20 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) UNIT_ASSERT_VALUES_EQUAL(1, channels.size()); UNIT_ASSERT_VALUES_EQUAL(3, channels.front()); + tablet.AdvanceTime(TDuration::Seconds(15)); + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(5)); + + TTestRegistryVisitor visitor; + // clang-format off + registry->Visit(TInstant::Zero(), visitor); + visitor.ValidateExpectedCounters({ + {{{"sensor", "ReassignCount"}, {"filesystem", "test"}}, 2}, + {{{"sensor", "WritableChannelCount"}, {"filesystem", "test"}}, 3}, + {{{"sensor", "UnwritableChannelCount"}, {"filesystem", "test"}}, 1}, + {{{"sensor", "ChannelsToMoveCount"}, {"filesystem", "test"}}, 1}, + }); + // clang-format on + tablet.DestroyHandle(handle); } @@ -3136,7 +3422,7 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) tablet.DestroyHandle(handle); } - TABLET_TEST(ShouldDeduplicateOutOfOrderCompactionMapChunkLoads) + TABLET_TEST(ShouldNotAllowTruncateDuringCompactionMapLoading) { const auto block = tabletConfig.BlockSize; @@ -3150,18 +3436,15 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) env.CreateSubDomain("nfs"); - TVector requests; + bool intercepted = false; TAutoPtr loadChunk; - env.GetRuntime().SetEventFilter([&] (auto& runtime, TAutoPtr& event) { + env.GetRuntime().SetEventFilter([&] (auto& runtime, auto& event) { Y_UNUSED(runtime); switch (event->GetTypeRewrite()) { case TEvIndexTabletPrivate::EvLoadCompactionMapChunkRequest: { - using TEv = - TEvIndexTabletPrivate::TEvLoadCompactionMapChunkRequest; - requests.push_back(*event->Get()); - - if (requests.size() == 1) { + if (!intercepted) { + intercepted = true; loadChunk = event.Release(); return true; } @@ -3182,61 +3465,242 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) tablet.InitSession("client", "session"); auto id = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); - auto handle = CreateHandle(tablet, id); - for (ui32 i = 0; i < 10; ++i) { - tablet.SendWriteDataRequest(handle, 0, block, 'a'); - { - auto response = tablet.RecvWriteDataResponse(); - UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); - } + // testing file size change - may trigger compaction map update via a + // truncate call + TSetNodeAttrArgs args(id); + args.SetFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE); + args.SetSize(4 * block); + tablet.SendSetNodeAttrRequest(args); + { + auto response = tablet.RecvSetNodeAttrResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); } - UNIT_ASSERT(loadChunk); - UNIT_ASSERT_VALUES_EQUAL(1, requests.size()); + // testing some other requests that can potentially trigger compaction + // map update + tablet.SendAllocateDataRequest(0, 0, 0, 0); + { + auto response = tablet.RecvAllocateDataResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + } - env.GetRuntime().Send(loadChunk.Release(), nodeIdx); - tablet.SendWriteDataRequest(handle, 0, block, 'a'); + tablet.SendDestroyCheckpointRequest("c"); { - auto response = tablet.RecvWriteDataResponse(); - UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + auto response = tablet.RecvDestroyCheckpointResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); } - UNIT_ASSERT_VALUES_EQUAL(2, requests.size()); - } + tablet.SendDestroyHandleRequest(0); + { + auto response = tablet.RecvDestroyHandleResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + } - TABLET_TEST(BackgroundOperationsShouldNotGetStuckForeverDuringCompactionMapLoading) - { - const auto block = tabletConfig.BlockSize; + tablet.SendDestroySessionRequest(0); + { + auto response = tablet.RecvDestroySessionResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + } - NProto::TStorageConfig storageConfig; - // hard to test anything apart from Compaction - it shares - // EOperationState with Cleanup and FlushBytes - storageConfig.SetCompactionThreshold(2); - // Flush has a separate EOperationState - storageConfig.SetFlushThreshold(1); - storageConfig.SetLoadedCompactionRangesPerTx(2); - storageConfig.SetWriteBlobThreshold(2 * block); + tablet.SendTruncateRequest(0, 0); + { + auto response = tablet.RecvTruncateResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + } - TTestEnv env({}, std::move(storageConfig)); + tablet.SendUnlinkNodeRequest(0, "", false); + { + auto response = tablet.RecvUnlinkNodeResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + } - env.CreateSubDomain("nfs"); + tablet.SendZeroRangeRequest(0, 0, 0); + { + auto response = tablet.RecvZeroRangeResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + } - ui32 nodeIdx = env.CreateNode("nfs"); - ui64 tabletId = env.BootIndexTablet(nodeIdx); + // checking that our state wasn't changed + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlocksCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlobsCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(stats.GetDeletionMarkersCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(stats.GetUsedBlocksCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(stats.GetUsedCompactionRanges(), 0); + } - TIndexTabletClient tablet( - env.GetRuntime(), - nodeIdx, - tabletId, - tabletConfig); - tablet.InitSession("client", "session"); + UNIT_ASSERT(loadChunk); - auto id = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); - auto handle = CreateHandle(tablet, id); + env.GetRuntime().Send(loadChunk.Release(), nodeIdx); - // generating at least one compaction range - tablet.WriteData(handle, 0, block, 'a'); + // compaction map should be loaded now - SetNodeAttr should succeed + tablet.SendSetNodeAttrRequest(args); + { + auto response = tablet.RecvSetNodeAttrResponse(); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + } + + // we have some used blocks now + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlocksCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlobsCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(stats.GetDeletionMarkersCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(stats.GetUsedBlocksCount(), 4); + UNIT_ASSERT_VALUES_EQUAL(stats.GetUsedCompactionRanges(), 0); + } + + intercepted = false; + + tablet.RebootTablet(); + tablet.RecoverSession(); + + // truncate to 0 size should not succeed before compaction map gets + // loaded as well + args.SetSize(0); + tablet.SendSetNodeAttrRequest(args); + { + auto response = tablet.RecvSetNodeAttrResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + } + + // state not changed + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlocksCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlobsCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(stats.GetDeletionMarkersCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(stats.GetUsedBlocksCount(), 4); + UNIT_ASSERT_VALUES_EQUAL(stats.GetUsedCompactionRanges(), 0); + } + + env.GetRuntime().Send(loadChunk.Release(), nodeIdx); + // compaction map loaded - truncate request allowed + tablet.SendSetNodeAttrRequest(args); + { + auto response = tablet.RecvSetNodeAttrResponse(); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + } + + // we should see some deletion markers now + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlocksCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(stats.GetMixedBlobsCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(stats.GetDeletionMarkersCount(), 4); + UNIT_ASSERT_VALUES_EQUAL(stats.GetUsedBlocksCount(), 0); + // we have some deletion markers in one of the ranges now + UNIT_ASSERT_VALUES_EQUAL(stats.GetUsedCompactionRanges(), 1); + } + } + + TABLET_TEST(ShouldDeduplicateOutOfOrderCompactionMapChunkLoads) + { + const auto block = tabletConfig.BlockSize; + + NProto::TStorageConfig storageConfig; + storageConfig.SetCompactionThreshold(999'999); + storageConfig.SetCleanupThreshold(999'999); + storageConfig.SetLoadedCompactionRangesPerTx(2); + storageConfig.SetWriteBlobThreshold(block); + + TTestEnv env({}, std::move(storageConfig)); + + env.CreateSubDomain("nfs"); + + TVector requests; + TAutoPtr loadChunk; + env.GetRuntime().SetEventFilter([&] (auto& runtime, TAutoPtr& event) { + Y_UNUSED(runtime); + + switch (event->GetTypeRewrite()) { + case TEvIndexTabletPrivate::EvLoadCompactionMapChunkRequest: { + using TEv = + TEvIndexTabletPrivate::TEvLoadCompactionMapChunkRequest; + requests.push_back(*event->Get()); + + if (requests.size() == 1) { + loadChunk = event.Release(); + return true; + } + } + } + + return false; + }); + + ui32 nodeIdx = env.CreateNode("nfs"); + ui64 tabletId = env.BootIndexTablet(nodeIdx); + + TIndexTabletClient tablet( + env.GetRuntime(), + nodeIdx, + tabletId, + tabletConfig); + tablet.InitSession("client", "session"); + + auto id = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); + auto handle = CreateHandle(tablet, id); + + for (ui32 i = 0; i < 10; ++i) { + tablet.SendWriteDataRequest(handle, 0, block, 'a'); + { + auto response = tablet.RecvWriteDataResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); + } + } + + UNIT_ASSERT(loadChunk); + UNIT_ASSERT_VALUES_EQUAL(1, requests.size()); + + env.GetRuntime().Send(loadChunk.Release(), nodeIdx); + tablet.SendWriteDataRequest(handle, 0, block, 'a'); + { + auto response = tablet.RecvWriteDataResponse(); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + } + + UNIT_ASSERT_VALUES_EQUAL(2, requests.size()); + } + + TABLET_TEST(BackgroundOperationsShouldNotGetStuckForeverDuringCompactionMapLoading) + { + const auto block = tabletConfig.BlockSize; + + NProto::TStorageConfig storageConfig; + // hard to test anything apart from Compaction - it shares + // EOperationState with Cleanup and FlushBytes + storageConfig.SetCompactionThreshold(2); + // Flush has a separate EOperationState + storageConfig.SetFlushThreshold(1); + storageConfig.SetLoadedCompactionRangesPerTx(2); + storageConfig.SetWriteBlobThreshold(2 * block); + + TTestEnv env({}, std::move(storageConfig)); + + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + ui64 tabletId = env.BootIndexTablet(nodeIdx); + + TIndexTabletClient tablet( + env.GetRuntime(), + nodeIdx, + tabletId, + tabletConfig); + tablet.InitSession("client", "session"); + + auto id = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); + auto handle = CreateHandle(tablet, id); + + // generating at least one compaction range + tablet.WriteData(handle, 0, block, 'a'); TAutoPtr loadChunk; ui32 loadChunkCount = 0; @@ -3322,6 +3786,9 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) static_cast(stats.GetBlobIndexOpState())); } + env.GetRuntime().Send(loadChunk.Release(), nodeIdx); + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(1)); + tablet.DestroyHandle(handle); } @@ -3415,6 +3882,9 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) static_cast(stats.GetFlushState())); } + env.GetRuntime().Send(loadChunk.Release(), nodeIdx); + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(1)); + tablet.DestroyHandle(handle); } @@ -3763,9 +4233,11 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) return false; }); - env.GetRuntime().DispatchEvents(TDispatchOptions(), TDuration::Seconds(1)); - - UNIT_ASSERT(putSeen); + TDispatchOptions options; + options.CustomFinalCondition = [&] { + return putSeen; + }; + env.GetRuntime().DispatchEvents(options); tablet.RebootTablet(); @@ -3774,6 +4246,10 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) E_REJECTED, response->GetStatus(), response->GetErrorReason()); + + UNIT_ASSERT_VALUES_EQUAL( + "tablet is shutting down", + response->GetErrorReason()); } TABLET_TEST(ShouldCancelWriteRequestsIfTabletIsRebooted) @@ -3826,9 +4302,12 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) return false; }); - env.GetRuntime().DispatchEvents(TDispatchOptions(), TDuration::Seconds(1)); + TDispatchOptions options; + options.CustomFinalCondition = [&] { + return getSeen; + }; - UNIT_ASSERT(getSeen); + env.GetRuntime().DispatchEvents(options); failGet = false; tablet.RebootTablet(); @@ -3838,6 +4317,430 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) E_REJECTED, response->GetStatus(), response->GetErrorReason()); + + UNIT_ASSERT_VALUES_EQUAL( + "tablet is shutting down", + response->GetErrorReason()); + } + +#define CHECK_GENERATED_BLOB(offset, length, expected) \ + { \ + ui64 currentOffset = offset; \ + TVector expectedSizes = expected; \ + auto blobs = tablet.GenerateBlobIds(node, handle, offset, length); \ + auto commitId = blobs->Record.GetCommitId(); \ + const auto [generation, step] = ParseCommitId(commitId); \ + UNIT_ASSERT_VALUES_EQUAL( \ + blobs->Record.BlobsSize(), \ + expectedSizes.size()); \ + for (size_t i = 0; i < expectedSizes.size(); ++i) { \ + auto generatedBlob = blobs->Record.GetBlobs(i); \ + auto blob = LogoBlobIDFromLogoBlobID(generatedBlob.GetBlobId()); \ + UNIT_ASSERT_VALUES_EQUAL(blob.BlobSize(), expectedSizes[i]); \ + UNIT_ASSERT_VALUES_EQUAL(blob.Generation(), generation); \ + UNIT_ASSERT_VALUES_EQUAL(blob.Step(), step); \ + UNIT_ASSERT_VALUES_EQUAL( \ + generatedBlob.GetOffset(), \ + currentOffset); \ + currentOffset += expectedSizes[i]; \ + } \ + } + + TABLET_TEST(ShouldGenerateBlobIds) + { + auto block = tabletConfig.BlockSize; + + TTestEnv env; + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + ui64 tabletId = env.BootIndexTablet(nodeIdx); + + TIndexTabletClient tablet( + env.GetRuntime(), + nodeIdx, + tabletId, + tabletConfig); + tablet.InitSession("client", "session"); + + auto node = + CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); + ui64 handle = CreateHandle(tablet, node); + + CHECK_GENERATED_BLOB(0, block, TVector{block}); + CHECK_GENERATED_BLOB(block, block, TVector{block}); + CHECK_GENERATED_BLOB(3 * block, 2 * block, TVector{2 * block}); + CHECK_GENERATED_BLOB( + BlockGroupSize * block / 2, + block * BlockGroupSize, + (TVector{ + BlockGroupSize * block / 2, + BlockGroupSize * block / 2})); + CHECK_GENERATED_BLOB( + 3 * block * BlockGroupSize, + 3 * block * BlockGroupSize, + (TVector{ + block * BlockGroupSize, + block * BlockGroupSize, + block * BlockGroupSize})); + CHECK_GENERATED_BLOB( + block, + 2 * block * BlockGroupSize, + (TVector{ + block * (BlockGroupSize - 1), + block * BlockGroupSize, + block})); + } + +#undef CHECK_GENERATED_BLOB + + TABLET_TEST(ShouldAcquireLockForCollectGarbageOnGenerateBlobIds) + { + auto block = tabletConfig.BlockSize; + + TTestEnv env; + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + ui64 tabletId = env.BootIndexTablet(nodeIdx); + + TIndexTabletClient tablet( + env.GetRuntime(), + nodeIdx, + tabletId, + tabletConfig); + tablet.InitSession("client", "session"); + + auto moveBarrier = [&tablet, block] + { + auto node = + CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); + ui64 handle = CreateHandle(tablet, node); + tablet.WriteData(handle, 0, block, 'a'); + tablet.Flush(); + tablet.DestroyHandle(handle); + tablet.UnlinkNode(RootNodeId, "test", false); + tablet.CollectGarbage(); + }; + moveBarrier(); + + ui64 lastCollectGarbage = 0; + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + lastCollectGarbage = stats.GetLastCollectCommitId(); + } + UNIT_ASSERT_GT(lastCollectGarbage, 0); + + auto node = + CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test2")); + ui64 handle = CreateHandle(tablet, node); + + auto blobs = tablet.GenerateBlobIds(node, handle, 0, block); + auto commitId = blobs->Record.GetCommitId(); + + moveBarrier(); + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_LE(stats.GetLastCollectCommitId(), commitId); + } + + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(15)); + + moveBarrier(); + // After the GenerateBlobIdsReleaseCollectBarrierTimeout has passed, we + // can observe that the last collect garbage has moved beyond the commit + // id of the generated blob. + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + UNIT_ASSERT_GT(stats.GetLastCollectCommitId(), commitId); + } + + // Now we validate that the barrier is released even if the TX fails + TVector blobIds; + + NProto::TError error; + error.SetCode(E_REJECTED); + + auto filter = [&](auto& runtime, auto& event) + { + Y_UNUSED(runtime); + + switch (event->GetTypeRewrite()) { + case TEvBlobStorage::EvPutResult: { + using TResponse = TEvBlobStorage::TEvPutResult; + auto* msg = event->template Get(); + if (msg->Id.Channel() >= TIndexTabletSchema::DataChannel) { + blobIds.push_back(msg->Id); + } + return false; + } + case TEvIndexTabletPrivate::EvWriteBlobResponse: { + using TResponse = + TEvIndexTabletPrivate::TEvWriteBlobResponse; + auto* msg = event->template Get(); + auto& e = const_cast(msg->Error); + e.SetCode(E_REJECTED); + return false; + } + } + + return false; + }; + + env.GetRuntime().SetEventFilter(filter); + + auto generateResult = + tablet.GenerateBlobIds(node, handle, 0, block * BlockGroupSize); + commitId = generateResult->Record.GetCommitId(); + + // intercepted blob was successfully written to BlobStorage, yet the + // following operation is expected to fail + tablet.AssertWriteDataFailed(handle, 0, block * BlockGroupSize, 'x'); + + env.GetRuntime().SetEventFilter( + TTestActorRuntimeBase::DefaultFilterFunc); + + // because we use handle + 1 instead of handle, it is expected that the + // handler will fail will fail + tablet.AssertAddDataFailed( + node, + handle + 1, + 0, + block * BlockGroupSize, + blobIds, + commitId); + + moveBarrier(); + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + // We expect that upon failre the barrier was released, thus moving + // the last collect garbage beyond the commit id of the issued blob + UNIT_ASSERT_GT(stats.GetLastCollectCommitId(), commitId); + } + + node = + CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test3")); + handle = CreateHandle(tablet, node, "", TCreateHandleArgs::RDNLY); + + + // now we do the same thing, but expect HandleAddData to execute tx, yet + // the tx to fail + generateResult = + tablet.GenerateBlobIds(node, handle, 0, block * BlockGroupSize); + commitId = generateResult->Record.GetCommitId(); + + tablet.AssertAddDataFailed( + node, + handle, + 0, + block * BlockGroupSize, + blobIds, + commitId); + + moveBarrier(); + { + auto response = tablet.GetStorageStats(); + const auto& stats = response->Record.GetStats(); + // We expect that upon failre the barrier was released, thus moving + // the last collect garbage beyond the commit id of the issued blob + UNIT_ASSERT_GT(stats.GetLastCollectCommitId(), commitId); + } + } + + TABLET_TEST(ShouldAddData) + { + const auto block = tabletConfig.BlockSize; + + NProto::TStorageConfig storageConfig; + storageConfig.SetCompactionThreshold(999'999); + storageConfig.SetCleanupThreshold(999'999); + storageConfig.SetWriteBlobThreshold(block); + + TTestEnv env({}, std::move(storageConfig)); + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + ui64 tabletId = env.BootIndexTablet(nodeIdx); + + TIndexTabletClient tablet( + env.GetRuntime(), + nodeIdx, + tabletId, + tabletConfig); + tablet.InitSession("client", "session"); + + auto id = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); + auto handle = CreateHandle(tablet, id); + + TVector blobIds; + bool shouldDropPutResult = true; + + // We can't make direct writes to BlobStorage, so we store the blob ids + // from an ordinary write and then use them in AddData + env.GetRuntime().SetObserverFunc( + [&](TAutoPtr& event) + { + switch (event->GetTypeRewrite()) { + case TEvBlobStorage::EvPutResult: { + // We intercept all PutResult events in order for tablet + // to consider them as written. Nevertheless, these + // blobs are already written and we will use them in + // AddData + auto* msg = event->Get(); + if (msg->Id.Channel() >= + TIndexTabletSchema::DataChannel) { + blobIds.push_back(msg->Id); + } + if (shouldDropPutResult) { + return TTestActorRuntime::EEventAction::DROP; + } + break; + } + } + + return TTestActorRuntime::DefaultObserverFunc(event); + }); + + TString data(block * BlockGroupSize * 2, '\0'); + for (size_t i = 0; i < data.size(); ++i) { + // 77 and 256 are coprimes + data[i] = static_cast(i % 77); + } + + tablet.SendWriteDataRequest(handle, 0, data.size(), data.data()); + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(blobIds.size(), 2); + shouldDropPutResult = false; + + // We acquire commitId just so there is something to release on + // completion + auto commitId = tablet.GenerateBlobIds(id, handle, 0, block) + ->Record.GetCommitId(); + + auto id2 = + CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test2")); + auto handle2 = CreateHandle(tablet, id2); + + Sort(blobIds.begin(), blobIds.end()); + + // Now we try to submit the same blobs for another node + auto request = tablet.CreateAddDataRequest( + id2, + handle2, + 0, + data.size(), + blobIds, + commitId); + + tablet.SendRequest(std::move(request)); + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(2)); + + auto readData = tablet.ReadData(handle2, 0, data.size()); + + // After AddData, we should receive AddDataResponse + auto response = tablet.RecvAddDataResponse(); + UNIT_ASSERT_VALUES_EQUAL(S_OK, response->GetStatus()); + + // Use DescribeData to check that proper blobs were added + auto describe = tablet.DescribeData(handle2, 0, data.size()); + UNIT_ASSERT_VALUES_EQUAL(describe->Record.BlobPiecesSize(), 2); + for (auto [i, blobPiece]: Enumerate(describe->Record.GetBlobPieces())) { + UNIT_ASSERT_VALUES_EQUAL(1, blobPiece.RangesSize()); + UNIT_ASSERT_VALUES_EQUAL( + i * (block * BlockGroupSize), + blobPiece.GetRanges(0).GetOffset()); + UNIT_ASSERT_VALUES_EQUAL(0, blobPiece.GetRanges(0).GetBlobOffset()); + UNIT_ASSERT_VALUES_EQUAL( + block * BlockGroupSize, + blobPiece.GetRanges(0).GetLength()); + + auto blobId = LogoBlobIDFromLogoBlobID(blobPiece.GetBlobId()); + UNIT_ASSERT_VALUES_EQUAL(blobId, blobIds[i]); + } + + // validate, that no more BlobStorage requests were made + UNIT_ASSERT_VALUES_EQUAL(blobIds.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(data, readData->Record.GetBuffer()); + } + + TABLET_TEST(ShouldRejectAddDataIfCollectBarrierIsAlreadyReleased) + { + const auto block = tabletConfig.BlockSize; + + NProto::TStorageConfig storageConfig; + storageConfig.SetCompactionThreshold(999'999); + storageConfig.SetCleanupThreshold(999'999); + storageConfig.SetWriteBlobThreshold(block); + + TTestEnv env({}, std::move(storageConfig)); + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + ui64 tabletId = env.BootIndexTablet(nodeIdx); + + TIndexTabletClient tablet( + env.GetRuntime(), + nodeIdx, + tabletId, + tabletConfig); + tablet.InitSession("client", "session"); + + auto id = CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); + auto handle = CreateHandle(tablet, id); + + TVector blobIds; + + env.GetRuntime().SetObserverFunc( + [&](TAutoPtr& event) + { + switch (event->GetTypeRewrite()) { + case TEvBlobStorage::EvPutResult: { + auto* msg = event->Get(); + if (msg->Id.Channel() >= + TIndexTabletSchema::DataChannel) { + blobIds.push_back(msg->Id); + return TTestActorRuntime::EEventAction::DROP; + } + break; + } + } + + return TTestActorRuntime::DefaultObserverFunc(event); + }); + + TString data(block * BlockGroupSize * 2, 'x'); + + tablet.SendWriteDataRequest(handle, 0, data.size(), data.data()); + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(blobIds.size(), 2); + + auto commitId = tablet.GenerateBlobIds(id, handle, 0, block) + ->Record.GetCommitId(); + + // We wait for the collect barrier lease to expire. We expect that the + // following AddData request will be rejected + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(15)); + + auto id2 = + CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test2")); + auto handle2 = CreateHandle(tablet, id2); + + Sort(blobIds.begin(), blobIds.end()); + + tablet.SendAddDataRequest( + id2, + handle2, + 0, + data.size(), + blobIds, + commitId); + + auto response = tablet.RecvAddDataResponse(); + UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); } #undef TABLET_TEST diff --git a/cloud/filestore/libs/storage/tablet/ya.make b/cloud/filestore/libs/storage/tablet/ya.make index fd9ba4456bb..765e3080cab 100644 --- a/cloud/filestore/libs/storage/tablet/ya.make +++ b/cloud/filestore/libs/storage/tablet/ya.make @@ -15,6 +15,7 @@ SRCS( tablet_actor_accessnode.cpp tablet_actor_acquirelock.cpp tablet_actor_addblob.cpp + tablet_actor_adddata.cpp tablet_actor_allocatedata.cpp tablet_actor_change_storage_config.cpp tablet_actor_cleanup.cpp @@ -87,6 +88,7 @@ PEERDIR( cloud/filestore/libs/storage/api cloud/filestore/libs/storage/core cloud/filestore/libs/storage/model + cloud/filestore/libs/storage/tablet/actors cloud/filestore/libs/storage/tablet/model cloud/filestore/libs/storage/tablet/protos @@ -95,6 +97,7 @@ PEERDIR( cloud/storage/core/libs/diagnostics cloud/storage/core/libs/tablet cloud/storage/core/libs/tablet/model + cloud/storage/core/libs/viewer cloud/storage/core/protos contrib/ydb/library/actors/core diff --git a/cloud/filestore/libs/storage/testlib/tablet_client.h b/cloud/filestore/libs/storage/testlib/tablet_client.h index 9c60fb51f5c..bcc61ea0d13 100644 --- a/cloud/filestore/libs/storage/testlib/tablet_client.h +++ b/cloud/filestore/libs/storage/testlib/tablet_client.h @@ -557,6 +557,44 @@ class TIndexTabletClient return request; } + auto CreateGenerateBlobIdsRequest( + ui64 nodeId, + ui64 handle, + ui64 offset, + ui64 length) + { + auto request = + CreateSessionRequest(); + request->Record.SetNodeId(nodeId); + request->Record.SetHandle(handle); + request->Record.SetOffset(offset); + request->Record.SetLength(length); + return request; + } + + auto CreateAddDataRequest( + ui64 nodeId, + ui64 handle, + ui64 offset, + ui32 length, + const TVector& blobIds, + ui64 commitId) + { + auto request = CreateSessionRequest< + TEvIndexTablet::TEvAddDataRequest>(); + request->Record.SetNodeId(nodeId); + request->Record.SetHandle(handle); + request->Record.SetOffset(offset); + request->Record.SetLength(length); + for (const auto& blobId: blobIds) { + NKikimr::LogoBlobIDFromLogoBlobID( + blobId, + request->Record.MutableBlobIds()->Add()); + } + request->Record.SetCommitId(commitId); + return request; + } + auto CreateAcquireLockRequest( ui64 handle, ui64 owner, diff --git a/cloud/filestore/libs/storage/testlib/test_env.cpp b/cloud/filestore/libs/storage/testlib/test_env.cpp index 4595936ef52..475aa8cc94c 100644 --- a/cloud/filestore/libs/storage/testlib/test_env.cpp +++ b/cloud/filestore/libs/storage/testlib/test_env.cpp @@ -711,6 +711,107 @@ void TTestEnv::WaitForSchemeShardTx(ui64 txId) //////////////////////////////////////////////////////////////////////////////// +void TTestRegistryVisitor::OnStreamBegin() +{ + CurrentEntry = TMetricsEntry(); + MetricsEntries.clear(); +} + +void TTestRegistryVisitor::OnStreamEnd() +{} + +void TTestRegistryVisitor::OnMetricBegin( + TInstant time, + NMetrics::EAggregationType aggrType, + NMetrics::EMetricType metrType) +{ + CurrentEntry.Time = time; + CurrentEntry.AggrType = aggrType; + CurrentEntry.MetrType = metrType; +} + +void TTestRegistryVisitor::OnMetricEnd() +{ + MetricsEntries.emplace_back(std::move(CurrentEntry)); +} + +void TTestRegistryVisitor::OnLabelsBegin() +{} + +void TTestRegistryVisitor::OnLabelsEnd() +{} + +void TTestRegistryVisitor::OnLabel(TStringBuf name, TStringBuf value) +{ + CurrentEntry.Labels.emplace(TString(name), TString(value)); +} + +void TTestRegistryVisitor::OnValue(i64 value) +{ + CurrentEntry.Value = value; +} + +const TVector& TTestRegistryVisitor::GetEntries() const +{ + return MetricsEntries; +} + +void TTestRegistryVisitor::ValidateExpectedCounters( + const TVector, i64>>& expectedCounters) +{ + for (const auto& [labels, value]: expectedCounters) { + const auto labelsStr = LabelsToString(labels); + + int matchingCountersCount = 0; + for (const auto& entry: MetricsEntries) { + if (entry.Matches(labels)) { + ++matchingCountersCount; + UNIT_ASSERT_VALUES_EQUAL_C(entry.Value, value, labelsStr); + } + } + UNIT_ASSERT_VALUES_EQUAL_C(matchingCountersCount, 1, labelsStr); + } +} + +void TTestRegistryVisitor::ValidateExpectedHistogram( + const TVector, i64>>& expectedCounters, + bool checkEqual) +{ + for (const auto& [labels, value]: expectedCounters) { + const auto labelsStr = LabelsToString(labels); + i64 total = 0; + + int matchingCountersCount = 0; + for (const auto& entry: MetricsEntries) { + if (entry.Matches(labels)) { + ++matchingCountersCount; + total += entry.Value; + } + } + if (checkEqual) { + UNIT_ASSERT_VALUES_EQUAL_C(total, value, labelsStr); + } else { + UNIT_ASSERT_VALUES_UNEQUAL_C(total, value, labelsStr); + } + UNIT_ASSERT_VALUES_UNEQUAL_C(matchingCountersCount, 0, labelsStr); + } +} + +TString TTestRegistryVisitor::LabelsToString(const TVector& labels) +{ + TStringBuilder labelsStr; + for (const auto& label: labels) { + if (labelsStr) { + labelsStr << ", "; + } + labelsStr << label.GetName() << "=" << label.GetValue(); + } + + return labelsStr; +} + +//////////////////////////////////////////////////////////////////////////////// + void CheckForkJoin(const NLWTrace::TShuttleTrace& trace, bool forkRequired) { UNIT_ASSERT(trace.GetEvents().size() > 0); diff --git a/cloud/filestore/libs/storage/testlib/test_env.h b/cloud/filestore/libs/storage/testlib/test_env.h index f0c44ee6b26..58d08613820 100644 --- a/cloud/filestore/libs/storage/testlib/test_env.h +++ b/cloud/filestore/libs/storage/testlib/test_env.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include #include #include @@ -153,4 +155,61 @@ class TTestEnv void WaitForSchemeShardTx(ui64 txId); }; +//////////////////////////////////////////////////////////////////////////////// + +class TTestRegistryVisitor + : public NMetrics::IRegistryVisitor +{ +public: + using TLabel = NCloud::NFileStore::NMetrics::TLabel; + +private: + struct TMetricsEntry + { + TInstant Time; + NMetrics::EAggregationType AggrType; + NMetrics::EMetricType MetrType; + THashMap Labels; + i64 Value; + + bool Matches(const TVector& labels) const + { + for (auto& label: labels) { + auto it = Labels.find(label.GetName()); + if (it == Labels.end() || it->second != label.GetValue()) { + return false; + } + } + return true; + } + }; + + TVector MetricsEntries; + TMetricsEntry CurrentEntry; + +public: + void OnStreamBegin() override; + void OnStreamEnd() override; + void OnMetricBegin( + TInstant time, + NMetrics::EAggregationType aggrType, + NMetrics::EMetricType metrType) override; + void OnMetricEnd() override; + void OnLabelsBegin() override; + void OnLabelsEnd() override; + void OnLabel(TStringBuf name, TStringBuf value) override; + void OnValue(i64 value) override; + +public: + const TVector& GetEntries() const; + void ValidateExpectedCounters( + const TVector, i64>>& expectedCounters); + void ValidateExpectedHistogram( + const TVector, i64>>& expectedCounters, + bool checkEqual); + +private: + static TString LabelsToString(const TVector& labels); +}; + } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/vfs_fuse/cache.cpp b/cloud/filestore/libs/vfs_fuse/cache.cpp index d64bdd34d46..d9253c17a5f 100644 --- a/cloud/filestore/libs/vfs_fuse/cache.cpp +++ b/cloud/filestore/libs/vfs_fuse/cache.cpp @@ -37,7 +37,7 @@ void TCache::ForgetNode(ui64 ino, size_t count) if (it == Nodes.end()) { // we lose our cache after restart, so we should expect forget requests // targeting nodes that are absent from our cache - // see https://st.yandex-team.ru/NBS-2102 + // see NBS-2102 return; } diff --git a/cloud/filestore/libs/vfs_fuse/cache.h b/cloud/filestore/libs/vfs_fuse/cache.h index 9e6a175cdf6..4b391a303e3 100644 --- a/cloud/filestore/libs/vfs_fuse/cache.h +++ b/cloud/filestore/libs/vfs_fuse/cache.h @@ -37,7 +37,7 @@ struct TNode { // we lose actual RefCount values after restart, so we should expect // forget requests that try to subtract values greater than RefCount - // see https://st.yandex-team.ru/NBS-2102 + // see NBS-2102 RefCount -= Min(RefCount, count); return RefCount; } diff --git a/cloud/filestore/libs/vfs_fuse/tsan.supp b/cloud/filestore/libs/vfs_fuse/tsan.supp new file mode 100644 index 00000000000..ab6c0b23fb0 --- /dev/null +++ b/cloud/filestore/libs/vfs_fuse/tsan.supp @@ -0,0 +1,2 @@ +# There is a race between fuse_session_loop & fuse_session_exit +race:fuse/lib/fuse_session.c diff --git a/cloud/filestore/private/api/protos/actions.proto b/cloud/filestore/private/api/protos/actions.proto index 265af910362..e6619efa335 100644 --- a/cloud/filestore/private/api/protos/actions.proto +++ b/cloud/filestore/private/api/protos/actions.proto @@ -5,7 +5,7 @@ package NCloud.NFileStore.NProtoPrivate; option go_package = "github.com/ydb-platform/nbs/cloud/filestore/private/api/protos"; //////////////////////////////////////////////////////////////////////////////// -// Actions +// DrainNode message TDrainNodeRequest { @@ -15,3 +15,16 @@ message TDrainNodeRequest message TDrainNodeResponse { } + +//////////////////////////////////////////////////////////////////////////////// +// ReassignTablet + +message TReassignTabletRequest +{ + uint64 TabletId = 1; + repeated uint32 Channels = 2; +} + +message TReassignTabletResponse +{ +} diff --git a/cloud/filestore/private/api/protos/tablet.proto b/cloud/filestore/private/api/protos/tablet.proto index 35d8efc8960..8efded8ebca 100644 --- a/cloud/filestore/private/api/protos/tablet.proto +++ b/cloud/filestore/private/api/protos/tablet.proto @@ -117,6 +117,7 @@ message TStorageStats uint64 FreshBytesCount = 109; uint64 AllocatedCompactionRanges = 110; uint64 UsedCompactionRanges = 111; + uint64 LastCollectCommitId = 112; // channel stats uint64 TabletChannelCount = 1000; @@ -368,3 +369,88 @@ message TDescribeSessionsResponse // All tablet sessions. repeated TTabletSessionInfo Sessions = 2; } + +//////////////////////////////////////////////////////////////////////////////// +// GenerateBlobIds request/response. + +message TGenerateBlobIdsRequest +{ + // Optional request headers. + NProto.THeaders Headers = 1; + + // FileSystem identifier. + string FileSystemId = 2; + + // Node. + uint64 NodeId = 3; + + // IO handle. + uint64 Handle = 4; + + // Starting offset for write. Expected to be aligned to the block size. + uint64 Offset = 5; + + // Length of data to write. Expected to be aligned to the block size. + uint64 Length = 6; +} + +message TGeneratedBlob +{ + // Blob id. + NKikimrProto.TLogoBlobID BlobId = 1; + + // Offset + uint64 Offset = 2; + + // Group id. + uint32 BSGroupId = 3; +} + +message TGenerateBlobIdsResponse +{ + // Optional error, set only if error happened. + NCloud.NProto.TError Error = 1; + + // Blob ids, in the same order as in the request. + repeated TGeneratedBlob Blobs = 2; + + // AcquireCollectBarrier has been executed for this commit id. + uint64 CommitId = 4; +} + +//////////////////////////////////////////////////////////////////////////////// + +// ThreeStageWrite request/response. + +message TAddDataRequest +{ + // Optional request headers. + NProto.THeaders Headers = 1; + + // FileSystem identifier. + string FileSystemId = 2; + + // Node. + uint64 NodeId = 3; + + // IO handle. + uint64 Handle = 4; + + // Starting offset for write. + uint64 Offset = 5; + + // Data size. + uint64 Length = 6; + + // Blob ids to be added. Ordered by the offset in the original data. + repeated NKikimrProto.TLogoBlobID BlobIds = 7; + + // Commit id. + uint64 CommitId = 8; +} + +message TAddDataResponse +{ + // Optional error, set only if error happened. + NCloud.NProto.TError Error = 1; +} diff --git a/cloud/filestore/public/api/protos/fs.proto b/cloud/filestore/public/api/protos/fs.proto index ad472c2b515..728df48f9e2 100644 --- a/cloud/filestore/public/api/protos/fs.proto +++ b/cloud/filestore/public/api/protos/fs.proto @@ -17,6 +17,7 @@ message TFileStoreFeatures uint32 EntryTimeout = 2; uint32 NegativeEntryTimeout = 3; uint32 AttrTimeout = 4; + bool ThreeStageWriteEnabled = 5; } message TFileStore diff --git a/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh b/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh index 98c6b026190..1c46a48766f 100644 --- a/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh +++ b/cloud/filestore/tests/build_arcadia_test/scripts/coreutils.sh @@ -5,7 +5,7 @@ sudo apt -y install python3 python-is-python3 git build-essential autoconf autom cd $mountPath if [ "$cloneOriginalRepo" == true ]; then - # If cloneOriginalRepo is set to true, fetch stable version from original repo + # If cloneOriginalRepo is set to true, fetch stable version from original repo git clone --depth=1 --branch v9.3 --single-branch https://git.savannah.gnu.org/git/coreutils.git cd coreutils ./bootstrap @@ -13,7 +13,7 @@ else # If cloneOriginalRepo is set to false, fetch uploaded version to sanbox (for isolation and reproducibility purposes) # Download latest published resource with matching attribute - wget 'https://proxy.sandbox.yandex-team.ru/last/OTHER_RESOURCE?attrs={"name":"nbs_coreutils.tar"}' -O arch.tar + wget 'https://link/to/nbs_coreutils.tar' -O arch.tar tar xvf arch.tar cd coreutils git config --global --add safe.directory $$(pwd) diff --git a/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/nfs-storage.txt b/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/nfs-storage.txt index 0e9df52efd0..14fae8613da 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/nfs-storage.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/nfs-storage.txt @@ -1 +1,3 @@ TwoStageReadEnabled: true +NewCompactionEnabled: true +NewCleanupEnabled: true diff --git a/cloud/filestore/tests/recipes/large.inc b/cloud/filestore/tests/recipes/large.inc index 6c3a26ae9b4..d9c4b8f711f 100644 --- a/cloud/filestore/tests/recipes/large.inc +++ b/cloud/filestore/tests/recipes/large.inc @@ -2,7 +2,7 @@ SIZE(LARGE) TIMEOUT(3600) # use ya:not_autocheck and ya:manual with sanitizers until we have sufficient -# quota in the YC_NBS group: https://sandbox.yandex-team.ru/admin/groups/YC_NBS/general +# quota in the YC_NBS group IF (SANITIZER_TYPE) TAG( ya:fat diff --git a/cloud/filestore/tests/xfs_suite/README.md b/cloud/filestore/tests/xfs_suite/README.md index 31cd81fe967..bb64e379dfe 100644 --- a/cloud/filestore/tests/xfs_suite/README.md +++ b/cloud/filestore/tests/xfs_suite/README.md @@ -28,9 +28,9 @@ $ export HW_NBS_STABLE_LAB_SEED_IP=$(host $(pssh list C@cloud_hw-nbs-stable-lab_ $ echo -e "\n# ycp hack for hw-nbs-stable-lab\n$HW_NBS_STABLE_LAB_SEED_IP local-lb.cloud-lab.yandex.net" | sudo tee -a /etc/hosts ``` -Create `/etc/ssl/certs/hw-nbs-stable-lab.pem` with the content https://paste.yandex-team.ru/3966169 +Create `/etc/ssl/certs/hw-nbs-stable-lab.pem` with the content of certfile -Save private ssh key from https://yav.yandex-team.ru/secret/sec-01ehpt7c9ez5g4j9nx4g4yegj3/explore/version/ver-01ehpt7c9ng1t28517aqjpavd5 to ~/.ssh/overlay, then execute: +Save private ssh key from https://secret/url to ~/.ssh/overlay, then execute: ```(bash) $ ssh-add ~/.ssh/overlay` @@ -43,7 +43,5 @@ $ ya make ## Example ```(bash) -$ YAV_TOKEN= ./yc-nfs-ci-xfs-test-suite --cluster preprod --test-type virtiofs --test-device nfs-test-dev --test-dir /mnt/test --scratch-type virtiofs --scratch-device nfs-scratch-dev --scratch-dir /mnt/scratch --script-name default.sh +$ SECRET_STORAGE_TOKEN= ./yc-nfs-ci-xfs-test-suite --cluster preprod --test-type virtiofs --test-device nfs-test-dev --test-dir /mnt/test --scratch-type virtiofs --scratch-device nfs-scratch-dev --scratch-dir /mnt/scratch --script-name default.sh ``` - -You can find how to get OAuth YAV_TOKEN in [this instruction](https://docs.yandex-team.ru/yav-api/). diff --git a/cloud/filestore/tests/xfs_suite/lib/generic_100-199.sh b/cloud/filestore/tests/xfs_suite/lib/generic_100-199.sh index 04a13e9b785..7874ccd1ede 100644 --- a/cloud/filestore/tests/xfs_suite/lib/generic_100-199.sh +++ b/cloud/filestore/tests/xfs_suite/lib/generic_100-199.sh @@ -1,6 +1,6 @@ set -ex -# hack until https://st.yandex-team.ru/CLOUD-135098 or https://st.yandex-team.ru/CLOUD-132193 is solved +# hack until CLOUD-135098 or CLOUD-132193 is solved echo "2a02:6b8::183 mirror.yandex.ru" >> /etc/hosts echo "2604:1380:4601:e00::1 git.kernel.org" >> /etc/hosts systemctl restart systemd-hostnamed diff --git a/cloud/filestore/tools/analytics/profile_tool/README.md b/cloud/filestore/tools/analytics/profile_tool/README.md index c1dfa6464b4..cf46e3b003f 100644 --- a/cloud/filestore/tools/analytics/profile_tool/README.md +++ b/cloud/filestore/tools/analytics/profile_tool/README.md @@ -2,11 +2,11 @@ Profile log fills in three places: -* [Public API requests](https://a.yandex-team.ru/arcadia/cloud/filestore/libs/service/request.h?rev=r10845492#L84) are partially filled [on execute](https://a.yandex-team.ru/arcadia/cloud/filestore/libs/server/server.cpp?rev=r10812959#L437) in server and [on sending response](https://a.yandex-team.ru/arcadia/cloud/filestore/libs/server/server.cpp?rev=r10812959#L505). It will be dumped to profile-log [on complete](https://a.yandex-team.ru/arcadia/cloud/filestore/libs/server/server.cpp?rev=r10812959#L558). At the present time this approach is not used. +* [Public API requests](https://github.com/ydb-platform/nbs/blob/main/cloud/filestore/libs/service/request.h) are partially filled [on execute](https://github.com/ydb-platform/nbs/blob/main/cloud/filestore/libs/server/server.cpp) in server and [on sending response](https://github.com/ydb-platform/nbs/blob/main/cloud/filestore/libs/server/server.cpp). It will be dumped to profile-log [on complete](https://github.com/ydb-platform/nbs/blob/main/cloud/filestore/libs/server/server.cpp). At the present time this approach is not used. -* On client-side (vhost) [public API](https://a.yandex-team.ru/arcadia/cloud/filestore/libs/service/request.h?rev=r10845492#L84) requests are partially filled [on request forwarding](https://a.yandex-team.ru/arcadia/cloud/filestore/libs/storage/service/service_actor_forward.cpp?rev=r10812959#L46) in service actor and [on request completion](https://a.yandex-team.ru/arcadia/cloud/filestore/libs/storage/service/service_actor_complete.cpp?rev=r10845492#L35). It will be dumped to profile-log [on in-flight request completion](https://a.yandex-team.ru/arcadia/cloud/filestore/libs/storage/service/service_state.cpp?rev=r10845492#L39). Only significat requests with filled additional fields will be dumped. This profile log can be found in ```/var/log/nfs/vhost-server.log``` on hosts. +* On client-side (vhost) [public API](https://github.com/ydb-platform/nbs/blob/main/cloud/filestore/libs/service/request.h) requests are partially filled [on request forwarding](https://github.com/ydb-platform/nbs/blob/main/cloud/filestore/libs/storage/service/service_actor_forward.cpp) in service actor and [on request completion](https://github.com/ydb-platform/nbs/blob/main/cloud/filestore/libs/storage/service/service_actor_complete.cpp). It will be dumped to profile-log [on in-flight request completion](https://github.com/ydb-platform/nbs/blob/main/cloud/filestore/libs/storage/service/service_state.cpp). Only significant requests with filled additional fields will be dumped. This profile log can be found in ```/var/log/nfs/vhost-server.log``` on hosts. -* In tablet [private API](https://a.yandex-team.ru/arcadia/cloud/filestore/libs/storage/tablet/profile_log_events.h?rev=r10752920#L17) requests are filled in several places. You can try to grep ```InitProfileLogRequestInfo``` and ```FinalizeProfileLogRequestInfo``` in ```cloud/filestore/libs/storage/tablet/*``` and find all the places. This profile log will be dumped to ```/var/log/nfs/nfs-profile.log``` on nfs-control svms. +* In tablet [private API](https://github.com/ydb-platform/nbs/blob/main/cloud/filestore/libs/storage/tablet/profile_log_events.h) requests are filled in several places. You can try to grep ```InitProfileLogRequestInfo``` and ```FinalizeProfileLogRequestInfo``` in ```cloud/filestore/libs/storage/tablet/*``` and find all the places. This profile log will be dumped to ```/var/log/nfs/nfs-profile.log``` on nfs-control svms. ## Common options diff --git a/cloud/storage/core/config/features.proto b/cloud/storage/core/config/features.proto index 041919cd722..706436fe7a0 100644 --- a/cloud/storage/core/config/features.proto +++ b/cloud/storage/core/config/features.proto @@ -11,7 +11,7 @@ message TFilters repeated string CloudIds = 1; repeated string FolderIds = 2; - // DiskId for blockstore or FsId for filestore + // DiskId for blockstore or FsId for filestore. repeated string EntityIds = 3; } @@ -22,19 +22,19 @@ message TFeatureConfig // Feature name. optional string Name = 1; - // Feature parameters. - oneof Params - { - TFilters Whitelist = 2; - TFilters Blacklist = 3; - } + // Enable feature for Clouds and Folders from this list. + optional TFilters Whitelist = 2; + + // Disable feature for Clouds and Folders from this list. + optional TFilters Blacklist = 3; // Optional value - for non-binary features. optional string Value = 4; - // If set, feature can be enabled (for whitelist-features) or disabled - // (for blacklist-features) even if CloudId and FolderId are not matched by - // filters - with these probabilities. + // By setting these probabilities, feature can be enabled for Ids that + // aren't in the whitelist. If blacklist is specified and whitelist is not, + // default value is 1 for compatibility with configs that only use + // blacklist. In other cases it's 0. optional double CloudProbability = 5; optional double FolderProbability = 6; }; diff --git a/cloud/storage/core/libs/common/helpers.h b/cloud/storage/core/libs/common/helpers.h index ff3e5bbec95..dcc7980b1fc 100644 --- a/cloud/storage/core/libs/common/helpers.h +++ b/cloud/storage/core/libs/common/helpers.h @@ -2,6 +2,8 @@ #include "public.h" +#include + #include namespace NCloud { @@ -24,4 +26,12 @@ void SetProtoFlag(ui32& flags, const T flag) } } +template +void SetErrorProtoFlag(NProto::TError& error, const T flag) +{ + auto flags = error.GetFlags(); + SetProtoFlag(flags, flag); + error.SetFlags(flags); +} + } // namespace NCloud diff --git a/cloud/storage/core/libs/diagnostics/critical_events.cpp b/cloud/storage/core/libs/diagnostics/critical_events.cpp index 998ade9be74..40d245e93a6 100644 --- a/cloud/storage/core/libs/diagnostics/critical_events.cpp +++ b/cloud/storage/core/libs/diagnostics/critical_events.cpp @@ -1,7 +1,7 @@ #include "critical_events.h" - #include "public.h" +#include #include #include @@ -13,11 +13,17 @@ using namespace NMonitoring; namespace { NMonitoring::TDynamicCountersPtr CriticalEvents; +TLog Log; } // namespace //////////////////////////////////////////////////////////////////////////////// +void SetCriticalEventsLog(TLog log) +{ + Log = std::move(log); +} + void InitCriticalEventsCounter(NMonitoring::TDynamicCountersPtr counters) { CriticalEvents = std::move(counters); @@ -50,7 +56,15 @@ TString ReportCriticalEvent( fullMessage << "CRITICAL_EVENT:" << sensorName; if (message) { fullMessage << ":" << message; - Cerr << fullMessage << Endl; + } + + if (Log.IsNotNullLog()) { + Log.AddLog("%s", fullMessage.c_str()); + } else { + // Write message and \n in one call. This will reduce the chance of + // shuffling with writings of other threads. + Cerr << fullMessage + '\n'; + Cerr.Flush(); } return fullMessage; @@ -74,4 +88,22 @@ TString ReportCriticalEvent( STORAGE_CRITICAL_EVENTS(STORAGE_DEFINE_CRITICAL_EVENT_ROUTINE) #undef STORAGE_DEFINE_CRITICAL_EVENT_ROUTINE +//////////////////////////////////////////////////////////////////////////////// + +void ReportPreconditionFailed( + TStringBuf file, + int line, + TStringBuf func, + TStringBuf expr) +{ + ReportCriticalEvent( + "PreconditionFailed", + TStringBuilder() + << file << ":" << line + << " " << func << "(): requirement " << expr + << " failed", + true // verifyDebug + ); +} + } // namespace NCloud diff --git a/cloud/storage/core/libs/diagnostics/critical_events.h b/cloud/storage/core/libs/diagnostics/critical_events.h index 9f84cd7a23e..2754c38bde5 100644 --- a/cloud/storage/core/libs/diagnostics/critical_events.h +++ b/cloud/storage/core/libs/diagnostics/critical_events.h @@ -15,6 +15,7 @@ namespace NCloud { //////////////////////////////////////////////////////////////////////////////// +void SetCriticalEventsLog(TLog log); void InitCriticalEventsCounter(NMonitoring::TDynamicCountersPtr counters); TString ReportCriticalEvent( @@ -30,4 +31,21 @@ TString ReportCriticalEvent( STORAGE_CRITICAL_EVENTS(STORAGE_DECLARE_CRITICAL_EVENT_ROUTINE) #undef STORAGE_DECLARE_CRITICAL_EVENT_ROUTINE +void ReportPreconditionFailed( + TStringBuf file, + int line, + TStringBuf func, + TStringBuf expr); + +//////////////////////////////////////////////////////////////////////////////// + +#define STORAGE_CHECK_PRECONDITION(expr) \ + if (!(expr)) { \ + ReportPreconditionFailed( \ + __SOURCE_FILE_IMPL__, \ + __LINE__, \ + __FUNCTION__, \ + #expr); \ + } \ + } // namespace NCloud diff --git a/cloud/storage/core/libs/features/features_config.cpp b/cloud/storage/core/libs/features/features_config.cpp index 4cca38525d0..1ed22934269 100644 --- a/cloud/storage/core/libs/features/features_config.cpp +++ b/cloud/storage/core/libs/features/features_config.cpp @@ -17,34 +17,26 @@ bool ProbabilityMatch(double p, const TString& s) //////////////////////////////////////////////////////////////////////////////// -TFeaturesConfig::TFeaturesConfig( - NProto::TFeaturesConfig config) - : Config(std::move(config)) +TFeaturesConfig::TFeatureInfo::TFeatureInfo(NProto::TFeatureConfig config) + : Whitelist(config.GetWhitelist()) + , Blacklist(config.GetBlacklist()) + , Value(config.GetValue()) { - for (const auto& feature: Config.GetFeatures()) { - TFeatureInfo info; - info.IsBlacklist = feature.HasBlacklist(); - info.CloudProbability = feature.GetCloudProbability(); - info.FolderProbability = feature.GetFolderProbability(); + double defaultProbability = config.HasBlacklist() && !config.HasWhitelist() + ? 1.0 : 0.0; - const auto& cloudList = feature.HasBlacklist() - ? feature.GetBlacklist() : feature.GetWhitelist(); + CloudProbability = config.HasCloudProbability() + ? config.GetCloudProbability() : defaultProbability; - for (const auto& cloudId: cloudList.GetCloudIds()) { - info.CloudIds.emplace(cloudId); - } - - for (const auto& folderId: cloudList.GetFolderIds()) { - info.FolderIds.emplace(folderId); - } - - for (const auto& entityId: cloudList.GetEntityIds()) { - info.EntityIds.emplace(entityId); - } - - info.Value = feature.GetValue(); + FolderProbability = config.HasFolderProbability() + ? config.GetFolderProbability() : defaultProbability; +} - Features.emplace(feature.GetName(), std::move(info)); +TFeaturesConfig::TFeaturesConfig(NProto::TFeaturesConfig config) + : Config(std::move(config)) +{ + for (const auto& feature: Config.GetFeatures()) { + Features.emplace(feature.GetName(), feature); } } @@ -89,31 +81,27 @@ bool TFeaturesConfig::GetFeature( const TString& featureName, TString* value) const { - bool result = false; - auto it = Features.find(featureName); + if (it != Features.end()) { - auto isBlacklist = it->second.IsBlacklist; - const bool probabilityMatch = - ProbabilityMatch(it->second.CloudProbability, cloudId) - || ProbabilityMatch(it->second.FolderProbability, folderId); - - if (it->second.CloudIds.contains(cloudId) - || it->second.FolderIds.contains(folderId) - || it->second.EntityIds.contains(entityId) - || probabilityMatch) - { - result = !isBlacklist; - } else { - result = isBlacklist; + auto& feature = it->second; + + if (feature.Blacklist.Contains(cloudId, folderId, entityId)) { + return false; } - if (result && value) { - *value = it->second.Value; + if (feature.Whitelist.Contains(cloudId, folderId, entityId) || + ProbabilityMatch(feature.CloudProbability, cloudId) || + ProbabilityMatch(feature.FolderProbability, folderId)) + { + if (value) { + *value = feature.Value; + } + return true; } } - return result; + return false; } } // namespace NCloud::NFeatures diff --git a/cloud/storage/core/libs/features/features_config.h b/cloud/storage/core/libs/features/features_config.h index 07f5cee6141..1f02e4a6c3a 100644 --- a/cloud/storage/core/libs/features/features_config.h +++ b/cloud/storage/core/libs/features/features_config.h @@ -2,9 +2,10 @@ #include "public.h" +#include "filters.h" + #include -#include #include #include @@ -16,13 +17,13 @@ class TFeaturesConfig { struct TFeatureInfo { - THashSet CloudIds; - THashSet FolderIds; - THashSet EntityIds; // DiskIds or FsIds - bool IsBlacklist = false; + TFilters Whitelist; + TFilters Blacklist; double CloudProbability = 0; double FolderProbability = 0; TString Value; + + explicit TFeatureInfo(NProto::TFeatureConfig config); }; private: @@ -31,7 +32,7 @@ class TFeaturesConfig THashMap Features; public: - TFeaturesConfig(NProto::TFeaturesConfig config = {}); + explicit TFeaturesConfig(NProto::TFeaturesConfig config = {}); bool IsValid() const; diff --git a/cloud/storage/core/libs/features/features_config_ut.cpp b/cloud/storage/core/libs/features/features_config_ut.cpp index 830dce6a386..6549df7df99 100644 --- a/cloud/storage/core/libs/features/features_config_ut.cpp +++ b/cloud/storage/core/libs/features/features_config_ut.cpp @@ -148,6 +148,45 @@ Y_UNIT_TEST_SUITE(TFeaturesConfigTest) "whitelisted_id", f->GetName())); } + + Y_UNIT_TEST(ShouldSupportWhiteAndBlacklistSimultaneously) + { + NProto::TFeaturesConfig fc; + auto* f = fc.AddFeatures(); + f->SetName("feature"); + + *f->MutableWhitelist()->AddCloudIds() = "white"; + *f->MutableWhitelist()->AddCloudIds() = "gray"; + + *f->MutableBlacklist()->AddCloudIds() = "black"; + *f->MutableBlacklist()->AddCloudIds() = "gray"; + + TFeaturesConfig config(fc); + + UNIT_ASSERT(config.IsFeatureEnabled("white", {}, {}, f->GetName())); + UNIT_ASSERT(!config.IsFeatureEnabled("black", {}, {}, f->GetName())); + + // blacklist takes precedence + UNIT_ASSERT(!config.IsFeatureEnabled("gray", {}, {}, f->GetName())); + } + + Y_UNIT_TEST(ShouldEnableByDefaultIfWhitelistIsEmpty) + { + NProto::TFeaturesConfig fc; + auto* f = fc.AddFeatures(); + f->SetName("feature"); + *f->MutableBlacklist()->AddCloudIds() = "black"; + TFeaturesConfig config(fc); + + auto clouds = RandomStrings(1000); + int matches = 0; + for (const auto& cloud: clouds) { + matches += config.IsFeatureEnabled(cloud, {}, {}, f->GetName()); + } + + UNIT_ASSERT_C(matches == 1000, TStringBuilder() + << "match count: " << matches); + } } } // namespace NCloud::NFeatures diff --git a/cloud/storage/core/libs/features/filters.cpp b/cloud/storage/core/libs/features/filters.cpp new file mode 100644 index 00000000000..27d0f24ad3d --- /dev/null +++ b/cloud/storage/core/libs/features/filters.cpp @@ -0,0 +1,29 @@ +#include "filters.h" + +namespace NCloud::NFeatures { + +//////////////////////////////////////////////////////////////////////////////// + +TFilters::TFilters(NProto::TFilters config) +{ + for (const auto& cloudId: config.GetCloudIds()) { + CloudIds.emplace(cloudId); + } + for (const auto& folderId: config.GetFolderIds()) { + FolderIds.emplace(folderId); + } + for (const auto& entityId: config.GetEntityIds()) { + EntityIds.emplace(entityId); + } +} + +bool TFilters::Contains( + const TString& cloudId, + const TString& folderId, + const TString& entityId) const +{ + return CloudIds.contains(cloudId) || FolderIds.contains(folderId) || + EntityIds.contains(entityId); +} + +} // namespace NCloud::NFeatures diff --git a/cloud/storage/core/libs/features/filters.h b/cloud/storage/core/libs/features/filters.h new file mode 100644 index 00000000000..d61ffc3a613 --- /dev/null +++ b/cloud/storage/core/libs/features/filters.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include +#include + +namespace NCloud::NFeatures { + +//////////////////////////////////////////////////////////////////////////////// + +class TFilters +{ +private: + THashSet CloudIds; + THashSet FolderIds; + THashSet EntityIds; // DiskIds or FsIds + +public: + explicit TFilters(NProto::TFilters config); + + bool Contains( + const TString& cloudId, + const TString& folderId, + const TString& entityId) const; +}; + +} // namespace NCloud::NFeatures diff --git a/cloud/storage/core/libs/features/ya.make b/cloud/storage/core/libs/features/ya.make index a49712237e0..e7842f9b48d 100644 --- a/cloud/storage/core/libs/features/ya.make +++ b/cloud/storage/core/libs/features/ya.make @@ -2,6 +2,7 @@ LIBRARY() SRCS( features_config.cpp + filters.cpp ) PEERDIR( diff --git a/cloud/storage/core/libs/grpc/ut/ya.make b/cloud/storage/core/libs/grpc/ut/ya.make index fca5e5cdae0..9ce30c92811 100644 --- a/cloud/storage/core/libs/grpc/ut/ya.make +++ b/cloud/storage/core/libs/grpc/ut/ya.make @@ -2,6 +2,7 @@ UNITTEST_FOR(cloud/storage/core/libs/grpc) SRCS( executor_ut.cpp + utils_ut.cpp ) PEERDIR( diff --git a/cloud/storage/core/libs/grpc/utils.cpp b/cloud/storage/core/libs/grpc/utils.cpp index b16d4b86cac..bedfdde9cdb 100644 --- a/cloud/storage/core/libs/grpc/utils.cpp +++ b/cloud/storage/core/libs/grpc/utils.cpp @@ -2,10 +2,24 @@ #include +#include + namespace NCloud { //////////////////////////////////////////////////////////////////////////////// +bool TryParseSourceFd(const TStringBuf& peer, ui32* fd) +{ + static constexpr TStringBuf PeerFdPrefix = "fd:"; + TStringBuf peerFd; + if (!peer.AfterPrefix(PeerFdPrefix, peerFd)) { + return false; + } + return TryFromString(peerFd, *fd); +} + +//////////////////////////////////////////////////////////////////////////////// + size_t SetExecutorThreadsLimit(size_t count) { return grpc_core::Executor::SetThreadsLimit(count); diff --git a/cloud/storage/core/libs/grpc/utils.h b/cloud/storage/core/libs/grpc/utils.h index 692cabff2db..b6137915b06 100644 --- a/cloud/storage/core/libs/grpc/utils.h +++ b/cloud/storage/core/libs/grpc/utils.h @@ -2,12 +2,15 @@ #include "public.h" +#include #include namespace NCloud { //////////////////////////////////////////////////////////////////////////////// +bool TryParseSourceFd(const TStringBuf& peer, ui32* fd); + size_t SetExecutorThreadsLimit(size_t count); } // namespace NCloud diff --git a/cloud/storage/core/libs/grpc/utils_ut.cpp b/cloud/storage/core/libs/grpc/utils_ut.cpp new file mode 100644 index 00000000000..8b8c8945f7c --- /dev/null +++ b/cloud/storage/core/libs/grpc/utils_ut.cpp @@ -0,0 +1,35 @@ +#include "utils.h" + +#include + +namespace NCloud::NStorage::NGrpc { + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(TUtilsTest) +{ + Y_UNIT_TEST(ShouldExtractFdFromGrpcPeerString) + { + const TStringBuf b{"fd:12345"}; + ui32 fd{0}; + UNIT_ASSERT(TryParseSourceFd(b, &fd)); + UNIT_ASSERT_VALUES_EQUAL(12345, fd); + } + + Y_UNIT_TEST(ShouldFailIfPeerStringIsIncorrect) + { + { + const TStringBuf b{"fd12345"}; + ui32 fd{0}; + UNIT_ASSERT(!TryParseSourceFd(b, &fd)); + } + + { + const TStringBuf b{"fd:abcd"}; + ui32 fd{0}; + UNIT_ASSERT(!TryParseSourceFd(b, &fd)); + } + } +} + +} // namespace NCloud::NStorage::NGrpc diff --git a/cloud/storage/core/libs/hive_proxy/ut/ya.make b/cloud/storage/core/libs/hive_proxy/ut/ya.make index b0c593d6b55..e92840992f6 100644 --- a/cloud/storage/core/libs/hive_proxy/ut/ya.make +++ b/cloud/storage/core/libs/hive_proxy/ut/ya.make @@ -15,6 +15,7 @@ SRCS( ) PEERDIR( + contrib/ydb/core/testlib contrib/ydb/core/testlib/default contrib/ydb/core/testlib/basics ) diff --git a/cloud/storage/core/libs/hive_proxy/ya.make b/cloud/storage/core/libs/hive_proxy/ya.make index 2b9c8e19e0b..8b339c8cc74 100644 --- a/cloud/storage/core/libs/hive_proxy/ya.make +++ b/cloud/storage/core/libs/hive_proxy/ya.make @@ -24,8 +24,6 @@ PEERDIR( contrib/ydb/core/mind contrib/ydb/core/tablet contrib/ydb/core/tablet_flat - contrib/ydb/core/testlib - contrib/ydb/core/testlib/basics contrib/ydb/library/actors/core ) diff --git a/cloud/storage/core/libs/kikimr/components.h b/cloud/storage/core/libs/kikimr/components.h index 6a4e904e783..bfa7a8cabe0 100644 --- a/cloud/storage/core/libs/kikimr/components.h +++ b/cloud/storage/core/libs/kikimr/components.h @@ -74,22 +74,4 @@ struct TStoragePrivateEvents "END expected to be < EventSpaceEnd(NKikimr::TKikimrEvents::ES_CLOUD_STORAGE_PRIVATE)"); }; -//////////////////////////////////////////////////////////////////////////////// - -struct TStorageActivities -{ - enum - { -#define STORAGE_DECLARE_COMPONENT(component) \ - component = NKikimrServices::TActivity::CLOUD_STORAGE_##component, \ -// STORAGE_DECLARE_COMPONENT - - STORAGE_ACTORS(STORAGE_DECLARE_COMPONENT) - - AUTH = NKikimrServices::TActivity::BLOCKSTORE_AUTH, - USER_STATS = NKikimrServices::TActivity::BLOCKSTORE_USER_STATS -#undef STORAGE_DECLARE_COMPONENT - }; -}; - } // namespace NCloud diff --git a/cloud/storage/core/libs/kikimr/components_start.cpp b/cloud/storage/core/libs/kikimr/components_start.cpp new file mode 100644 index 00000000000..41b42a22439 --- /dev/null +++ b/cloud/storage/core/libs/kikimr/components_start.cpp @@ -0,0 +1 @@ +#include "components_start.h" diff --git a/cloud/storage/core/libs/kikimr/components_start.h b/cloud/storage/core/libs/kikimr/components_start.h new file mode 100644 index 00000000000..00edf1cca0d --- /dev/null +++ b/cloud/storage/core/libs/kikimr/components_start.h @@ -0,0 +1,16 @@ +#pragma once + +namespace NCloud { + +//////////////////////////////////////////////////////////////////////////////// + +struct TComponentsStart +{ + enum + { + BlockStoreComponentsStart = 1024, + FileStoreComponentsStart = 2048 + }; +}; + +} // namespace NCloud diff --git a/cloud/storage/core/libs/kikimr/ya.make b/cloud/storage/core/libs/kikimr/ya.make index 01b7881a6d3..417dfd07fc0 100644 --- a/cloud/storage/core/libs/kikimr/ya.make +++ b/cloud/storage/core/libs/kikimr/ya.make @@ -3,6 +3,7 @@ LIBRARY() SRCS( actorsystem.cpp components.cpp + components_start.cpp config_initializer.cpp events.cpp helpers.cpp @@ -17,11 +18,13 @@ PEERDIR( cloud/storage/core/libs/common cloud/storage/core/libs/diagnostics + library/cpp/getopt/small + library/cpp/lwtrace + contrib/ydb/library/actors/core contrib/ydb/library/actors/util contrib/ydb/library/actors/wilson - library/cpp/getopt/small - library/cpp/lwtrace + contrib/ydb/library/keys contrib/ydb/core/base contrib/ydb/core/mind diff --git a/cloud/storage/core/libs/user_stats/user_stats.cpp b/cloud/storage/core/libs/user_stats/user_stats.cpp index 88a0a17b18c..0ae411f114f 100644 --- a/cloud/storage/core/libs/user_stats/user_stats.cpp +++ b/cloud/storage/core/libs/user_stats/user_stats.cpp @@ -7,11 +7,13 @@ namespace NCloud::NStorage::NUserStats { //////////////////////////////////////////////////////////////////////////////// NActors::IActorPtr CreateStorageUserStats( + int component, TString path, TString title, TVector providers) { return std::make_unique( + component, std::move(path), std::move(title), std::move(providers)); diff --git a/cloud/storage/core/libs/user_stats/user_stats.h b/cloud/storage/core/libs/user_stats/user_stats.h index ca6441426e6..2102fa6363e 100644 --- a/cloud/storage/core/libs/user_stats/user_stats.h +++ b/cloud/storage/core/libs/user_stats/user_stats.h @@ -8,6 +8,7 @@ namespace NCloud::NStorage::NUserStats { //////////////////////////////////////////////////////////////////////////////// NActors::IActorPtr CreateStorageUserStats( + int component, TString path, TString title, TVector providers); diff --git a/cloud/storage/core/libs/user_stats/user_stats_actor.cpp b/cloud/storage/core/libs/user_stats/user_stats_actor.cpp index f37c5480e0b..7cdb955997b 100644 --- a/cloud/storage/core/libs/user_stats/user_stats_actor.cpp +++ b/cloud/storage/core/libs/user_stats/user_stats_actor.cpp @@ -19,10 +19,12 @@ namespace NCloud::NStorage::NUserStats { //////////////////////////////////////////////////////////////////////////////// TUserStatsActor::TUserStatsActor( + int component, TString path, TString title, TVector providers) : Providers(std::move(providers)) + , Component(component) , Path(std::move(path)) , Title(std::move(title)) {} @@ -144,7 +146,7 @@ STFUNC(TUserStatsActor::StateWork) HFunc(TEvUserStats::TEvUserStatsProviderCreate, HandleUserStatsProviderCreate); default: - HandleUnexpectedEvent(ev, TStorageActivities::USER_STATS); + HandleUnexpectedEvent(ev, Component); } } diff --git a/cloud/storage/core/libs/user_stats/user_stats_actor.h b/cloud/storage/core/libs/user_stats/user_stats_actor.h index 9f55c81b7b7..630e6f239eb 100644 --- a/cloud/storage/core/libs/user_stats/user_stats_actor.h +++ b/cloud/storage/core/libs/user_stats/user_stats_actor.h @@ -18,11 +18,13 @@ class TUserStatsActor final TRWMutex Lock; TVector Providers; + const int Component; const TString Path; const TString Title; public: TUserStatsActor( + int component, TString path, TString title, TVector providers); diff --git a/cloud/storage/core/libs/version_ydb/version.cpp b/cloud/storage/core/libs/version_ydb/version.cpp new file mode 100644 index 00000000000..e86c2325f51 --- /dev/null +++ b/cloud/storage/core/libs/version_ydb/version.cpp @@ -0,0 +1,13 @@ +#include + +NKikimrConfig::TCurrentCompatibilityInfo +NKikimr::TCompatibilityInfo::MakeCurrent() +{ + using namespace NKikimr; + using TCurrentConstructor = + TCompatibilityInfo::TProtoConstructor::TCurrentCompatibilityInfo; + + return TCurrentConstructor{ + .Application = "nbs", + }.ToPB(); +} diff --git a/cloud/storage/core/libs/version_ydb/ya.make b/cloud/storage/core/libs/version_ydb/ya.make new file mode 100644 index 00000000000..10bd30524c5 --- /dev/null +++ b/cloud/storage/core/libs/version_ydb/ya.make @@ -0,0 +1,13 @@ +OWNER(g:cloud-nbs) + +LIBRARY() + +SRCS( + version.cpp +) + +PEERDIR( + contrib/ydb/core/driver_lib/version +) + +END() diff --git a/cloud/storage/core/libs/viewer/tablet_monitoring.cpp b/cloud/storage/core/libs/viewer/tablet_monitoring.cpp new file mode 100644 index 00000000000..a3218e07bbe --- /dev/null +++ b/cloud/storage/core/libs/viewer/tablet_monitoring.cpp @@ -0,0 +1,132 @@ +#include "tablet_monitoring.h" + +#include + +namespace NCloud::NStorage { + +using namespace NKikimr; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +void DumpChannel( + IOutputStream& out, + const TChannelMonInfo& channelInfo, + const ui64 tabletId, + const TTabletChannelInfo& channel, + const TGetMonitoringYDBGroupUrl& getGroupUrl, + const TBuildReassignChannelButton& buildReassignButton, + ui64 hiveTabletId) +{ + HTML(out) { + TABLER() { + TABLED() { out << "Channel: " << channel.Channel; } + TABLED() { out << "StoragePool: " << channel.StoragePool; } + + auto latestEntry = channel.LatestEntry(); + if (!latestEntry) { + return; + } + + TABLED() { out << "Id: " << latestEntry->GroupID; } + TABLED() { out << "Gen: " << latestEntry->FromGeneration; } + TABLED() { out << "PoolKind: " << channelInfo.PoolKind; } + TABLED() { out << "DataKind: " << channelInfo.DataKind; } + TABLED() { + TStringBuf label; + TStringBuf color; + if (channelInfo.SystemWritable) { + if (channelInfo.Writable) { + color = "green"; + label = "Writable"; + } else { + color = "yellow"; + label = "SystemWritable"; + } + } else { + if (channelInfo.Writable) { + color = "pink"; + label = "WeirdState"; + } else { + color = "orange"; + label = "Readonly"; + } + } + + SPAN_CLASS_STYLE( + "label", + TStringBuilder() << "background-color: " << color) + { + out << label; + } + } + TABLED() { + out << "Status"; + } + const auto groupUrl = + getGroupUrl(latestEntry->GroupID, channel.StoragePool); + if (groupUrl) { + TABLED() { + out << "Graphs"; + } + } + TABLED() { + buildReassignButton( + out, + hiveTabletId, + tabletId, + channel.Channel); + } + } + } +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +void DumpChannels( + IOutputStream& out, + const TVector& channelInfos, + const TTabletStorageInfo& storage, + const TGetMonitoringYDBGroupUrl& getGroupUrl, + const TBuildReassignChannelButton& buildReassignButton, + ui64 hiveTabletId) +{ + HTML(out) { + DIV() { + out << "

Channel history

"; + } + + TABLE_CLASS("table table-condensed") { + TABLEBODY() { + for (const auto& channel: storage.Channels) { + TChannelMonInfo channelInfo; + // we need this check for legacy volumes + // see NBS-752 + if (channel.Channel < channelInfos.size()) { + channelInfo = channelInfos[channel.Channel]; + } + + DumpChannel( + out, + channelInfo, + storage.TabletID, + channel, + getGroupUrl, + buildReassignButton, + hiveTabletId); + } + } + } + } +} + +} // namespace NCloud::NStorage diff --git a/cloud/storage/core/libs/viewer/tablet_monitoring.h b/cloud/storage/core/libs/viewer/tablet_monitoring.h new file mode 100644 index 00000000000..052e0ddaade --- /dev/null +++ b/cloud/storage/core/libs/viewer/tablet_monitoring.h @@ -0,0 +1,38 @@ +#pragma once + +#include + +#include +#include + +namespace NCloud::NStorage { + +//////////////////////////////////////////////////////////////////////////////// + +struct TChannelMonInfo +{ + TString PoolKind; + TString DataKind; + bool Writable = false; + bool SystemWritable = false; +}; + +using TGetMonitoringYDBGroupUrl = std::function; + +using TBuildReassignChannelButton = std::function; + +void DumpChannels( + IOutputStream& out, + const TVector& channelInfos, + const NKikimr::TTabletStorageInfo& storage, + const TGetMonitoringYDBGroupUrl& getGroupUrl, + const TBuildReassignChannelButton& buildReassignButton, + ui64 hiveTabletId); + +} // namespace NCloud::NStorage diff --git a/cloud/storage/core/libs/viewer/ya.make b/cloud/storage/core/libs/viewer/ya.make new file mode 100644 index 00000000000..07eb9f3b574 --- /dev/null +++ b/cloud/storage/core/libs/viewer/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +SRCS( + tablet_monitoring.cpp +) + +PEERDIR( + library/cpp/monlib/service/pages + + contrib/ydb/core/base +) + +END() diff --git a/cloud/storage/core/libs/ya.make b/cloud/storage/core/libs/ya.make index f8608a244a3..99947cb606b 100644 --- a/cloud/storage/core/libs/ya.make +++ b/cloud/storage/core/libs/ya.make @@ -17,4 +17,7 @@ RECURSE( uds user_stats version + version_ydb + vhost-client + viewer ) diff --git a/cloud/storage/core/tests/recipes/large.inc b/cloud/storage/core/tests/recipes/large.inc index 6c3a26ae9b4..b4f84eb64d9 100644 --- a/cloud/storage/core/tests/recipes/large.inc +++ b/cloud/storage/core/tests/recipes/large.inc @@ -2,7 +2,7 @@ SIZE(LARGE) TIMEOUT(3600) # use ya:not_autocheck and ya:manual with sanitizers until we have sufficient -# quota in the YC_NBS group: https://sandbox.yandex-team.ru/admin/groups/YC_NBS/general +# quota IF (SANITIZER_TYPE) TAG( ya:fat diff --git a/cloud/storage/core/tools/breakpad/launcher/error_collector.py b/cloud/storage/core/tools/breakpad/launcher/error_collector.py index 112d61651c2..c32f1980283 100644 --- a/cloud/storage/core/tools/breakpad/launcher/error_collector.py +++ b/cloud/storage/core/tools/breakpad/launcher/error_collector.py @@ -5,7 +5,7 @@ class ErrorCollector(object): - # https://a.yandex-team.ru/arc/trunk/arcadia/util/system/yassert.cpp?rev=2401334#L39-41 + # https://github.com/ydb-platform/nbs/blob/main/util/system/yassert.cpp ERROR_LINES = 3 # Message consist of three lines ERROR_PREFIXES = ["VERIFY failed:", "FAIL:", "VERIFY failed (", "FAIL ("] diff --git a/cloud/storage/core/tools/breakpad/sender/conductor.py b/cloud/storage/core/tools/breakpad/sender/conductor.py index c797492eaa1..048a06d1b7a 100644 --- a/cloud/storage/core/tools/breakpad/sender/conductor.py +++ b/cloud/storage/core/tools/breakpad/sender/conductor.py @@ -15,7 +15,7 @@ class ConductorError(Exception): class Conductor(object): HTTP_TIMEOUT = 5 - API_URL = "http://c.yandex-team.ru/api/hosts2groups/" + API_URL = "" FILTERED_GROUPS = list() def __init__(self, hostname=None): diff --git a/cloud/storage/core/tools/breakpad/sender/sender.py b/cloud/storage/core/tools/breakpad/sender/sender.py index 4671275a1f0..709842382af 100644 --- a/cloud/storage/core/tools/breakpad/sender/sender.py +++ b/cloud/storage/core/tools/breakpad/sender/sender.py @@ -86,7 +86,7 @@ def _send_to_aggregator(self): @retry(max_times=10, delay=60) def _send_email(self): self._logger.info("Send core to email %r", self.emails) - mail_from = "devnull@yandex-team.ru" + mail_from = "devnull@example.com" mail_to = ", ".join(self.emails) message_body = [self._header()] if self._core_url: diff --git a/cloud/storage/core/tools/ci/etc/crontab b/cloud/storage/core/tools/ci/etc/crontab index 2663d800cf1..c2f84e2227e 100644 --- a/cloud/storage/core/tools/ci/etc/crontab +++ b/cloud/storage/core/tools/ci/etc/crontab @@ -20,7 +20,7 @@ SHELL=/bin/sh 25 6 * * * root test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.daily ) 47 6 * * 7 root test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.weekly ) 52 6 1 * * root test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.monthly ) -0 */6 * * * nbsci rm -rf /tmp/last_ci && mkdir /tmp/last_ci && cd /tmp/last_ci && /root/runner/run_all.sh +0 */1 * * * nbsci rm -rf /tmp/last_ci && mkdir /tmp/last_ci && cd /tmp/last_ci && /root/runner/run_all.sh 0 6 * * * root /root/runner/run_fio.sh 0 10 * * * root /root/runner/run_degradation_test.sh 0 9 * * * root /root/runner/run_corruption.sh diff --git a/cloud/storage/core/tools/ci/runner/run_all.sh b/cloud/storage/core/tools/ci/runner/run_all.sh index eeeb9d47d0c..2c0a33215ca 100755 --- a/cloud/storage/core/tools/ci/runner/run_all.sh +++ b/cloud/storage/core/tools/ci/runner/run_all.sh @@ -1,6 +1,15 @@ #!/usr/bin/env bash +lockfile="/var/tmp/_run_all.lock" +if { set -C; true 2>/dev/null > $lockfile; }; then + # shellcheck disable=SC2064 + trap "rm -f $lockfile; echo 'lock file removed'" EXIT +else + echo "lock file exists…" + exit +fi + logs_root="/var/www/build/logs" logs_dir="${logs_root}/run_$(date +%y_%m_%d__%H)" && @@ -8,7 +17,6 @@ rm -rf "$logs_dir" && mkdir -p "$logs_dir" exec 3>&1 4>&2 -trap 'exec 2>&4 1>&3' 0 1 2 3 exec 1>"${logs_dir}/run_all.out" 2>&1 d="/root" @@ -16,15 +24,6 @@ scripts="${d}/runner" nbspath="$d/github/blockstore/nbs" cwd=$(pwd) -lockfile="/var/tmp/_run_all.lock" -if { set -C; true 2>/dev/null > $lockfile; }; then - # shellcheck disable=SC2064 - trap "rm -f $lockfile; echo 'lock file removed'" EXIT -else - echo "lock file exists…" - exit -fi - cd $nbspath && git reset --hard && git pull && diff --git a/cloud/storage/core/tools/common/go/configurator/configurator.go b/cloud/storage/core/tools/common/go/configurator/configurator.go index a57e7d14a33..e7d3fd6073b 100644 --- a/cloud/storage/core/tools/common/go/configurator/configurator.go +++ b/cloud/storage/core/tools/common/go/configurator/configurator.go @@ -552,15 +552,15 @@ func contains(collection []string, target string) bool { return false } -func (g *ConfigGenerator) Generate(ctx context.Context, whileListCluster []string) error { +func (g *ConfigGenerator) Generate(ctx context.Context, whiteListCluster []string) error { g.LogInfo( ctx, "Start generation for service: %v", g.spec.ServiceSpec.ServiceName) - genAllClusters := len(whileListCluster) == 0 + genAllClusters := len(whiteListCluster) == 0 for cluster, clusterConfig := range g.spec.ServiceSpec.Clusters { - if !genAllClusters && !contains(whileListCluster, cluster) { + if !genAllClusters && !contains(whiteListCluster, cluster) { continue } for _, zone := range clusterConfig.Zones { diff --git a/cloud/storage/core/tools/common/python/core_pattern.py b/cloud/storage/core/tools/common/python/core_pattern.py index f0b1a02791a..fd9f86fe90f 100644 --- a/cloud/storage/core/tools/common/python/core_pattern.py +++ b/cloud/storage/core/tools/common/python/core_pattern.py @@ -18,7 +18,6 @@ def core_pattern(binary_path, cwd=None): if not p.startswith('|'): return p.replace('%e', mask) - # see: https://a.yandex-team.ru/arc/trunk/arcadia/sandbox/bin/coredumper.py?rev=r5646070#L28 # > filename = ".".join((name, pid, sig)) name = ".".join((mask, '%p', '%s')) diff --git a/cloud/storage/core/tools/common/python/daemon.py b/cloud/storage/core/tools/common/python/daemon.py index 732a721f8a5..e4c7998ca5b 100644 --- a/cloud/storage/core/tools/common/python/daemon.py +++ b/cloud/storage/core/tools/common/python/daemon.py @@ -87,12 +87,16 @@ def __terminate_process(self): self.__verify_process() logger.info("terminating process") self.__process.terminate() + self.__process.wait(check_exit_code=False) + self.__process = None # Should be guarded by self.__lock. def __kill_process(self): self.__verify_process() logger.info("killing process") self.__process.kill() + self.__process.wait(check_exit_code=False) + self.__process = None # Should be guarded by self.__lock. def __ping(self): @@ -182,7 +186,6 @@ def stop(self): self.__timer = None self.__terminate_process() - self.__process = None def kill(self): with self.__lock: @@ -192,7 +195,6 @@ def kill(self): self.__timer = None self.__kill_process() - self.__process = None @property def command(self): diff --git a/cloud/storage/core/tools/testing/qemu/build-image/README.md b/cloud/storage/core/tools/testing/qemu/build-image/README.md index e2e45b7d8f1..bf2b2070c2f 100644 --- a/cloud/storage/core/tools/testing/qemu/build-image/README.md +++ b/cloud/storage/core/tools/testing/qemu/build-image/README.md @@ -7,7 +7,7 @@ This script allows to build qcow2 image for local and cloud tests based on vanil - Create image ``` -./build-image --user root --ssh-key [nbs key](https://yav.yandex-team.ru/secret/sec-01g6n81wt8wc6ph8kagb5b33bf/explore/version/ver-01g6n81wtp52w8pacjp41xhrpg) --out ubuntu-2004-eternal.qcow +./build-image --user root --ssh-key nbs-key --out ubuntu-2004-eternal.qcow ``` - Upload it to the s3 bucket @@ -22,14 +22,14 @@ curl -o /dev/null -# https://$BUCKET.storage.cloudil.com/$FILE -T $FILE -H "X-Ya - Create a new image out of s3 uploaded file via console -- Update it for required cluster in [tests configs](https://a.yandex-team.ru/arcadia/cloud/blockstore/pylibs/clusters/test_config.py?rev=r10300383#L41-232) +- Update it for required cluster in tests configs ### Building for local tests - Create image ``` -./build-image --user qemu --plain-pwd --ssh-key [nbs key](https://yav.yandex-team.ru/secret/sec-01g6n81wt8wc6ph8kagb5b33bf/explore/version/ver-01g6n81wtp52w8pacjp41xhrpg) --out rootfs.img +./build-image --user qemu --plain-pwd --ssh-key nbs-key --out rootfs.img ``` - Optionally compress image @@ -43,4 +43,4 @@ qemu-img convert -c -p -f qcow2 -O qcow2 rootfs.img rootfs-compressed.img ya upload --ttl=inf -T NBS_QEMU_DISK_IMAGE -d 'Customized Ubuntu Cloud Image in QCOW2 format' ``` -- Update [resource](https://a.yandex-team.ru/arcadia/cloud/storage/core/tools/testing/qemu/image/ya.make?rev=r9706918#L8) in ya.make +- Update [resource](https://github.com/ydb-platform/nbs/blob/main/cloud/storage/core/tools/testing/qemu/image/ya.make) in ya.make diff --git a/cloud/storage/core/tools/testing/qemu/build/__main__.py b/cloud/storage/core/tools/testing/qemu/build/__main__.py index cad612002d9..ae7ce1d2d6b 100755 --- a/cloud/storage/core/tools/testing/qemu/build/__main__.py +++ b/cloud/storage/core/tools/testing/qemu/build/__main__.py @@ -203,7 +203,7 @@ def main(args): "--git", help="source of the qemu", action="store", - default="ssh://git@bb.yandex-team.ru/cloud/qemu.git") + default="ssh://git@github.com:qemu/qemu.git") parser.add_argument( "--git-tag", help="specific tag", diff --git a/cloud/storage/core/tools/testing/qemu/lib/recipe.py b/cloud/storage/core/tools/testing/qemu/lib/recipe.py index f9dfca4cd19..911f2afef2e 100644 --- a/cloud/storage/core/tools/testing/qemu/lib/recipe.py +++ b/cloud/storage/core/tools/testing/qemu/lib/recipe.py @@ -5,7 +5,7 @@ import yatest.common import yatest.common.network import retrying -import pipes +import shlex import tarfile import library.python.fs as fs @@ -339,7 +339,7 @@ def _get_vm_use_virtiofs_server(args): def _prepare_test_environment(ssh, virtio): - # Workaround for https://st.yandex-team.ru/DISKMAN-63 + # Workaround for DISKMAN-63 if "TMPDIR" in os.environ: ssh("sudo mkdir -m a=rwx -p {}".format(os.environ['TMPDIR'])) @@ -386,7 +386,7 @@ def _prepare_test_environment(ssh, virtio): [ "#!/bin/bash" ] + [ - "export {}={}".format(k, pipes.quote(v)) + "export {}={}".format(k, shlex.quote(v)) for k, v in vm_env.items() ] + [ '"$@"', diff --git a/cloud/tasks/acceptance_tests/recipe/tasks/chain_task.go b/cloud/tasks/acceptance_tests/recipe/tasks/chain_task.go index a76ac4328a2..301b72875e1 100644 --- a/cloud/tasks/acceptance_tests/recipe/tasks/chain_task.go +++ b/cloud/tasks/acceptance_tests/recipe/tasks/chain_task.go @@ -134,7 +134,6 @@ func (t *ChainTask) Cancel( func (t *ChainTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/tasks/acceptance_tests/recipe/tasks/panic_task.go b/cloud/tasks/acceptance_tests/recipe/tasks/panic_task.go index e7db44d80dc..ecdf1f95e03 100644 --- a/cloud/tasks/acceptance_tests/recipe/tasks/panic_task.go +++ b/cloud/tasks/acceptance_tests/recipe/tasks/panic_task.go @@ -41,7 +41,6 @@ func (t *PanicTask) Cancel( func (t *PanicTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/tasks/blank_task.go b/cloud/tasks/blank_task.go index 822975952bb..d3646478d32 100644 --- a/cloud/tasks/blank_task.go +++ b/cloud/tasks/blank_task.go @@ -37,7 +37,6 @@ func (t *blankTask) Cancel( func (t *blankTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/tasks/clear_ended_tasks_task.go b/cloud/tasks/clear_ended_tasks_task.go index 5f5d538e8ce..ba9b32dd054 100644 --- a/cloud/tasks/clear_ended_tasks_task.go +++ b/cloud/tasks/clear_ended_tasks_task.go @@ -44,7 +44,6 @@ func (t *clearEndedTasksTask) Cancel( func (t *clearEndedTasksTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/tasks/collect_lister_metrics_task.go b/cloud/tasks/collect_lister_metrics_task.go index 28d3da8d441..213105fc82a 100644 --- a/cloud/tasks/collect_lister_metrics_task.go +++ b/cloud/tasks/collect_lister_metrics_task.go @@ -107,7 +107,6 @@ func (c collectListerMetricsTask) Cancel( func (c collectListerMetricsTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/tasks/controller.go b/cloud/tasks/controller.go index edc9f743046..0a08669dd4f 100644 --- a/cloud/tasks/controller.go +++ b/cloud/tasks/controller.go @@ -102,8 +102,10 @@ func NewController( host string, ) Controller { + runContext := logging.WithComponent(ctx, logging.ComponentTaskRunner) + return &controller{ - runContext: ctx, + runContext: runContext, running: atomic.Bool{}, taskStorage: taskStorage, registry: registry, diff --git a/cloud/tasks/logging/fields.go b/cloud/tasks/logging/fields.go index 8cccdcf2e02..8a15d115c29 100644 --- a/cloud/tasks/logging/fields.go +++ b/cloud/tasks/logging/fields.go @@ -16,6 +16,10 @@ type Field = log.Field //////////////////////////////////////////////////////////////////////////////// const ComponentYDB = "YDB" +const ComponentS3 = "S3" +const ComponentTaskRunner = "TASK_RUNNER" +const ComponentTaskScheduler = "TASK_SCHEDULER" +const ComponentTask = "TASK" //////////////////////////////////////////////////////////////////////////////// @@ -61,22 +65,14 @@ func NewTaskIDField(value string) Field { return String("TASK_ID", value) } -func NewDiskIDField(value string) Field { - return String("DISK_ID", value) -} - -func NewSnapshotIDField(value string) Field { - return String("SNAPSHOT_ID", value) -} +//////////////////////////////////////////////////////////////////////////////// -func NewImageIDField(value string) Field { - return String("IMAGE_ID", value) +func WithComponent(ctx context.Context, value string) context.Context { + return WithFields(ctx, NewComponentField(value)) } -//////////////////////////////////////////////////////////////////////////////// - -func WithFields(ctx context.Context, fields ...Field) context.Context { - return ctxlog.WithFields(ctx, fields...) +func WithTaskID(ctx context.Context, value string) context.Context { + return WithFields(ctx, NewTaskIDField(value)) } func WithCommonFields(ctx context.Context) context.Context { @@ -101,3 +97,7 @@ func WithCommonFields(ctx context.Context) context.Context { return ctxlog.WithFields(ctx, fields...) } + +func WithFields(ctx context.Context, fields ...Field) context.Context { + return ctxlog.WithFields(ctx, fields...) +} diff --git a/cloud/tasks/logging/logger_test.go b/cloud/tasks/logging/logger_test.go index 2e2bf51c81f..61d326fd239 100644 --- a/cloud/tasks/logging/logger_test.go +++ b/cloud/tasks/logging/logger_test.go @@ -206,3 +206,34 @@ func TestLogFatal(t *testing.T) { Fatal(ctx, "Stuff happened") } + +func TestLogWithFields(t *testing.T) { + logger := createMockLogger() + ctx := SetLogger(newContext("idempID", "reqID", "opID"), logger) + + ctx = WithComponent(ctx, "component") + ctx = WithTaskID(ctx, "taskID") + + ctx = WithFields( + ctx, + log.String("field_1", "value_1"), + log.Int("field_2", 713), + ) + + logger.On( + "Info", + "Stuff happened", + []log.Field{ + log.String(idempotencyKeyKey, "idempID"), + log.String(requestIDKey, "reqID"), + log.String(operationIDKey, "opID"), + log.String(syslogIdentifierKey, "disk-manager"), + log.String("COMPONENT", "component"), + log.String("TASK_ID", "taskID"), + log.String("field_1", "value_1"), + log.Int("field_2", 713), + }, + ) + + Info(ctx, "Stuff happened") +} diff --git a/cloud/tasks/persistence/s3.go b/cloud/tasks/persistence/s3.go index eb64f3e066b..e76dc43a5c8 100644 --- a/cloud/tasks/persistence/s3.go +++ b/cloud/tasks/persistence/s3.go @@ -21,6 +21,12 @@ import ( //////////////////////////////////////////////////////////////////////////////// +func withComponentLoggingField(ctx context.Context) context.Context { + return logging.WithComponent(ctx, logging.ComponentS3) +} + +//////////////////////////////////////////////////////////////////////////////// + type S3Client struct { s3 *aws_s3.S3 callTimeout time.Duration @@ -92,6 +98,7 @@ func (c *S3Client) CreateBucket( bucket string, ) (err error) { + ctx = withComponentLoggingField(ctx) logging.Debug(ctx, "creating bucket %v in s3", bucket) ctx, cancel := context.WithTimeout(ctx, c.callTimeout) @@ -125,6 +132,7 @@ func (c *S3Client) GetObject( key string, ) (o S3Object, err error) { + ctx = withComponentLoggingField(ctx) logging.Debug(ctx, "getting object from s3, bucket %v, key %v", bucket, key) ctx, cancel := context.WithTimeout(ctx, c.callTimeout) @@ -170,6 +178,7 @@ func (c *S3Client) PutObject( object S3Object, ) (err error) { + ctx = withComponentLoggingField(ctx) logging.Debug(ctx, "putting object from s3, bucket %v, key %v", bucket, key) ctx, cancel := context.WithTimeout(ctx, c.callTimeout) @@ -204,6 +213,7 @@ func (c *S3Client) DeleteObject( key string, ) (err error) { + ctx = withComponentLoggingField(ctx) logging.Debug( ctx, "deleting object from s3, bucket %v, key %v", diff --git a/cloud/tasks/persistence/ydb_logger.go b/cloud/tasks/persistence/ydb_logger.go index a4df199aa6b..d487e0bb962 100644 --- a/cloud/tasks/persistence/ydb_logger.go +++ b/cloud/tasks/persistence/ydb_logger.go @@ -54,10 +54,7 @@ func (l *logger) Log(ctx context.Context, msg string, fields ...ydb_log.Field) { ctx = logging.SetLogger(ctx, l.logger) } - ctx = logging.WithFields( - ctx, - logging.NewComponentField(logging.ComponentYDB), - ) + ctx = logging.WithComponent(ctx, logging.ComponentYDB) if names := ydb_log.NamesFromContext(ctx); len(names) != 0 { ctx = logging.WithFields( ctx, diff --git a/cloud/tasks/runner.go b/cloud/tasks/runner.go index 3c8fb7cf421..379deff2af1 100644 --- a/cloud/tasks/runner.go +++ b/cloud/tasks/runner.go @@ -291,7 +291,13 @@ func (r *runnerForRun) run( } }() - return task.Run(ctx, execCtx) + return task.Run( + logging.WithTaskID( + logging.WithComponent(ctx, logging.ComponentTask), + execCtx.GetTaskID(), + ), + execCtx, + ) } func (r *runnerForRun) lockAndExecuteTask( @@ -360,7 +366,13 @@ func (r *runnerForCancel) executeTask( taskID, ) - err := task.Cancel(ctx, execCtx) + err := task.Cancel( + logging.WithTaskID( + logging.WithComponent(ctx, logging.ComponentTask), + execCtx.GetTaskID(), + ), + execCtx, + ) if ctx.Err() != nil { logging.Info( diff --git a/cloud/tasks/runner_test.go b/cloud/tasks/runner_test.go index 13dfeb85c92..6edae90fdb0 100644 --- a/cloud/tasks/runner_test.go +++ b/cloud/tasks/runner_test.go @@ -284,7 +284,7 @@ func TestRunnerForRun(t *testing.T) { State: []byte{2, 3, 4}, } task.On("Save").Return(state.State, nil) - task.On("Run", ctx, execCtx).Return(nil) + task.On("Run", mock.Anything, execCtx).Return(nil) taskStorage.On("UpdateTask", ctx, mock.MatchedBy(matchesState(t, state))).Return(state, nil) runnerMetrics := &mockRunnerMetrics{} @@ -304,7 +304,7 @@ func TestRunnerForRunCtxCancelled(t *testing.T) { Status: storage.TaskStatusReadyToRun, }) - task.On("Run", ctx, execCtx).Run(func(args mock.Arguments) { + task.On("Run", mock.Anything, execCtx).Run(func(args mock.Arguments) { cancel() }).Return(assert.AnError) @@ -329,7 +329,7 @@ func TestRunnerForRunGotAbortedError(t *testing.T) { err := errors.NewAbortedError(assert.AnError) - task.On("Run", ctx, execCtx).Run(func(args mock.Arguments) {}).Return(err) + task.On("Run", mock.Anything, execCtx).Run(func(args mock.Arguments) {}).Return(err) state := storage.TaskState{ ID: "taskId", @@ -358,7 +358,7 @@ func TestRunnerForRunGotPanic(t *testing.T) { Status: storage.TaskStatusRunning, }) - task.On("Run", ctx, execCtx).Run(func(args mock.Arguments) { + task.On("Run", mock.Anything, execCtx).Run(func(args mock.Arguments) { panic("test panic") }).Return(nil) @@ -389,7 +389,7 @@ func TestRunnerForRunPanicCountExceeded(t *testing.T) { PanicCount: 1, }) - task.On("Run", ctx, execCtx).Run(func(args mock.Arguments) { + task.On("Run", mock.Anything, execCtx).Run(func(args mock.Arguments) { panic("test panic") }).Return(nil) @@ -425,7 +425,7 @@ func TestRunnerForRunGotRetriableError(t *testing.T) { err := errors.NewRetriableError(assert.AnError) - task.On("Run", ctx, execCtx).Run(func(args mock.Arguments) {}).Return(err) + task.On("Run", mock.Anything, execCtx).Run(func(args mock.Arguments) {}).Return(err) state := storage.TaskState{ ID: "taskId", @@ -459,7 +459,7 @@ func TestRunnerForRunRetriableErrorCountExceeded(t *testing.T) { err := errors.NewRetriableError(assert.AnError) - task.On("Run", ctx, execCtx).Run(func(args mock.Arguments) {}).Return(err) + task.On("Run", mock.Anything, execCtx).Run(func(args mock.Arguments) {}).Return(err) state := storage.TaskState{ ID: "taskId", @@ -495,7 +495,7 @@ func TestRunnerForRunIgnoreRetryLimit(t *testing.T) { }) err := errors.NewRetriableErrorWithIgnoreRetryLimit(assert.AnError) - task.On("Run", ctx, execCtx).Run(func(args mock.Arguments) { + task.On("Run", mock.Anything, execCtx).Run(func(args mock.Arguments) { }).Return(err) state := storage.TaskState{ @@ -529,7 +529,7 @@ func TestRunnerForRunGotNonRetriableError1(t *testing.T) { // Non retriable error beats retriable error. failure := errors.NewNonRetriableError(errors.NewRetriableError(assert.AnError)) - task.On("Run", ctx, execCtx).Return(failure) + task.On("Run", mock.Anything, execCtx).Return(failure) state := storage.TaskState{ ID: "taskId", @@ -560,7 +560,7 @@ func TestRunnerForRunGotNonRetriableError2(t *testing.T) { // Non retriable error beats retriable error. failure := errors.NewRetriableError(errors.NewNonRetriableError(assert.AnError)) - task.On("Run", ctx, execCtx).Return(failure) + task.On("Run", mock.Anything, execCtx).Return(failure) state := storage.TaskState{ ID: "taskId", @@ -590,7 +590,7 @@ func TestRunnerForRunGotNonCancellableError(t *testing.T) { }) failure := errors.NewRetriableError(errors.NewNonCancellableError(assert.AnError)) - task.On("Run", ctx, execCtx).Return(failure) + task.On("Run", mock.Anything, execCtx).Return(failure) state := storage.TaskState{ ID: "taskId", @@ -621,7 +621,7 @@ func TestRunnerForRunWrongGeneration(t *testing.T) { err := errors.NewWrongGenerationError() - task.On("Run", ctx, execCtx).Return(err) + task.On("Run", mock.Anything, execCtx).Return(err) runnerMetrics := &mockRunnerMetrics{} runnerMetrics.On("OnExecutionError", err).Return().Once() @@ -644,7 +644,7 @@ func TestRunnerForRunWrongGenerationWrappedIntoRetriableError(t *testing.T) { err := errors.NewRetriableError(errors.NewWrongGenerationError()) - task.On("Run", ctx, execCtx).Return(err) + task.On("Run", mock.Anything, execCtx).Return(err) runnerMetrics := &mockRunnerMetrics{} runnerMetrics.On("OnExecutionError", err).Return().Once() @@ -669,7 +669,7 @@ func TestRunnerForRunInterruptExecution(t *testing.T) { err := errors.NewInterruptExecutionError() - task.On("Run", ctx, execCtx).Return(err) + task.On("Run", mock.Anything, execCtx).Return(err) runnerMetrics := &mockRunnerMetrics{} runnerMetrics.On("OnExecutionError", err).Return().Once() @@ -692,7 +692,7 @@ func TestRunnerForRunInterruptExecutionWrappedIntoRetriableError(t *testing.T) { err := errors.NewRetriableError(errors.NewInterruptExecutionError()) - task.On("Run", ctx, execCtx).Return(err) + task.On("Run", mock.Anything, execCtx).Return(err) runnerMetrics := &mockRunnerMetrics{} runnerMetrics.On("OnExecutionError", err).Return().Once() @@ -715,7 +715,7 @@ func TestRunnerForRunFailWithError(t *testing.T) { Status: storage.TaskStatusReadyToRun, }) - task.On("Run", ctx, execCtx).Return(assert.AnError) + task.On("Run", mock.Anything, execCtx).Return(assert.AnError) state := storage.TaskState{ ID: "taskId", Status: storage.TaskStatusReadyToCancel, @@ -751,7 +751,7 @@ func TestRunnerForCancel(t *testing.T) { State: []byte{2, 3, 4}, } task.On("Save").Return(state.State, nil) - task.On("Cancel", ctx, execCtx).Return(nil) + task.On("Cancel", mock.Anything, execCtx).Return(nil) taskStorage.On("UpdateTask", ctx, mock.MatchedBy(matchesState(t, state))).Return(state, nil) runnerMetrics := &mockRunnerMetrics{} @@ -771,7 +771,7 @@ func TestRunnerForCancelCtxCancelled(t *testing.T) { Status: storage.TaskStatusReadyToCancel, }) - task.On("Cancel", ctx, execCtx).Run(func(args mock.Arguments) { + task.On("Cancel", mock.Anything, execCtx).Run(func(args mock.Arguments) { cancel() }).Return(assert.AnError) @@ -794,7 +794,7 @@ func TestRunnerForCancelWrongGeneration(t *testing.T) { err := errors.NewWrongGenerationError() - task.On("Cancel", ctx, execCtx).Return(err) + task.On("Cancel", mock.Anything, execCtx).Return(err) runnerMetrics := &mockRunnerMetrics{} runnerMetrics.On("OnExecutionError", err).Return().Once() @@ -814,7 +814,7 @@ func TestRunnerForCancelFailWithError(t *testing.T) { Status: storage.TaskStatusReadyToCancel, }) - task.On("Cancel", ctx, execCtx).Return(assert.AnError) + task.On("Cancel", mock.Anything, execCtx).Return(assert.AnError) runnerMetrics := &mockRunnerMetrics{} runnerMetrics.On("OnExecutionError", assert.AnError).Return().Once() @@ -844,7 +844,7 @@ func TestRunnerForCancelGotNonRetriableError(t *testing.T) { State: []byte{2, 3, 4}, } task.On("Save").Return(state.State, nil) - task.On("Cancel", ctx, execCtx).Return(err) + task.On("Cancel", mock.Anything, execCtx).Return(err) taskStorage.On("UpdateTask", ctx, mock.MatchedBy(matchesState(t, state))).Return(state, nil) runnerMetrics := &mockRunnerMetrics{} @@ -875,7 +875,7 @@ func TestRunnerForCancelGotNonCancellableError(t *testing.T) { State: []byte{2, 3, 4}, } task.On("Save").Return(state.State, nil) - task.On("Cancel", ctx, execCtx).Return(err) + task.On("Cancel", mock.Anything, execCtx).Return(err) taskStorage.On("UpdateTask", ctx, mock.MatchedBy(matchesState(t, state))).Return(state, nil) runnerMetrics := &mockRunnerMetrics{} diff --git a/cloud/tasks/scheduler_impl.go b/cloud/tasks/scheduler_impl.go index ef0c87200e1..1a84de4d64c 100644 --- a/cloud/tasks/scheduler_impl.go +++ b/cloud/tasks/scheduler_impl.go @@ -22,6 +22,12 @@ import ( //////////////////////////////////////////////////////////////////////////////// +func withComponentLoggingField(ctx context.Context) context.Context { + return logging.WithComponent(ctx, logging.ComponentTaskScheduler) +} + +//////////////////////////////////////////////////////////////////////////////// + type scheduler struct { registry *Registry storage tasks_storage.Storage @@ -40,6 +46,7 @@ func (s *scheduler) ScheduleTask( folderID string, ) (string, error) { + ctx = withComponentLoggingField(ctx) return s.ScheduleZonalTask( ctx, taskType, @@ -61,6 +68,7 @@ func (s *scheduler) ScheduleZonalTask( folderID string, ) (string, error) { + ctx = withComponentLoggingField(ctx) logging.Debug(ctx, "scheduling task %v", taskType) marshalledRequest, err := proto.Marshal(request) @@ -120,6 +128,8 @@ func (s *scheduler) ScheduleRegularTasks( schedule TaskSchedule, ) { + ctx = withComponentLoggingField(ctx) + // TODO: Don't schedule new goroutine for each regular task type. go func() { for { @@ -185,6 +195,7 @@ func (s *scheduler) CancelTask( taskID string, ) (bool, error) { + ctx = withComponentLoggingField(ctx) logging.Info(ctx, "cancelling task %v", taskID) cancelling, err := s.storage.MarkForCancellation(ctx, taskID, time.Now()) @@ -238,6 +249,7 @@ func (s *scheduler) WaitTask( taskID string, ) (proto.Message, error) { + ctx = withComponentLoggingField(ctx) dependantTaskID := execCtx.GetTaskID() logging.Info(ctx, "waiting task %v by %v", taskID, dependantTaskID) @@ -256,6 +268,7 @@ func (s *scheduler) WaitAnyTasks( taskIDs []string, ) ([]string, error) { + ctx = withComponentLoggingField(ctx) timeout := time.After(s.taskWaitingTimeout) for { @@ -294,6 +307,7 @@ func (s *scheduler) WaitTaskEnded( taskID string, ) error { + ctx = withComponentLoggingField(ctx) timeout := time.After(s.taskWaitingTimeout) for { @@ -324,6 +338,7 @@ func (s *scheduler) GetTaskMetadata( taskID string, ) (proto.Message, error) { + ctx = withComponentLoggingField(ctx) logging.Debug(ctx, "getting task metadata %v", taskID) taskState, err := s.storage.GetTask(ctx, taskID) @@ -350,7 +365,7 @@ func (s *scheduler) GetTaskMetadata( return nil, err } - return task.GetMetadata(ctx, taskID) + return task.GetMetadata(ctx) } func (s *scheduler) SendEvent( @@ -359,6 +374,7 @@ func (s *scheduler) SendEvent( event int64, ) error { + ctx = withComponentLoggingField(ctx) return s.storage.SendEvent(ctx, taskID, event) } @@ -367,6 +383,7 @@ func (s *scheduler) GetOperation( taskID string, ) (*operation.Operation, error) { + ctx = withComponentLoggingField(ctx) logging.Debug(ctx, "getting operation proto %v", taskID) taskState, err := s.storage.GetTask(ctx, taskID) @@ -399,7 +416,7 @@ func (s *scheduler) GetOperation( return nil, err } - metadata, err := task.GetMetadata(ctx, taskID) + metadata, err := task.GetMetadata(ctx) if err != nil { logging.Warn(ctx, "failed to get task metadata %v: %v", taskID, err) return nil, err @@ -460,6 +477,7 @@ func (s *scheduler) WaitTaskSync( timeout time.Duration, ) (proto.Message, error) { + ctx = withComponentLoggingField(ctx) timeoutChannel := time.After(timeout) iteration := 0 diff --git a/cloud/tasks/scheduler_test.go b/cloud/tasks/scheduler_test.go index 1c8c25ac0b5..aefd3dbc992 100644 --- a/cloud/tasks/scheduler_test.go +++ b/cloud/tasks/scheduler_test.go @@ -136,7 +136,7 @@ func TestSchedulerScheduleTask(t *testing.T) { marshalledRequest, err := proto.Marshal(request) require.NoError(t, err) - storage.On("CreateTask", ctx, mock.MatchedBy(func(state tasks_storage.TaskState) bool { + storage.On("CreateTask", mock.Anything, mock.MatchedBy(func(state tasks_storage.TaskState) bool { ok := true ok = assert.Zero(t, state.ID) && ok ok = assert.Equal(t, "task", state.TaskType) && ok @@ -190,7 +190,7 @@ func TestSchedulerScheduleZonalTask(t *testing.T) { marshalledRequest, err := proto.Marshal(request) require.NoError(t, err) - storage.On("CreateTask", ctx, mock.MatchedBy(func(state tasks_storage.TaskState) bool { + storage.On("CreateTask", mock.Anything, mock.MatchedBy(func(state tasks_storage.TaskState) bool { ok := true ok = assert.Zero(t, state.ID) && ok ok = assert.Equal(t, "task", state.TaskType) && ok @@ -269,7 +269,7 @@ func TestSchedulerCancelTask(t *testing.T) { ) require.NoError(t, err) - storage.On("MarkForCancellation", ctx, "taskID", mock.Anything).Return(true, nil) + storage.On("MarkForCancellation", mock.Anything, "taskID", mock.Anything).Return(true, nil) cancelling, err := scheduler.CancelTask(ctx, "taskID") mock.AssertExpectationsForObjects(t, storage) @@ -298,12 +298,12 @@ func TestSchedulerGetTaskMetadata(t *testing.T) { taskID := "taskID" - storage.On("GetTask", ctx, taskID).Return(tasks_storage.TaskState{ + storage.On("GetTask", mock.Anything, taskID).Return(tasks_storage.TaskState{ TaskType: "task", Request: []byte{1, 2, 3}, }, nil) task.On("Load", []byte{1, 2, 3}, []byte(nil)).Return(nil) - task.On("GetMetadata", ctx, taskID).Return(&empty.Empty{}, nil) + task.On("GetMetadata", mock.Anything).Return(&empty.Empty{}, nil) metadata, err := scheduler.GetTaskMetadata(ctx, taskID) mock.AssertExpectationsForObjects(t, task, storage) @@ -332,7 +332,7 @@ func TestSchedulerGetOperationReadyToRun(t *testing.T) { taskID := "taskID" - storage.On("GetTask", ctx, taskID).Return(tasks_storage.TaskState{ + storage.On("GetTask", mock.Anything, taskID).Return(tasks_storage.TaskState{ TaskType: "task", Request: []byte{1, 2, 3}, CreatedAt: time.Unix(1, 2), @@ -343,7 +343,7 @@ func TestSchedulerGetOperationReadyToRun(t *testing.T) { Status: tasks_storage.TaskStatusReadyToRun, }, nil) task.On("Load", []byte{1, 2, 3}, []byte(nil)).Return(nil) - task.On("GetMetadata", ctx, taskID).Return(&empty.Empty{}, nil) + task.On("GetMetadata", mock.Anything).Return(&empty.Empty{}, nil) expectedMetadata, _ := ptypes.MarshalAny(&empty.Empty{}) @@ -388,7 +388,7 @@ func TestSchedulerGetOperationRunning(t *testing.T) { taskID := "taskID" - storage.On("GetTask", ctx, taskID).Return(tasks_storage.TaskState{ + storage.On("GetTask", mock.Anything, taskID).Return(tasks_storage.TaskState{ TaskType: "task", Request: []byte{1, 2, 3}, CreatedAt: time.Unix(1, 2), @@ -399,7 +399,7 @@ func TestSchedulerGetOperationRunning(t *testing.T) { Status: tasks_storage.TaskStatusRunning, }, nil) task.On("Load", []byte{1, 2, 3}, []byte(nil)).Return(nil) - task.On("GetMetadata", ctx, taskID).Return(&empty.Empty{}, nil) + task.On("GetMetadata", mock.Anything).Return(&empty.Empty{}, nil) expectedMetadata, _ := ptypes.MarshalAny(&empty.Empty{}) @@ -444,7 +444,7 @@ func TestSchedulerGetOperationFinished(t *testing.T) { taskID := "taskID" - storage.On("GetTask", ctx, taskID).Return(tasks_storage.TaskState{ + storage.On("GetTask", mock.Anything, taskID).Return(tasks_storage.TaskState{ TaskType: "task", Request: []byte{1, 2, 3}, CreatedAt: time.Unix(1, 2), @@ -455,7 +455,7 @@ func TestSchedulerGetOperationFinished(t *testing.T) { Status: tasks_storage.TaskStatusFinished, }, nil) task.On("Load", []byte{1, 2, 3}, []byte(nil)).Return(nil) - task.On("GetMetadata", ctx, taskID).Return(&empty.Empty{}, nil) + task.On("GetMetadata", mock.Anything).Return(&empty.Empty{}, nil) task.On("GetResponse").Return(&empty.Empty{}) expectedMetadata, _ := ptypes.MarshalAny(&empty.Empty{}) @@ -506,7 +506,7 @@ func TestSchedulerGetOperationReadyToCancel(t *testing.T) { taskError := grpc_status.New(grpc_codes.Canceled, "An error") taskID := "taskID" - storage.On("GetTask", ctx, taskID).Return(tasks_storage.TaskState{ + storage.On("GetTask", mock.Anything, taskID).Return(tasks_storage.TaskState{ TaskType: "task", Request: []byte{1, 2, 3}, CreatedAt: time.Unix(1, 2), @@ -519,7 +519,7 @@ func TestSchedulerGetOperationReadyToCancel(t *testing.T) { ErrorMessage: taskError.Message(), }, nil) task.On("Load", []byte{1, 2, 3}, []byte(nil)).Return(nil) - task.On("GetMetadata", ctx, taskID).Return(&empty.Empty{}, nil) + task.On("GetMetadata", mock.Anything).Return(&empty.Empty{}, nil) expectedMetadata, _ := ptypes.MarshalAny(&empty.Empty{}) @@ -568,7 +568,7 @@ func TestSchedulerGetOperationCancelling(t *testing.T) { taskError := grpc_status.New(grpc_codes.Canceled, "An error") taskID := "taskID" - storage.On("GetTask", ctx, taskID).Return(tasks_storage.TaskState{ + storage.On("GetTask", mock.Anything, taskID).Return(tasks_storage.TaskState{ TaskType: "task", Request: []byte{1, 2, 3}, CreatedAt: time.Unix(1, 2), @@ -581,7 +581,7 @@ func TestSchedulerGetOperationCancelling(t *testing.T) { ErrorMessage: taskError.Message(), }, nil) task.On("Load", []byte{1, 2, 3}, []byte(nil)).Return(nil) - task.On("GetMetadata", ctx, taskID).Return(&empty.Empty{}, nil) + task.On("GetMetadata", mock.Anything).Return(&empty.Empty{}, nil) expectedMetadata, _ := ptypes.MarshalAny(&empty.Empty{}) @@ -630,7 +630,7 @@ func TestSchedulerGetOperationCancelled(t *testing.T) { taskError := grpc_status.New(grpc_codes.Canceled, "An error") taskID := "taskID" - storage.On("GetTask", ctx, taskID).Return(tasks_storage.TaskState{ + storage.On("GetTask", mock.Anything, taskID).Return(tasks_storage.TaskState{ TaskType: "task", Request: []byte{1, 2, 3}, CreatedAt: time.Unix(1, 2), @@ -643,7 +643,7 @@ func TestSchedulerGetOperationCancelled(t *testing.T) { ErrorMessage: taskError.Message(), }, nil) task.On("Load", []byte{1, 2, 3}, []byte(nil)).Return(nil) - task.On("GetMetadata", ctx, taskID).Return(&empty.Empty{}, nil) + task.On("GetMetadata", mock.Anything).Return(&empty.Empty{}, nil) expectedMetadata, _ := ptypes.MarshalAny(&empty.Empty{}) diff --git a/cloud/tasks/storage/storage_ydb_impl.go b/cloud/tasks/storage/storage_ydb_impl.go index c44a4499274..2865dd2df01 100644 --- a/cloud/tasks/storage/storage_ydb_impl.go +++ b/cloud/tasks/storage/storage_ydb_impl.go @@ -711,7 +711,8 @@ func (s *storageYDB) createRegularTasks( if found { lastRunYear, lastRunMonth, lastRunDay := schState.scheduledAt.UTC().Date() - if year >= lastRunYear && month >= lastRunMonth && day > lastRunDay { + if year > lastRunYear || (year == lastRunYear && month > lastRunMonth) || + (year == lastRunYear && month == lastRunMonth && day > lastRunDay) { if hour >= schedule.Hour && min >= schedule.Min { shouldSchedule = true } diff --git a/cloud/tasks/task.go b/cloud/tasks/task.go index fa7437e392b..b89304cdb95 100644 --- a/cloud/tasks/task.go +++ b/cloud/tasks/task.go @@ -22,7 +22,7 @@ type Task interface { // Synchronously cancel the task. Cancel(ctx context.Context, execCtx ExecutionContext) error - GetMetadata(ctx context.Context, taskID string) (proto.Message, error) + GetMetadata(ctx context.Context) (proto.Message, error) // It only makes sense after Run has completed successfully. // But in that case it must not return nil. diff --git a/cloud/tasks/task_test.go b/cloud/tasks/task_test.go index 0146fd54d4c..40c313c2e5c 100644 --- a/cloud/tasks/task_test.go +++ b/cloud/tasks/task_test.go @@ -36,10 +36,9 @@ func (t *TaskMock) Cancel(ctx context.Context, execCtx ExecutionContext) error { func (t *TaskMock) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { - args := t.Called(ctx, taskID) + args := t.Called(ctx) res, _ := args.Get(0).(proto.Message) return res, args.Error(1) } diff --git a/cloud/tasks/tasks_tests/tasks_test.go b/cloud/tasks/tasks_tests/tasks_test.go index 43ad49f360c..74cbef744e7 100644 --- a/cloud/tasks/tasks_tests/tasks_test.go +++ b/cloud/tasks/tasks_tests/tasks_test.go @@ -232,7 +232,6 @@ func (t *doublerTask) Cancel(ctx context.Context, execCtx tasks.ExecutionContext func (t *doublerTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil @@ -286,7 +285,6 @@ func (t *longTask) Cancel(ctx context.Context, execCtx tasks.ExecutionContext) e func (t *longTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil @@ -359,7 +357,6 @@ func (t *unstableTask) Cancel(ctx context.Context, execCtx tasks.ExecutionContex func (t *unstableTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil @@ -417,7 +414,6 @@ func (*failureTask) Cancel(ctx context.Context, execCtx tasks.ExecutionContext) func (t *failureTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil @@ -497,7 +493,6 @@ func (t *sixTimesTask) Cancel(ctx context.Context, execCtx tasks.ExecutionContex func (t *sixTimesTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil @@ -561,7 +556,6 @@ func (t *regularTask) Cancel(ctx context.Context, execCtx tasks.ExecutionContext func (t *regularTask) GetMetadata( ctx context.Context, - taskID string, ) (proto.Message, error) { return &empty.Empty{}, nil diff --git a/cloud/vm/blockstore/bootstrap.cpp b/cloud/vm/blockstore/bootstrap.cpp index 5285e65a5a3..5d59478c517 100644 --- a/cloud/vm/blockstore/bootstrap.cpp +++ b/cloud/vm/blockstore/bootstrap.cpp @@ -288,10 +288,10 @@ void TBootstrap::InitClientConfig() if (Options) { if (!TryParseProtoTextFromString(Options, PluginConfig)) { // Legacy branch: parsing Options as config file path - // see https://st.yandex-team.ru/CLOUD-42448#5e9865329a07685298d47d78 + // see CLOUD-42448 if (!TryParseFromTextFormat(Options, PluginConfig)) { // Legacy branch: parsing config as TClientConfig/TClientAppConfig - // see https://st.yandex-team.ru/NBS-503#5d2353a334894f001dd36b4a + // see NBS-503 if (TryParseFromTextFormat(Options, appConfig)) { clientConfigParsed = true; } else if (TryParseFromTextFormat(Options, *appConfig.MutableClientConfig())) { diff --git a/cloud/vm/blockstore/lib/helper.cpp b/cloud/vm/blockstore/lib/helper.cpp index 3c1a4c370cc..4eadd13c2c9 100644 --- a/cloud/vm/blockstore/lib/helper.cpp +++ b/cloud/vm/blockstore/lib/helper.cpp @@ -10,19 +10,19 @@ namespace NCloud::NBlockStore::NPlugin { //////////////////////////////////////////////////////////////////////////////// NProto::TClientAppConfig ParseClientAppConfig( - const NProto::TPluginConfig& pluginConfig, - const TString& defaultClientConfigPath, + const NProto::TPluginConfig& pluginConfig, + const TString& defaultClientConfigPath, const TString& fallbackClientConfigPath) { NProto::TClientAppConfig appConfig; - + auto configPath = pluginConfig.GetClientConfig(); if (!configPath) { configPath = defaultClientConfigPath; } if (!TryParseFromTextFormat(configPath, appConfig)) { // Use default config for all clusters when deploying with k8s - // see https://st.yandex-team.ru/NBS-3250 + // see NBS-3250 if (!TryParseFromTextFormat(fallbackClientConfigPath, appConfig)) { ythrow yexception() << "can't parse client config from file"; } diff --git a/contrib/go/_std_1.21/src/net/cgo_stub.go b/contrib/go/_std_1.21/src/net/cgo_stub.go new file mode 100644 index 00000000000..b26b11af8b4 --- /dev/null +++ b/contrib/go/_std_1.21/src/net/cgo_stub.go @@ -0,0 +1,40 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file holds stub versions of the cgo functions called on Unix systems. +// We build this file: +// - if using the netgo build tag on a Unix system +// - on a Unix system without the cgo resolver functions +// (Darwin always provides the cgo functions, in cgo_unix_syscall.go) +// - on wasip1, where cgo is never available + +//go:build (netgo && unix) || (unix && !cgo && !darwin) || wasip1 + +package net + +import "context" + +// cgoAvailable set to false to indicate that the cgo resolver +// is not available on this system. +const cgoAvailable = false + +func cgoLookupHost(ctx context.Context, name string) (addrs []string, err error) { + panic("cgo stub: cgo not available") +} + +func cgoLookupPort(ctx context.Context, network, service string) (port int, err error) { + panic("cgo stub: cgo not available") +} + +func cgoLookupIP(ctx context.Context, network, name string) (addrs []IPAddr, err error) { + panic("cgo stub: cgo not available") +} + +func cgoLookupCNAME(ctx context.Context, name string) (cname string, err error, completed bool) { + panic("cgo stub: cgo not available") +} + +func cgoLookupPTR(ctx context.Context, addr string) (ptrs []string, err error) { + panic("cgo stub: cgo not available") +} diff --git a/contrib/go/_std_1.21/src/os/user/listgroups_unix.go b/contrib/go/_std_1.21/src/os/user/listgroups_unix.go new file mode 100644 index 00000000000..67bd8a776e2 --- /dev/null +++ b/contrib/go/_std_1.21/src/os/user/listgroups_unix.go @@ -0,0 +1,109 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ((darwin || dragonfly || freebsd || (js && wasm) || wasip1 || (!android && linux) || netbsd || openbsd || solaris) && ((!cgo && !darwin) || osusergo)) || aix || illumos + +package user + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "os" + "strconv" +) + +func listGroupsFromReader(u *User, r io.Reader) ([]string, error) { + if u.Username == "" { + return nil, errors.New("user: list groups: empty username") + } + primaryGid, err := strconv.Atoi(u.Gid) + if err != nil { + return nil, fmt.Errorf("user: list groups for %s: invalid gid %q", u.Username, u.Gid) + } + + userCommas := []byte("," + u.Username + ",") // ,john, + userFirst := userCommas[1:] // john, + userLast := userCommas[:len(userCommas)-1] // ,john + userOnly := userCommas[1 : len(userCommas)-1] // john + + // Add primary Gid first. + groups := []string{u.Gid} + + rd := bufio.NewReader(r) + done := false + for !done { + line, err := rd.ReadBytes('\n') + if err != nil { + if err == io.EOF { + done = true + } else { + return groups, err + } + } + + // Look for username in the list of users. If user is found, + // append the GID to the groups slice. + + // There's no spec for /etc/passwd or /etc/group, but we try to follow + // the same rules as the glibc parser, which allows comments and blank + // space at the beginning of a line. + line = bytes.TrimSpace(line) + if len(line) == 0 || line[0] == '#' || + // If you search for a gid in a row where the group + // name (the first field) starts with "+" or "-", + // glibc fails to find the record, and so should we. + line[0] == '+' || line[0] == '-' { + continue + } + + // Format of /etc/group is + // groupname:password:GID:user_list + // for example + // wheel:x:10:john,paul,jack + // tcpdump:x:72: + listIdx := bytes.LastIndexByte(line, ':') + if listIdx == -1 || listIdx == len(line)-1 { + // No commas, or empty group list. + continue + } + if bytes.Count(line[:listIdx], colon) != 2 { + // Incorrect number of colons. + continue + } + list := line[listIdx+1:] + // Check the list for user without splitting or copying. + if !(bytes.Equal(list, userOnly) || bytes.HasPrefix(list, userFirst) || bytes.HasSuffix(list, userLast) || bytes.Contains(list, userCommas)) { + continue + } + + // groupname:password:GID + parts := bytes.Split(line[:listIdx], colon) + if len(parts) != 3 || len(parts[0]) == 0 { + continue + } + gid := string(parts[2]) + // Make sure it's numeric and not the same as primary GID. + numGid, err := strconv.Atoi(gid) + if err != nil || numGid == primaryGid { + continue + } + + groups = append(groups, gid) + } + + return groups, nil +} + +func listGroups(u *User) ([]string, error) { + f, err := os.Open(groupFile) + if err != nil { + return nil, err + } + defer f.Close() + + return listGroupsFromReader(u, f) +} diff --git a/contrib/go/_std_1.21/src/os/user/lookup_stubs.go b/contrib/go/_std_1.21/src/os/user/lookup_stubs.go new file mode 100644 index 00000000000..89dfe455b54 --- /dev/null +++ b/contrib/go/_std_1.21/src/os/user/lookup_stubs.go @@ -0,0 +1,83 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (!cgo && !darwin && !windows && !plan9) || android || (osusergo && !windows && !plan9) + +package user + +import ( + "fmt" + "os" + "runtime" + "strconv" +) + +var ( + // unused variables (in this implementation) + // modified during test to exercise code paths in the cgo implementation. + userBuffer = 0 + groupBuffer = 0 +) + +func current() (*User, error) { + uid := currentUID() + // $USER and /etc/passwd may disagree; prefer the latter if we can get it. + // See issue 27524 for more information. + u, err := lookupUserId(uid) + if err == nil { + return u, nil + } + + homeDir, _ := os.UserHomeDir() + u = &User{ + Uid: uid, + Gid: currentGID(), + Username: os.Getenv("USER"), + Name: "", // ignored + HomeDir: homeDir, + } + // On Android, return a dummy user instead of failing. + switch runtime.GOOS { + case "android": + if u.Uid == "" { + u.Uid = "1" + } + if u.Username == "" { + u.Username = "android" + } + } + // cgo isn't available, but if we found the minimum information + // without it, use it: + if u.Uid != "" && u.Username != "" && u.HomeDir != "" { + return u, nil + } + var missing string + if u.Username == "" { + missing = "$USER" + } + if u.HomeDir == "" { + if missing != "" { + missing += ", " + } + missing += "$HOME" + } + return u, fmt.Errorf("user: Current requires cgo or %s set in environment", missing) +} + +func currentUID() string { + if id := os.Getuid(); id >= 0 { + return strconv.Itoa(id) + } + // Note: Windows returns -1, but this file isn't used on + // Windows anyway, so this empty return path shouldn't be + // used. + return "" +} + +func currentGID() string { + if id := os.Getgid(); id >= 0 { + return strconv.Itoa(id) + } + return "" +} diff --git a/contrib/go/_std_1.21/src/os/user/lookup_unix.go b/contrib/go/_std_1.21/src/os/user/lookup_unix.go new file mode 100644 index 00000000000..a4308269e04 --- /dev/null +++ b/contrib/go/_std_1.21/src/os/user/lookup_unix.go @@ -0,0 +1,234 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ((unix && !android) || (js && wasm) || wasip1) && ((!cgo && !darwin) || osusergo) + +package user + +import ( + "bufio" + "bytes" + "errors" + "io" + "os" + "strconv" + "strings" +) + +// lineFunc returns a value, an error, or (nil, nil) to skip the row. +type lineFunc func(line []byte) (v any, err error) + +// readColonFile parses r as an /etc/group or /etc/passwd style file, running +// fn for each row. readColonFile returns a value, an error, or (nil, nil) if +// the end of the file is reached without a match. +// +// readCols is the minimum number of colon-separated fields that will be passed +// to fn; in a long line additional fields may be silently discarded. +func readColonFile(r io.Reader, fn lineFunc, readCols int) (v any, err error) { + rd := bufio.NewReader(r) + + // Read the file line-by-line. + for { + var isPrefix bool + var wholeLine []byte + + // Read the next line. We do so in chunks (as much as reader's + // buffer is able to keep), check if we read enough columns + // already on each step and store final result in wholeLine. + for { + var line []byte + line, isPrefix, err = rd.ReadLine() + + if err != nil { + // We should return (nil, nil) if EOF is reached + // without a match. + if err == io.EOF { + err = nil + } + return nil, err + } + + // Simple common case: line is short enough to fit in a + // single reader's buffer. + if !isPrefix && len(wholeLine) == 0 { + wholeLine = line + break + } + + wholeLine = append(wholeLine, line...) + + // Check if we read the whole line (or enough columns) + // already. + if !isPrefix || bytes.Count(wholeLine, []byte{':'}) >= readCols { + break + } + } + + // There's no spec for /etc/passwd or /etc/group, but we try to follow + // the same rules as the glibc parser, which allows comments and blank + // space at the beginning of a line. + wholeLine = bytes.TrimSpace(wholeLine) + if len(wholeLine) == 0 || wholeLine[0] == '#' { + continue + } + v, err = fn(wholeLine) + if v != nil || err != nil { + return + } + + // If necessary, skip the rest of the line + for ; isPrefix; _, isPrefix, err = rd.ReadLine() { + if err != nil { + // We should return (nil, nil) if EOF is reached without a match. + if err == io.EOF { + err = nil + } + return nil, err + } + } + } +} + +func matchGroupIndexValue(value string, idx int) lineFunc { + var leadColon string + if idx > 0 { + leadColon = ":" + } + substr := []byte(leadColon + value + ":") + return func(line []byte) (v any, err error) { + if !bytes.Contains(line, substr) || bytes.Count(line, colon) < 3 { + return + } + // wheel:*:0:root + parts := strings.SplitN(string(line), ":", 4) + if len(parts) < 4 || parts[0] == "" || parts[idx] != value || + // If the file contains +foo and you search for "foo", glibc + // returns an "invalid argument" error. Similarly, if you search + // for a gid for a row where the group name starts with "+" or "-", + // glibc fails to find the record. + parts[0][0] == '+' || parts[0][0] == '-' { + return + } + if _, err := strconv.Atoi(parts[2]); err != nil { + return nil, nil + } + return &Group{Name: parts[0], Gid: parts[2]}, nil + } +} + +func findGroupId(id string, r io.Reader) (*Group, error) { + if v, err := readColonFile(r, matchGroupIndexValue(id, 2), 3); err != nil { + return nil, err + } else if v != nil { + return v.(*Group), nil + } + return nil, UnknownGroupIdError(id) +} + +func findGroupName(name string, r io.Reader) (*Group, error) { + if v, err := readColonFile(r, matchGroupIndexValue(name, 0), 3); err != nil { + return nil, err + } else if v != nil { + return v.(*Group), nil + } + return nil, UnknownGroupError(name) +} + +// returns a *User for a row if that row's has the given value at the +// given index. +func matchUserIndexValue(value string, idx int) lineFunc { + var leadColon string + if idx > 0 { + leadColon = ":" + } + substr := []byte(leadColon + value + ":") + return func(line []byte) (v any, err error) { + if !bytes.Contains(line, substr) || bytes.Count(line, colon) < 6 { + return + } + // kevin:x:1005:1006::/home/kevin:/usr/bin/zsh + parts := strings.SplitN(string(line), ":", 7) + if len(parts) < 6 || parts[idx] != value || parts[0] == "" || + parts[0][0] == '+' || parts[0][0] == '-' { + return + } + if _, err := strconv.Atoi(parts[2]); err != nil { + return nil, nil + } + if _, err := strconv.Atoi(parts[3]); err != nil { + return nil, nil + } + u := &User{ + Username: parts[0], + Uid: parts[2], + Gid: parts[3], + Name: parts[4], + HomeDir: parts[5], + } + // The pw_gecos field isn't quite standardized. Some docs + // say: "It is expected to be a comma separated list of + // personal data where the first item is the full name of the + // user." + u.Name, _, _ = strings.Cut(u.Name, ",") + return u, nil + } +} + +func findUserId(uid string, r io.Reader) (*User, error) { + i, e := strconv.Atoi(uid) + if e != nil { + return nil, errors.New("user: invalid userid " + uid) + } + if v, err := readColonFile(r, matchUserIndexValue(uid, 2), 6); err != nil { + return nil, err + } else if v != nil { + return v.(*User), nil + } + return nil, UnknownUserIdError(i) +} + +func findUsername(name string, r io.Reader) (*User, error) { + if v, err := readColonFile(r, matchUserIndexValue(name, 0), 6); err != nil { + return nil, err + } else if v != nil { + return v.(*User), nil + } + return nil, UnknownUserError(name) +} + +func lookupGroup(groupname string) (*Group, error) { + f, err := os.Open(groupFile) + if err != nil { + return nil, err + } + defer f.Close() + return findGroupName(groupname, f) +} + +func lookupGroupId(id string) (*Group, error) { + f, err := os.Open(groupFile) + if err != nil { + return nil, err + } + defer f.Close() + return findGroupId(id, f) +} + +func lookupUser(username string) (*User, error) { + f, err := os.Open(userFile) + if err != nil { + return nil, err + } + defer f.Close() + return findUsername(username, f) +} + +func lookupUserId(uid string) (*User, error) { + f, err := os.Open(userFile) + if err != nil { + return nil, err + } + defer f.Close() + return findUserId(uid, f) +} diff --git a/contrib/python/cffi/py2/lsan.supp b/contrib/python/cffi/py2/lsan.supp new file mode 100644 index 00000000000..8533ad0d29b --- /dev/null +++ b/contrib/python/cffi/py2/lsan.supp @@ -0,0 +1,2 @@ +leak:b_init_cffi_1_0_external_module +leak:lib_build_and_cache_attr diff --git a/contrib/python/cffi/py3/lsan.supp b/contrib/python/cffi/py3/lsan.supp new file mode 100644 index 00000000000..8533ad0d29b --- /dev/null +++ b/contrib/python/cffi/py3/lsan.supp @@ -0,0 +1,2 @@ +leak:b_init_cffi_1_0_external_module +leak:lib_build_and_cache_attr diff --git a/contrib/tools/python/lib/lsan.supp b/contrib/tools/python/lib/lsan.supp new file mode 100644 index 00000000000..a7c29ccb39c --- /dev/null +++ b/contrib/tools/python/lib/lsan.supp @@ -0,0 +1,8 @@ +leak:_PyGILState_Init +leak:_PyImport_AcquireLock +leak:_PyImport_ReInitLock +leak:Py_MakePendingCalls +leak:PyEval_InitThreads +leak:PyEval_ReInitThreads +leak:PyString_FromStringAndSize +leak:PyThread_ReInitTLS diff --git a/contrib/tools/python3/lib/lsan.supp b/contrib/tools/python3/lib/lsan.supp new file mode 100644 index 00000000000..37121682e78 --- /dev/null +++ b/contrib/tools/python3/lib/lsan.supp @@ -0,0 +1,2 @@ +leak:PyBytes_FromStringAndSize +leak:PyUnicode_New diff --git a/contrib/ydb/apps/version/version_definition.cpp b/contrib/ydb/apps/version/version_definition.cpp new file mode 100644 index 00000000000..8ee62418fe6 --- /dev/null +++ b/contrib/ydb/apps/version/version_definition.cpp @@ -0,0 +1,10 @@ +#include + +NKikimrConfig::TCurrentCompatibilityInfo NKikimr::TCompatibilityInfo::MakeCurrent() { + using TCurrentConstructor = NKikimr::TCompatibilityInfo::TProtoConstructor::TCurrentCompatibilityInfo; + + return TCurrentConstructor{ + .Application = "ydb", + }.ToPB(); +} + diff --git a/contrib/ydb/apps/version/ya.make b/contrib/ydb/apps/version/ya.make new file mode 100644 index 00000000000..a952b9b0e7a --- /dev/null +++ b/contrib/ydb/apps/version/ya.make @@ -0,0 +1,12 @@ +LIBRARY(version_definition) + +SRCS( + version_definition.cpp +) + +PEERDIR( + contrib/ydb/core/driver_lib/version +) + +END() + diff --git a/contrib/ydb/apps/ydbd/ya.make b/contrib/ydb/apps/ydbd/ya.make index 3331cd3894c..619d6c22a69 100644 --- a/contrib/ydb/apps/ydbd/ya.make +++ b/contrib/ydb/apps/ydbd/ya.make @@ -35,6 +35,7 @@ IF (ARCH_X86_64) ENDIF() PEERDIR( + contrib/ydb/apps/version contrib/ydb/core/driver_lib/run contrib/ydb/core/protos contrib/ydb/core/security diff --git a/contrib/ydb/core/driver_lib/version/ut/ya.make b/contrib/ydb/core/driver_lib/version/ut/ya.make index d0b05c45c70..1de808f3459 100644 --- a/contrib/ydb/core/driver_lib/version/ut/ya.make +++ b/contrib/ydb/core/driver_lib/version/ut/ya.make @@ -6,6 +6,7 @@ TIMEOUT(300) SIZE(MEDIUM) PEERDIR( + contrib/ydb/apps/version contrib/ydb/core/driver_lib/version ) diff --git a/contrib/ydb/core/driver_lib/version/version.cpp b/contrib/ydb/core/driver_lib/version/version.cpp index 8959eb9b457..2741a450576 100644 --- a/contrib/ydb/core/driver_lib/version/version.cpp +++ b/contrib/ydb/core/driver_lib/version/version.cpp @@ -17,18 +17,14 @@ using EComponentId = NKikimrConfig::TCompatibilityRule; using TComponentId = NKikimrConfig::TCompatibilityRule::EComponentId; TCompatibilityInfo::TCompatibilityInfo() { - using TCurrentConstructor = TCompatibilityInfo::TProtoConstructor::TCurrentCompatibilityInfo; using TStoredConstructor = TCompatibilityInfo::TProtoConstructor::TStoredCompatibilityInfo; - // using TCompatibilityRuleConstructor = TCompatibilityInfo::TProtoConstructor::TCompatibilityRule; using TVersionConstructor = TCompatibilityInfo::TProtoConstructor::TVersion; ///////////////////////////////////////////////////////// // Current CompatibilityInfo ///////////////////////////////////////////////////////// - auto current = TCurrentConstructor{ - .Application = "ydb" - }.ToPB(); + auto current = MakeCurrent(); // bool success = CompleteFromTag(current); // Y_ABORT_UNLESS(success); diff --git a/contrib/ydb/core/driver_lib/version/version.h b/contrib/ydb/core/driver_lib/version/version.h index 9c2d9bf38f7..b4587b73663 100644 --- a/contrib/ydb/core/driver_lib/version/version.h +++ b/contrib/ydb/core/driver_lib/version/version.h @@ -141,6 +141,8 @@ class TCompatibilityInfo { bool CompleteFromTag(NKikimrConfig::TCurrentCompatibilityInfo& current); + static NKikimrConfig::TCurrentCompatibilityInfo MakeCurrent(); + NKikimrConfig::TStoredCompatibilityInfo MakeStored(TComponentId componentId) const; NKikimrConfig::TStoredCompatibilityInfo MakeStored(TComponentId componentId, const NKikimrConfig::TCurrentCompatibilityInfo* current) const; diff --git a/contrib/ydb/core/testlib/actors/ya.make b/contrib/ydb/core/testlib/actors/ya.make index 51cde256f85..363f1aed249 100644 --- a/contrib/ydb/core/testlib/actors/ya.make +++ b/contrib/ydb/core/testlib/actors/ya.make @@ -5,6 +5,7 @@ SRCS( ) PEERDIR( + contrib/ydb/apps/version contrib/ydb/library/actors/testlib library/cpp/testing/unittest contrib/ydb/core/base diff --git a/contrib/ydb/core/testlib/ya.make b/contrib/ydb/core/testlib/ya.make index 7b1c93a5b51..f31f53d7d64 100644 --- a/contrib/ydb/core/testlib/ya.make +++ b/contrib/ydb/core/testlib/ya.make @@ -22,6 +22,7 @@ SRCS( ) PEERDIR( + contrib/ydb/apps/version contrib/ydb/library/actors/core contrib/ydb/library/actors/interconnect contrib/ydb/library/grpc/client diff --git a/doc/blockstore/overview/overview.drawio b/doc/blockstore/overview/overview.drawio new file mode 100644 index 00000000000..8ab45838380 --- /dev/null +++ b/doc/blockstore/overview/overview.drawio @@ -0,0 +1,1034 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/blockstore/overview/overview.png b/doc/blockstore/overview/overview.png new file mode 100644 index 00000000000..32d0140dec7 Binary files /dev/null and b/doc/blockstore/overview/overview.png differ diff --git a/example/nbs/nbs-storage.txt b/example/nbs/nbs-storage.txt index e5f51ee0421..7e85bed4f39 100644 --- a/example/nbs/nbs-storage.txt +++ b/example/nbs/nbs-storage.txt @@ -6,3 +6,4 @@ NonReplicatedMigrationStartAllowed: true AllocationUnitMirror2SSD: 1 AllocationUnitMirror3SSD: 1 NonReplicatedDontSuspendDevices: true +UseShadowDisksForNonreplDiskCheckpoints: true diff --git a/vendor/github.com/container-storage-interface/spec/LICENSE b/vendor/github.com/container-storage-interface/spec/LICENSE new file mode 100644 index 00000000000..8dada3edaf5 --- /dev/null +++ b/vendor/github.com/container-storage-interface/spec/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go new file mode 100644 index 00000000000..ad6cc651ca8 --- /dev/null +++ b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go @@ -0,0 +1,7393 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: github.com/container-storage-interface/spec/csi.proto + +package csi + +import ( + context "context" + fmt "fmt" + proto "github.com/golang/protobuf/proto" + descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" + timestamp "github.com/golang/protobuf/ptypes/timestamp" + wrappers "github.com/golang/protobuf/ptypes/wrappers" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +type PluginCapability_Service_Type int32 + +const ( + PluginCapability_Service_UNKNOWN PluginCapability_Service_Type = 0 + // CONTROLLER_SERVICE indicates that the Plugin provides RPCs for + // the ControllerService. Plugins SHOULD provide this capability. + // In rare cases certain plugins MAY wish to omit the + // ControllerService entirely from their implementation, but such + // SHOULD NOT be the common case. + // The presence of this capability determines whether the CO will + // attempt to invoke the REQUIRED ControllerService RPCs, as well + // as specific RPCs as indicated by ControllerGetCapabilities. + PluginCapability_Service_CONTROLLER_SERVICE PluginCapability_Service_Type = 1 + // VOLUME_ACCESSIBILITY_CONSTRAINTS indicates that the volumes for + // this plugin MAY NOT be equally accessible by all nodes in the + // cluster. The CO MUST use the topology information returned by + // CreateVolumeRequest along with the topology information + // returned by NodeGetInfo to ensure that a given volume is + // accessible from a given node when scheduling workloads. + PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS PluginCapability_Service_Type = 2 + // GROUP_CONTROLLER_SERVICE indicates that the Plugin provides + // RPCs for operating on groups of volumes. Plugins MAY provide + // this capability. + // The presence of this capability determines whether the CO will + // attempt to invoke the REQUIRED GroupController service RPCs, as + // well as specific RPCs as indicated by + // GroupControllerGetCapabilities. + PluginCapability_Service_GROUP_CONTROLLER_SERVICE PluginCapability_Service_Type = 3 +) + +var PluginCapability_Service_Type_name = map[int32]string{ + 0: "UNKNOWN", + 1: "CONTROLLER_SERVICE", + 2: "VOLUME_ACCESSIBILITY_CONSTRAINTS", + 3: "GROUP_CONTROLLER_SERVICE", +} + +var PluginCapability_Service_Type_value = map[string]int32{ + "UNKNOWN": 0, + "CONTROLLER_SERVICE": 1, + "VOLUME_ACCESSIBILITY_CONSTRAINTS": 2, + "GROUP_CONTROLLER_SERVICE": 3, +} + +func (x PluginCapability_Service_Type) String() string { + return proto.EnumName(PluginCapability_Service_Type_name, int32(x)) +} + +func (PluginCapability_Service_Type) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{4, 0, 0} +} + +type PluginCapability_VolumeExpansion_Type int32 + +const ( + PluginCapability_VolumeExpansion_UNKNOWN PluginCapability_VolumeExpansion_Type = 0 + // ONLINE indicates that volumes may be expanded when published to + // a node. When a Plugin implements this capability it MUST + // implement either the EXPAND_VOLUME controller capability or the + // EXPAND_VOLUME node capability or both. When a plugin supports + // ONLINE volume expansion and also has the EXPAND_VOLUME + // controller capability then the plugin MUST support expansion of + // volumes currently published and available on a node. When a + // plugin supports ONLINE volume expansion and also has the + // EXPAND_VOLUME node capability then the plugin MAY support + // expansion of node-published volume via NodeExpandVolume. + // + // Example 1: Given a shared filesystem volume (e.g. GlusterFs), + // + // the Plugin may set the ONLINE volume expansion capability and + // implement ControllerExpandVolume but not NodeExpandVolume. + // + // Example 2: Given a block storage volume type (e.g. EBS), the + // + // Plugin may set the ONLINE volume expansion capability and + // implement both ControllerExpandVolume and NodeExpandVolume. + // + // Example 3: Given a Plugin that supports volume expansion only + // + // upon a node, the Plugin may set the ONLINE volume + // expansion capability and implement NodeExpandVolume but not + // ControllerExpandVolume. + PluginCapability_VolumeExpansion_ONLINE PluginCapability_VolumeExpansion_Type = 1 + // OFFLINE indicates that volumes currently published and + // available on a node SHALL NOT be expanded via + // ControllerExpandVolume. When a plugin supports OFFLINE volume + // expansion it MUST implement either the EXPAND_VOLUME controller + // capability or both the EXPAND_VOLUME controller capability and + // the EXPAND_VOLUME node capability. + // + // Example 1: Given a block storage volume type (e.g. Azure Disk) + // + // that does not support expansion of "node-attached" (i.e. + // controller-published) volumes, the Plugin may indicate + // OFFLINE volume expansion support and implement both + // ControllerExpandVolume and NodeExpandVolume. + PluginCapability_VolumeExpansion_OFFLINE PluginCapability_VolumeExpansion_Type = 2 +) + +var PluginCapability_VolumeExpansion_Type_name = map[int32]string{ + 0: "UNKNOWN", + 1: "ONLINE", + 2: "OFFLINE", +} + +var PluginCapability_VolumeExpansion_Type_value = map[string]int32{ + "UNKNOWN": 0, + "ONLINE": 1, + "OFFLINE": 2, +} + +func (x PluginCapability_VolumeExpansion_Type) String() string { + return proto.EnumName(PluginCapability_VolumeExpansion_Type_name, int32(x)) +} + +func (PluginCapability_VolumeExpansion_Type) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{4, 1, 0} +} + +type VolumeCapability_AccessMode_Mode int32 + +const ( + VolumeCapability_AccessMode_UNKNOWN VolumeCapability_AccessMode_Mode = 0 + // Can only be published once as read/write on a single node, at + // any given time. + VolumeCapability_AccessMode_SINGLE_NODE_WRITER VolumeCapability_AccessMode_Mode = 1 + // Can only be published once as readonly on a single node, at + // any given time. + VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY VolumeCapability_AccessMode_Mode = 2 + // Can be published as readonly at multiple nodes simultaneously. + VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY VolumeCapability_AccessMode_Mode = 3 + // Can be published at multiple nodes simultaneously. Only one of + // the node can be used as read/write. The rest will be readonly. + VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER VolumeCapability_AccessMode_Mode = 4 + // Can be published as read/write at multiple nodes + // simultaneously. + VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER VolumeCapability_AccessMode_Mode = 5 + // Can only be published once as read/write at a single workload + // on a single node, at any given time. SHOULD be used instead of + // SINGLE_NODE_WRITER for COs using the experimental + // SINGLE_NODE_MULTI_WRITER capability. + VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER VolumeCapability_AccessMode_Mode = 6 + // Can be published as read/write at multiple workloads on a + // single node simultaneously. SHOULD be used instead of + // SINGLE_NODE_WRITER for COs using the experimental + // SINGLE_NODE_MULTI_WRITER capability. + VolumeCapability_AccessMode_SINGLE_NODE_MULTI_WRITER VolumeCapability_AccessMode_Mode = 7 +) + +var VolumeCapability_AccessMode_Mode_name = map[int32]string{ + 0: "UNKNOWN", + 1: "SINGLE_NODE_WRITER", + 2: "SINGLE_NODE_READER_ONLY", + 3: "MULTI_NODE_READER_ONLY", + 4: "MULTI_NODE_SINGLE_WRITER", + 5: "MULTI_NODE_MULTI_WRITER", + 6: "SINGLE_NODE_SINGLE_WRITER", + 7: "SINGLE_NODE_MULTI_WRITER", +} + +var VolumeCapability_AccessMode_Mode_value = map[string]int32{ + "UNKNOWN": 0, + "SINGLE_NODE_WRITER": 1, + "SINGLE_NODE_READER_ONLY": 2, + "MULTI_NODE_READER_ONLY": 3, + "MULTI_NODE_SINGLE_WRITER": 4, + "MULTI_NODE_MULTI_WRITER": 5, + "SINGLE_NODE_SINGLE_WRITER": 6, + "SINGLE_NODE_MULTI_WRITER": 7, +} + +func (x VolumeCapability_AccessMode_Mode) String() string { + return proto.EnumName(VolumeCapability_AccessMode_Mode_name, int32(x)) +} + +func (VolumeCapability_AccessMode_Mode) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{10, 2, 0} +} + +type ControllerServiceCapability_RPC_Type int32 + +const ( + ControllerServiceCapability_RPC_UNKNOWN ControllerServiceCapability_RPC_Type = 0 + ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME ControllerServiceCapability_RPC_Type = 1 + ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME ControllerServiceCapability_RPC_Type = 2 + ControllerServiceCapability_RPC_LIST_VOLUMES ControllerServiceCapability_RPC_Type = 3 + ControllerServiceCapability_RPC_GET_CAPACITY ControllerServiceCapability_RPC_Type = 4 + // Currently the only way to consume a snapshot is to create + // a volume from it. Therefore plugins supporting + // CREATE_DELETE_SNAPSHOT MUST support creating volume from + // snapshot. + ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT ControllerServiceCapability_RPC_Type = 5 + ControllerServiceCapability_RPC_LIST_SNAPSHOTS ControllerServiceCapability_RPC_Type = 6 + // Plugins supporting volume cloning at the storage level MAY + // report this capability. The source volume MUST be managed by + // the same plugin. Not all volume sources and parameters + // combinations MAY work. + ControllerServiceCapability_RPC_CLONE_VOLUME ControllerServiceCapability_RPC_Type = 7 + // Indicates the SP supports ControllerPublishVolume.readonly + // field. + ControllerServiceCapability_RPC_PUBLISH_READONLY ControllerServiceCapability_RPC_Type = 8 + // See VolumeExpansion for details. + ControllerServiceCapability_RPC_EXPAND_VOLUME ControllerServiceCapability_RPC_Type = 9 + // Indicates the SP supports the + // ListVolumesResponse.entry.published_node_ids field and the + // ControllerGetVolumeResponse.published_node_ids field. + // The SP MUST also support PUBLISH_UNPUBLISH_VOLUME. + ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES ControllerServiceCapability_RPC_Type = 10 + // Indicates that the Controller service can report volume + // conditions. + // An SP MAY implement `VolumeCondition` in only the Controller + // Plugin, only the Node Plugin, or both. + // If `VolumeCondition` is implemented in both the Controller and + // Node Plugins, it SHALL report from different perspectives. + // If for some reason Controller and Node Plugins report + // misaligned volume conditions, CO SHALL assume the worst case + // is the truth. + // Note that, for alpha, `VolumeCondition` is intended be + // informative for humans only, not for automation. + ControllerServiceCapability_RPC_VOLUME_CONDITION ControllerServiceCapability_RPC_Type = 11 + // Indicates the SP supports the ControllerGetVolume RPC. + // This enables COs to, for example, fetch per volume + // condition after a volume is provisioned. + ControllerServiceCapability_RPC_GET_VOLUME ControllerServiceCapability_RPC_Type = 12 + // Indicates the SP supports the SINGLE_NODE_SINGLE_WRITER and/or + // SINGLE_NODE_MULTI_WRITER access modes. + // These access modes are intended to replace the + // SINGLE_NODE_WRITER access mode to clarify the number of writers + // for a volume on a single node. Plugins MUST accept and allow + // use of the SINGLE_NODE_WRITER access mode when either + // SINGLE_NODE_SINGLE_WRITER and/or SINGLE_NODE_MULTI_WRITER are + // supported, in order to permit older COs to continue working. + ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER ControllerServiceCapability_RPC_Type = 13 + // Indicates the SP supports modifying volume with mutable + // parameters. See ControllerModifyVolume for details. + ControllerServiceCapability_RPC_MODIFY_VOLUME ControllerServiceCapability_RPC_Type = 14 +) + +var ControllerServiceCapability_RPC_Type_name = map[int32]string{ + 0: "UNKNOWN", + 1: "CREATE_DELETE_VOLUME", + 2: "PUBLISH_UNPUBLISH_VOLUME", + 3: "LIST_VOLUMES", + 4: "GET_CAPACITY", + 5: "CREATE_DELETE_SNAPSHOT", + 6: "LIST_SNAPSHOTS", + 7: "CLONE_VOLUME", + 8: "PUBLISH_READONLY", + 9: "EXPAND_VOLUME", + 10: "LIST_VOLUMES_PUBLISHED_NODES", + 11: "VOLUME_CONDITION", + 12: "GET_VOLUME", + 13: "SINGLE_NODE_MULTI_WRITER", + 14: "MODIFY_VOLUME", +} + +var ControllerServiceCapability_RPC_Type_value = map[string]int32{ + "UNKNOWN": 0, + "CREATE_DELETE_VOLUME": 1, + "PUBLISH_UNPUBLISH_VOLUME": 2, + "LIST_VOLUMES": 3, + "GET_CAPACITY": 4, + "CREATE_DELETE_SNAPSHOT": 5, + "LIST_SNAPSHOTS": 6, + "CLONE_VOLUME": 7, + "PUBLISH_READONLY": 8, + "EXPAND_VOLUME": 9, + "LIST_VOLUMES_PUBLISHED_NODES": 10, + "VOLUME_CONDITION": 11, + "GET_VOLUME": 12, + "SINGLE_NODE_MULTI_WRITER": 13, + "MODIFY_VOLUME": 14, +} + +func (x ControllerServiceCapability_RPC_Type) String() string { + return proto.EnumName(ControllerServiceCapability_RPC_Type_name, int32(x)) +} + +func (ControllerServiceCapability_RPC_Type) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{33, 0, 0} +} + +type VolumeUsage_Unit int32 + +const ( + VolumeUsage_UNKNOWN VolumeUsage_Unit = 0 + VolumeUsage_BYTES VolumeUsage_Unit = 1 + VolumeUsage_INODES VolumeUsage_Unit = 2 +) + +var VolumeUsage_Unit_name = map[int32]string{ + 0: "UNKNOWN", + 1: "BYTES", + 2: "INODES", +} + +var VolumeUsage_Unit_value = map[string]int32{ + "UNKNOWN": 0, + "BYTES": 1, + "INODES": 2, +} + +func (x VolumeUsage_Unit) String() string { + return proto.EnumName(VolumeUsage_Unit_name, int32(x)) +} + +func (VolumeUsage_Unit) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{53, 0} +} + +type NodeServiceCapability_RPC_Type int32 + +const ( + NodeServiceCapability_RPC_UNKNOWN NodeServiceCapability_RPC_Type = 0 + NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME NodeServiceCapability_RPC_Type = 1 + // If Plugin implements GET_VOLUME_STATS capability + // then it MUST implement NodeGetVolumeStats RPC + // call for fetching volume statistics. + NodeServiceCapability_RPC_GET_VOLUME_STATS NodeServiceCapability_RPC_Type = 2 + // See VolumeExpansion for details. + NodeServiceCapability_RPC_EXPAND_VOLUME NodeServiceCapability_RPC_Type = 3 + // Indicates that the Node service can report volume conditions. + // An SP MAY implement `VolumeCondition` in only the Node + // Plugin, only the Controller Plugin, or both. + // If `VolumeCondition` is implemented in both the Node and + // Controller Plugins, it SHALL report from different + // perspectives. + // If for some reason Node and Controller Plugins report + // misaligned volume conditions, CO SHALL assume the worst case + // is the truth. + // Note that, for alpha, `VolumeCondition` is intended to be + // informative for humans only, not for automation. + NodeServiceCapability_RPC_VOLUME_CONDITION NodeServiceCapability_RPC_Type = 4 + // Indicates the SP supports the SINGLE_NODE_SINGLE_WRITER and/or + // SINGLE_NODE_MULTI_WRITER access modes. + // These access modes are intended to replace the + // SINGLE_NODE_WRITER access mode to clarify the number of writers + // for a volume on a single node. Plugins MUST accept and allow + // use of the SINGLE_NODE_WRITER access mode (subject to the + // processing rules for NodePublishVolume), when either + // SINGLE_NODE_SINGLE_WRITER and/or SINGLE_NODE_MULTI_WRITER are + // supported, in order to permit older COs to continue working. + NodeServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER NodeServiceCapability_RPC_Type = 5 + // Indicates that Node service supports mounting volumes + // with provided volume group identifier during node stage + // or node publish RPC calls. + NodeServiceCapability_RPC_VOLUME_MOUNT_GROUP NodeServiceCapability_RPC_Type = 6 +) + +var NodeServiceCapability_RPC_Type_name = map[int32]string{ + 0: "UNKNOWN", + 1: "STAGE_UNSTAGE_VOLUME", + 2: "GET_VOLUME_STATS", + 3: "EXPAND_VOLUME", + 4: "VOLUME_CONDITION", + 5: "SINGLE_NODE_MULTI_WRITER", + 6: "VOLUME_MOUNT_GROUP", +} + +var NodeServiceCapability_RPC_Type_value = map[string]int32{ + "UNKNOWN": 0, + "STAGE_UNSTAGE_VOLUME": 1, + "GET_VOLUME_STATS": 2, + "EXPAND_VOLUME": 3, + "VOLUME_CONDITION": 4, + "SINGLE_NODE_MULTI_WRITER": 5, + "VOLUME_MOUNT_GROUP": 6, +} + +func (x NodeServiceCapability_RPC_Type) String() string { + return proto.EnumName(NodeServiceCapability_RPC_Type_name, int32(x)) +} + +func (NodeServiceCapability_RPC_Type) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{57, 0, 0} +} + +type GroupControllerServiceCapability_RPC_Type int32 + +const ( + GroupControllerServiceCapability_RPC_UNKNOWN GroupControllerServiceCapability_RPC_Type = 0 + // Indicates that the group controller plugin supports + // creating, deleting, and getting details of a volume + // group snapshot. + GroupControllerServiceCapability_RPC_CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT GroupControllerServiceCapability_RPC_Type = 1 +) + +var GroupControllerServiceCapability_RPC_Type_name = map[int32]string{ + 0: "UNKNOWN", + 1: "CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT", +} + +var GroupControllerServiceCapability_RPC_Type_value = map[string]int32{ + "UNKNOWN": 0, + "CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT": 1, +} + +func (x GroupControllerServiceCapability_RPC_Type) String() string { + return proto.EnumName(GroupControllerServiceCapability_RPC_Type_name, int32(x)) +} + +func (GroupControllerServiceCapability_RPC_Type) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{64, 0, 0} +} + +type GetPluginInfoRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetPluginInfoRequest) Reset() { *m = GetPluginInfoRequest{} } +func (m *GetPluginInfoRequest) String() string { return proto.CompactTextString(m) } +func (*GetPluginInfoRequest) ProtoMessage() {} +func (*GetPluginInfoRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{0} +} + +func (m *GetPluginInfoRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetPluginInfoRequest.Unmarshal(m, b) +} +func (m *GetPluginInfoRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetPluginInfoRequest.Marshal(b, m, deterministic) +} +func (m *GetPluginInfoRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetPluginInfoRequest.Merge(m, src) +} +func (m *GetPluginInfoRequest) XXX_Size() int { + return xxx_messageInfo_GetPluginInfoRequest.Size(m) +} +func (m *GetPluginInfoRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetPluginInfoRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetPluginInfoRequest proto.InternalMessageInfo + +type GetPluginInfoResponse struct { + // The name MUST follow domain name notation format + // (https://tools.ietf.org/html/rfc1035#section-2.3.1). It SHOULD + // include the plugin's host company name and the plugin name, + // to minimize the possibility of collisions. It MUST be 63 + // characters or less, beginning and ending with an alphanumeric + // character ([a-z0-9A-Z]) with dashes (-), dots (.), and + // alphanumerics between. This field is REQUIRED. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // This field is REQUIRED. Value of this field is opaque to the CO. + VendorVersion string `protobuf:"bytes,2,opt,name=vendor_version,json=vendorVersion,proto3" json:"vendor_version,omitempty"` + // This field is OPTIONAL. Values are opaque to the CO. + Manifest map[string]string `protobuf:"bytes,3,rep,name=manifest,proto3" json:"manifest,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetPluginInfoResponse) Reset() { *m = GetPluginInfoResponse{} } +func (m *GetPluginInfoResponse) String() string { return proto.CompactTextString(m) } +func (*GetPluginInfoResponse) ProtoMessage() {} +func (*GetPluginInfoResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{1} +} + +func (m *GetPluginInfoResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetPluginInfoResponse.Unmarshal(m, b) +} +func (m *GetPluginInfoResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetPluginInfoResponse.Marshal(b, m, deterministic) +} +func (m *GetPluginInfoResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetPluginInfoResponse.Merge(m, src) +} +func (m *GetPluginInfoResponse) XXX_Size() int { + return xxx_messageInfo_GetPluginInfoResponse.Size(m) +} +func (m *GetPluginInfoResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetPluginInfoResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetPluginInfoResponse proto.InternalMessageInfo + +func (m *GetPluginInfoResponse) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *GetPluginInfoResponse) GetVendorVersion() string { + if m != nil { + return m.VendorVersion + } + return "" +} + +func (m *GetPluginInfoResponse) GetManifest() map[string]string { + if m != nil { + return m.Manifest + } + return nil +} + +type GetPluginCapabilitiesRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetPluginCapabilitiesRequest) Reset() { *m = GetPluginCapabilitiesRequest{} } +func (m *GetPluginCapabilitiesRequest) String() string { return proto.CompactTextString(m) } +func (*GetPluginCapabilitiesRequest) ProtoMessage() {} +func (*GetPluginCapabilitiesRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{2} +} + +func (m *GetPluginCapabilitiesRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetPluginCapabilitiesRequest.Unmarshal(m, b) +} +func (m *GetPluginCapabilitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetPluginCapabilitiesRequest.Marshal(b, m, deterministic) +} +func (m *GetPluginCapabilitiesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetPluginCapabilitiesRequest.Merge(m, src) +} +func (m *GetPluginCapabilitiesRequest) XXX_Size() int { + return xxx_messageInfo_GetPluginCapabilitiesRequest.Size(m) +} +func (m *GetPluginCapabilitiesRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetPluginCapabilitiesRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetPluginCapabilitiesRequest proto.InternalMessageInfo + +type GetPluginCapabilitiesResponse struct { + // All the capabilities that the controller service supports. This + // field is OPTIONAL. + Capabilities []*PluginCapability `protobuf:"bytes,1,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetPluginCapabilitiesResponse) Reset() { *m = GetPluginCapabilitiesResponse{} } +func (m *GetPluginCapabilitiesResponse) String() string { return proto.CompactTextString(m) } +func (*GetPluginCapabilitiesResponse) ProtoMessage() {} +func (*GetPluginCapabilitiesResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{3} +} + +func (m *GetPluginCapabilitiesResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetPluginCapabilitiesResponse.Unmarshal(m, b) +} +func (m *GetPluginCapabilitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetPluginCapabilitiesResponse.Marshal(b, m, deterministic) +} +func (m *GetPluginCapabilitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetPluginCapabilitiesResponse.Merge(m, src) +} +func (m *GetPluginCapabilitiesResponse) XXX_Size() int { + return xxx_messageInfo_GetPluginCapabilitiesResponse.Size(m) +} +func (m *GetPluginCapabilitiesResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetPluginCapabilitiesResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetPluginCapabilitiesResponse proto.InternalMessageInfo + +func (m *GetPluginCapabilitiesResponse) GetCapabilities() []*PluginCapability { + if m != nil { + return m.Capabilities + } + return nil +} + +// Specifies a capability of the plugin. +type PluginCapability struct { + // Types that are valid to be assigned to Type: + // + // *PluginCapability_Service_ + // *PluginCapability_VolumeExpansion_ + Type isPluginCapability_Type `protobuf_oneof:"type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PluginCapability) Reset() { *m = PluginCapability{} } +func (m *PluginCapability) String() string { return proto.CompactTextString(m) } +func (*PluginCapability) ProtoMessage() {} +func (*PluginCapability) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{4} +} + +func (m *PluginCapability) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PluginCapability.Unmarshal(m, b) +} +func (m *PluginCapability) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PluginCapability.Marshal(b, m, deterministic) +} +func (m *PluginCapability) XXX_Merge(src proto.Message) { + xxx_messageInfo_PluginCapability.Merge(m, src) +} +func (m *PluginCapability) XXX_Size() int { + return xxx_messageInfo_PluginCapability.Size(m) +} +func (m *PluginCapability) XXX_DiscardUnknown() { + xxx_messageInfo_PluginCapability.DiscardUnknown(m) +} + +var xxx_messageInfo_PluginCapability proto.InternalMessageInfo + +type isPluginCapability_Type interface { + isPluginCapability_Type() +} + +type PluginCapability_Service_ struct { + Service *PluginCapability_Service `protobuf:"bytes,1,opt,name=service,proto3,oneof"` +} + +type PluginCapability_VolumeExpansion_ struct { + VolumeExpansion *PluginCapability_VolumeExpansion `protobuf:"bytes,2,opt,name=volume_expansion,json=volumeExpansion,proto3,oneof"` +} + +func (*PluginCapability_Service_) isPluginCapability_Type() {} + +func (*PluginCapability_VolumeExpansion_) isPluginCapability_Type() {} + +func (m *PluginCapability) GetType() isPluginCapability_Type { + if m != nil { + return m.Type + } + return nil +} + +func (m *PluginCapability) GetService() *PluginCapability_Service { + if x, ok := m.GetType().(*PluginCapability_Service_); ok { + return x.Service + } + return nil +} + +func (m *PluginCapability) GetVolumeExpansion() *PluginCapability_VolumeExpansion { + if x, ok := m.GetType().(*PluginCapability_VolumeExpansion_); ok { + return x.VolumeExpansion + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*PluginCapability) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*PluginCapability_Service_)(nil), + (*PluginCapability_VolumeExpansion_)(nil), + } +} + +type PluginCapability_Service struct { + Type PluginCapability_Service_Type `protobuf:"varint,1,opt,name=type,proto3,enum=csi.v1.PluginCapability_Service_Type" json:"type,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PluginCapability_Service) Reset() { *m = PluginCapability_Service{} } +func (m *PluginCapability_Service) String() string { return proto.CompactTextString(m) } +func (*PluginCapability_Service) ProtoMessage() {} +func (*PluginCapability_Service) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{4, 0} +} + +func (m *PluginCapability_Service) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PluginCapability_Service.Unmarshal(m, b) +} +func (m *PluginCapability_Service) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PluginCapability_Service.Marshal(b, m, deterministic) +} +func (m *PluginCapability_Service) XXX_Merge(src proto.Message) { + xxx_messageInfo_PluginCapability_Service.Merge(m, src) +} +func (m *PluginCapability_Service) XXX_Size() int { + return xxx_messageInfo_PluginCapability_Service.Size(m) +} +func (m *PluginCapability_Service) XXX_DiscardUnknown() { + xxx_messageInfo_PluginCapability_Service.DiscardUnknown(m) +} + +var xxx_messageInfo_PluginCapability_Service proto.InternalMessageInfo + +func (m *PluginCapability_Service) GetType() PluginCapability_Service_Type { + if m != nil { + return m.Type + } + return PluginCapability_Service_UNKNOWN +} + +type PluginCapability_VolumeExpansion struct { + Type PluginCapability_VolumeExpansion_Type `protobuf:"varint,1,opt,name=type,proto3,enum=csi.v1.PluginCapability_VolumeExpansion_Type" json:"type,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PluginCapability_VolumeExpansion) Reset() { *m = PluginCapability_VolumeExpansion{} } +func (m *PluginCapability_VolumeExpansion) String() string { return proto.CompactTextString(m) } +func (*PluginCapability_VolumeExpansion) ProtoMessage() {} +func (*PluginCapability_VolumeExpansion) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{4, 1} +} + +func (m *PluginCapability_VolumeExpansion) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PluginCapability_VolumeExpansion.Unmarshal(m, b) +} +func (m *PluginCapability_VolumeExpansion) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PluginCapability_VolumeExpansion.Marshal(b, m, deterministic) +} +func (m *PluginCapability_VolumeExpansion) XXX_Merge(src proto.Message) { + xxx_messageInfo_PluginCapability_VolumeExpansion.Merge(m, src) +} +func (m *PluginCapability_VolumeExpansion) XXX_Size() int { + return xxx_messageInfo_PluginCapability_VolumeExpansion.Size(m) +} +func (m *PluginCapability_VolumeExpansion) XXX_DiscardUnknown() { + xxx_messageInfo_PluginCapability_VolumeExpansion.DiscardUnknown(m) +} + +var xxx_messageInfo_PluginCapability_VolumeExpansion proto.InternalMessageInfo + +func (m *PluginCapability_VolumeExpansion) GetType() PluginCapability_VolumeExpansion_Type { + if m != nil { + return m.Type + } + return PluginCapability_VolumeExpansion_UNKNOWN +} + +type ProbeRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ProbeRequest) Reset() { *m = ProbeRequest{} } +func (m *ProbeRequest) String() string { return proto.CompactTextString(m) } +func (*ProbeRequest) ProtoMessage() {} +func (*ProbeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{5} +} + +func (m *ProbeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ProbeRequest.Unmarshal(m, b) +} +func (m *ProbeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ProbeRequest.Marshal(b, m, deterministic) +} +func (m *ProbeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ProbeRequest.Merge(m, src) +} +func (m *ProbeRequest) XXX_Size() int { + return xxx_messageInfo_ProbeRequest.Size(m) +} +func (m *ProbeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ProbeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ProbeRequest proto.InternalMessageInfo + +type ProbeResponse struct { + // Readiness allows a plugin to report its initialization status back + // to the CO. Initialization for some plugins MAY be time consuming + // and it is important for a CO to distinguish between the following + // cases: + // + // 1. The plugin is in an unhealthy state and MAY need restarting. In + // this case a gRPC error code SHALL be returned. + // 2. The plugin is still initializing, but is otherwise perfectly + // healthy. In this case a successful response SHALL be returned + // with a readiness value of `false`. Calls to the plugin's + // Controller and/or Node services MAY fail due to an incomplete + // initialization state. + // 3. The plugin has finished initializing and is ready to service + // calls to its Controller and/or Node services. A successful + // response is returned with a readiness value of `true`. + // + // This field is OPTIONAL. If not present, the caller SHALL assume + // that the plugin is in a ready state and is accepting calls to its + // Controller and/or Node services (according to the plugin's reported + // capabilities). + Ready *wrappers.BoolValue `protobuf:"bytes,1,opt,name=ready,proto3" json:"ready,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ProbeResponse) Reset() { *m = ProbeResponse{} } +func (m *ProbeResponse) String() string { return proto.CompactTextString(m) } +func (*ProbeResponse) ProtoMessage() {} +func (*ProbeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{6} +} + +func (m *ProbeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ProbeResponse.Unmarshal(m, b) +} +func (m *ProbeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ProbeResponse.Marshal(b, m, deterministic) +} +func (m *ProbeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ProbeResponse.Merge(m, src) +} +func (m *ProbeResponse) XXX_Size() int { + return xxx_messageInfo_ProbeResponse.Size(m) +} +func (m *ProbeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ProbeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ProbeResponse proto.InternalMessageInfo + +func (m *ProbeResponse) GetReady() *wrappers.BoolValue { + if m != nil { + return m.Ready + } + return nil +} + +type CreateVolumeRequest struct { + // The suggested name for the storage space. This field is REQUIRED. + // It serves two purposes: + // 1. Idempotency - This name is generated by the CO to achieve + // idempotency. The Plugin SHOULD ensure that multiple + // `CreateVolume` calls for the same name do not result in more + // than one piece of storage provisioned corresponding to that + // name. If a Plugin is unable to enforce idempotency, the CO's + // error recovery logic could result in multiple (unused) volumes + // being provisioned. + // In the case of error, the CO MUST handle the gRPC error codes + // per the recovery behavior defined in the "CreateVolume Errors" + // section below. + // The CO is responsible for cleaning up volumes it provisioned + // that it no longer needs. If the CO is uncertain whether a volume + // was provisioned or not when a `CreateVolume` call fails, the CO + // MAY call `CreateVolume` again, with the same name, to ensure the + // volume exists and to retrieve the volume's `volume_id` (unless + // otherwise prohibited by "CreateVolume Errors"). + // 2. Suggested name - Some storage systems allow callers to specify + // an identifier by which to refer to the newly provisioned + // storage. If a storage system supports this, it can optionally + // use this name as the identifier for the new volume. + // + // Any Unicode string that conforms to the length limit is allowed + // except those containing the following banned characters: + // U+0000-U+0008, U+000B, U+000C, U+000E-U+001F, U+007F-U+009F. + // (These are control characters other than commonly used whitespace.) + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // This field is OPTIONAL. This allows the CO to specify the capacity + // requirement of the volume to be provisioned. If not specified, the + // Plugin MAY choose an implementation-defined capacity range. If + // specified it MUST always be honored, even when creating volumes + // from a source; which MAY force some backends to internally extend + // the volume after creating it. + CapacityRange *CapacityRange `protobuf:"bytes,2,opt,name=capacity_range,json=capacityRange,proto3" json:"capacity_range,omitempty"` + // The capabilities that the provisioned volume MUST have. SP MUST + // provision a volume that will satisfy ALL of the capabilities + // specified in this list. Otherwise SP MUST return the appropriate + // gRPC error code. + // The Plugin MUST assume that the CO MAY use the provisioned volume + // with ANY of the capabilities specified in this list. + // For example, a CO MAY specify two volume capabilities: one with + // access mode SINGLE_NODE_WRITER and another with access mode + // MULTI_NODE_READER_ONLY. In this case, the SP MUST verify that the + // provisioned volume can be used in either mode. + // This also enables the CO to do early validation: If ANY of the + // specified volume capabilities are not supported by the SP, the call + // MUST return the appropriate gRPC error code. + // This field is REQUIRED. + VolumeCapabilities []*VolumeCapability `protobuf:"bytes,3,rep,name=volume_capabilities,json=volumeCapabilities,proto3" json:"volume_capabilities,omitempty"` + // Plugin specific creation-time parameters passed in as opaque + // key-value pairs. This field is OPTIONAL. The Plugin is responsible + // for parsing and validating these parameters. COs will treat + // these as opaque. + Parameters map[string]string `protobuf:"bytes,4,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Secrets required by plugin to complete volume creation request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + Secrets map[string]string `protobuf:"bytes,5,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // If specified, the new volume will be pre-populated with data from + // this source. This field is OPTIONAL. + VolumeContentSource *VolumeContentSource `protobuf:"bytes,6,opt,name=volume_content_source,json=volumeContentSource,proto3" json:"volume_content_source,omitempty"` + // Specifies where (regions, zones, racks, etc.) the provisioned + // volume MUST be accessible from. + // An SP SHALL advertise the requirements for topological + // accessibility information in documentation. COs SHALL only specify + // topological accessibility information supported by the SP. + // This field is OPTIONAL. + // This field SHALL NOT be specified unless the SP has the + // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability. + // If this field is not specified and the SP has the + // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability, the SP MAY + // choose where the provisioned volume is accessible from. + AccessibilityRequirements *TopologyRequirement `protobuf:"bytes,7,opt,name=accessibility_requirements,json=accessibilityRequirements,proto3" json:"accessibility_requirements,omitempty"` + // Plugins MUST treat these + // as if they take precedence over the parameters field. + // This field SHALL NOT be specified unless the SP has the + // MODIFY_VOLUME plugin capability. + MutableParameters map[string]string `protobuf:"bytes,8,rep,name=mutable_parameters,json=mutableParameters,proto3" json:"mutable_parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CreateVolumeRequest) Reset() { *m = CreateVolumeRequest{} } +func (m *CreateVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*CreateVolumeRequest) ProtoMessage() {} +func (*CreateVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{7} +} + +func (m *CreateVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CreateVolumeRequest.Unmarshal(m, b) +} +func (m *CreateVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CreateVolumeRequest.Marshal(b, m, deterministic) +} +func (m *CreateVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateVolumeRequest.Merge(m, src) +} +func (m *CreateVolumeRequest) XXX_Size() int { + return xxx_messageInfo_CreateVolumeRequest.Size(m) +} +func (m *CreateVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_CreateVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_CreateVolumeRequest proto.InternalMessageInfo + +func (m *CreateVolumeRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *CreateVolumeRequest) GetCapacityRange() *CapacityRange { + if m != nil { + return m.CapacityRange + } + return nil +} + +func (m *CreateVolumeRequest) GetVolumeCapabilities() []*VolumeCapability { + if m != nil { + return m.VolumeCapabilities + } + return nil +} + +func (m *CreateVolumeRequest) GetParameters() map[string]string { + if m != nil { + return m.Parameters + } + return nil +} + +func (m *CreateVolumeRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +func (m *CreateVolumeRequest) GetVolumeContentSource() *VolumeContentSource { + if m != nil { + return m.VolumeContentSource + } + return nil +} + +func (m *CreateVolumeRequest) GetAccessibilityRequirements() *TopologyRequirement { + if m != nil { + return m.AccessibilityRequirements + } + return nil +} + +func (m *CreateVolumeRequest) GetMutableParameters() map[string]string { + if m != nil { + return m.MutableParameters + } + return nil +} + +// Specifies what source the volume will be created from. One of the +// type fields MUST be specified. +type VolumeContentSource struct { + // Types that are valid to be assigned to Type: + // + // *VolumeContentSource_Snapshot + // *VolumeContentSource_Volume + Type isVolumeContentSource_Type `protobuf_oneof:"type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeContentSource) Reset() { *m = VolumeContentSource{} } +func (m *VolumeContentSource) String() string { return proto.CompactTextString(m) } +func (*VolumeContentSource) ProtoMessage() {} +func (*VolumeContentSource) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{8} +} + +func (m *VolumeContentSource) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VolumeContentSource.Unmarshal(m, b) +} +func (m *VolumeContentSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VolumeContentSource.Marshal(b, m, deterministic) +} +func (m *VolumeContentSource) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeContentSource.Merge(m, src) +} +func (m *VolumeContentSource) XXX_Size() int { + return xxx_messageInfo_VolumeContentSource.Size(m) +} +func (m *VolumeContentSource) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeContentSource.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeContentSource proto.InternalMessageInfo + +type isVolumeContentSource_Type interface { + isVolumeContentSource_Type() +} + +type VolumeContentSource_Snapshot struct { + Snapshot *VolumeContentSource_SnapshotSource `protobuf:"bytes,1,opt,name=snapshot,proto3,oneof"` +} + +type VolumeContentSource_Volume struct { + Volume *VolumeContentSource_VolumeSource `protobuf:"bytes,2,opt,name=volume,proto3,oneof"` +} + +func (*VolumeContentSource_Snapshot) isVolumeContentSource_Type() {} + +func (*VolumeContentSource_Volume) isVolumeContentSource_Type() {} + +func (m *VolumeContentSource) GetType() isVolumeContentSource_Type { + if m != nil { + return m.Type + } + return nil +} + +func (m *VolumeContentSource) GetSnapshot() *VolumeContentSource_SnapshotSource { + if x, ok := m.GetType().(*VolumeContentSource_Snapshot); ok { + return x.Snapshot + } + return nil +} + +func (m *VolumeContentSource) GetVolume() *VolumeContentSource_VolumeSource { + if x, ok := m.GetType().(*VolumeContentSource_Volume); ok { + return x.Volume + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*VolumeContentSource) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*VolumeContentSource_Snapshot)(nil), + (*VolumeContentSource_Volume)(nil), + } +} + +type VolumeContentSource_SnapshotSource struct { + // Contains identity information for the existing source snapshot. + // This field is REQUIRED. Plugin is REQUIRED to support creating + // volume from snapshot if it supports the capability + // CREATE_DELETE_SNAPSHOT. + SnapshotId string `protobuf:"bytes,1,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeContentSource_SnapshotSource) Reset() { *m = VolumeContentSource_SnapshotSource{} } +func (m *VolumeContentSource_SnapshotSource) String() string { return proto.CompactTextString(m) } +func (*VolumeContentSource_SnapshotSource) ProtoMessage() {} +func (*VolumeContentSource_SnapshotSource) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{8, 0} +} + +func (m *VolumeContentSource_SnapshotSource) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VolumeContentSource_SnapshotSource.Unmarshal(m, b) +} +func (m *VolumeContentSource_SnapshotSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VolumeContentSource_SnapshotSource.Marshal(b, m, deterministic) +} +func (m *VolumeContentSource_SnapshotSource) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeContentSource_SnapshotSource.Merge(m, src) +} +func (m *VolumeContentSource_SnapshotSource) XXX_Size() int { + return xxx_messageInfo_VolumeContentSource_SnapshotSource.Size(m) +} +func (m *VolumeContentSource_SnapshotSource) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeContentSource_SnapshotSource.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeContentSource_SnapshotSource proto.InternalMessageInfo + +func (m *VolumeContentSource_SnapshotSource) GetSnapshotId() string { + if m != nil { + return m.SnapshotId + } + return "" +} + +type VolumeContentSource_VolumeSource struct { + // Contains identity information for the existing source volume. + // This field is REQUIRED. Plugins reporting CLONE_VOLUME + // capability MUST support creating a volume from another volume. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeContentSource_VolumeSource) Reset() { *m = VolumeContentSource_VolumeSource{} } +func (m *VolumeContentSource_VolumeSource) String() string { return proto.CompactTextString(m) } +func (*VolumeContentSource_VolumeSource) ProtoMessage() {} +func (*VolumeContentSource_VolumeSource) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{8, 1} +} + +func (m *VolumeContentSource_VolumeSource) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VolumeContentSource_VolumeSource.Unmarshal(m, b) +} +func (m *VolumeContentSource_VolumeSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VolumeContentSource_VolumeSource.Marshal(b, m, deterministic) +} +func (m *VolumeContentSource_VolumeSource) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeContentSource_VolumeSource.Merge(m, src) +} +func (m *VolumeContentSource_VolumeSource) XXX_Size() int { + return xxx_messageInfo_VolumeContentSource_VolumeSource.Size(m) +} +func (m *VolumeContentSource_VolumeSource) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeContentSource_VolumeSource.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeContentSource_VolumeSource proto.InternalMessageInfo + +func (m *VolumeContentSource_VolumeSource) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +type CreateVolumeResponse struct { + // Contains all attributes of the newly created volume that are + // relevant to the CO along with information required by the Plugin + // to uniquely identify the volume. This field is REQUIRED. + Volume *Volume `protobuf:"bytes,1,opt,name=volume,proto3" json:"volume,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CreateVolumeResponse) Reset() { *m = CreateVolumeResponse{} } +func (m *CreateVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*CreateVolumeResponse) ProtoMessage() {} +func (*CreateVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{9} +} + +func (m *CreateVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CreateVolumeResponse.Unmarshal(m, b) +} +func (m *CreateVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CreateVolumeResponse.Marshal(b, m, deterministic) +} +func (m *CreateVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateVolumeResponse.Merge(m, src) +} +func (m *CreateVolumeResponse) XXX_Size() int { + return xxx_messageInfo_CreateVolumeResponse.Size(m) +} +func (m *CreateVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_CreateVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_CreateVolumeResponse proto.InternalMessageInfo + +func (m *CreateVolumeResponse) GetVolume() *Volume { + if m != nil { + return m.Volume + } + return nil +} + +// Specify a capability of a volume. +type VolumeCapability struct { + // Specifies what API the volume will be accessed using. One of the + // following fields MUST be specified. + // + // Types that are valid to be assigned to AccessType: + // + // *VolumeCapability_Block + // *VolumeCapability_Mount + AccessType isVolumeCapability_AccessType `protobuf_oneof:"access_type"` + // This is a REQUIRED field. + AccessMode *VolumeCapability_AccessMode `protobuf:"bytes,3,opt,name=access_mode,json=accessMode,proto3" json:"access_mode,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeCapability) Reset() { *m = VolumeCapability{} } +func (m *VolumeCapability) String() string { return proto.CompactTextString(m) } +func (*VolumeCapability) ProtoMessage() {} +func (*VolumeCapability) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{10} +} + +func (m *VolumeCapability) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VolumeCapability.Unmarshal(m, b) +} +func (m *VolumeCapability) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VolumeCapability.Marshal(b, m, deterministic) +} +func (m *VolumeCapability) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeCapability.Merge(m, src) +} +func (m *VolumeCapability) XXX_Size() int { + return xxx_messageInfo_VolumeCapability.Size(m) +} +func (m *VolumeCapability) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeCapability.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeCapability proto.InternalMessageInfo + +type isVolumeCapability_AccessType interface { + isVolumeCapability_AccessType() +} + +type VolumeCapability_Block struct { + Block *VolumeCapability_BlockVolume `protobuf:"bytes,1,opt,name=block,proto3,oneof"` +} + +type VolumeCapability_Mount struct { + Mount *VolumeCapability_MountVolume `protobuf:"bytes,2,opt,name=mount,proto3,oneof"` +} + +func (*VolumeCapability_Block) isVolumeCapability_AccessType() {} + +func (*VolumeCapability_Mount) isVolumeCapability_AccessType() {} + +func (m *VolumeCapability) GetAccessType() isVolumeCapability_AccessType { + if m != nil { + return m.AccessType + } + return nil +} + +func (m *VolumeCapability) GetBlock() *VolumeCapability_BlockVolume { + if x, ok := m.GetAccessType().(*VolumeCapability_Block); ok { + return x.Block + } + return nil +} + +func (m *VolumeCapability) GetMount() *VolumeCapability_MountVolume { + if x, ok := m.GetAccessType().(*VolumeCapability_Mount); ok { + return x.Mount + } + return nil +} + +func (m *VolumeCapability) GetAccessMode() *VolumeCapability_AccessMode { + if m != nil { + return m.AccessMode + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*VolumeCapability) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*VolumeCapability_Block)(nil), + (*VolumeCapability_Mount)(nil), + } +} + +// Indicate that the volume will be accessed via the block device API. +type VolumeCapability_BlockVolume struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeCapability_BlockVolume) Reset() { *m = VolumeCapability_BlockVolume{} } +func (m *VolumeCapability_BlockVolume) String() string { return proto.CompactTextString(m) } +func (*VolumeCapability_BlockVolume) ProtoMessage() {} +func (*VolumeCapability_BlockVolume) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{10, 0} +} + +func (m *VolumeCapability_BlockVolume) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VolumeCapability_BlockVolume.Unmarshal(m, b) +} +func (m *VolumeCapability_BlockVolume) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VolumeCapability_BlockVolume.Marshal(b, m, deterministic) +} +func (m *VolumeCapability_BlockVolume) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeCapability_BlockVolume.Merge(m, src) +} +func (m *VolumeCapability_BlockVolume) XXX_Size() int { + return xxx_messageInfo_VolumeCapability_BlockVolume.Size(m) +} +func (m *VolumeCapability_BlockVolume) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeCapability_BlockVolume.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeCapability_BlockVolume proto.InternalMessageInfo + +// Indicate that the volume will be accessed via the filesystem API. +type VolumeCapability_MountVolume struct { + // The filesystem type. This field is OPTIONAL. + // An empty string is equal to an unspecified field value. + FsType string `protobuf:"bytes,1,opt,name=fs_type,json=fsType,proto3" json:"fs_type,omitempty"` + // The mount options that can be used for the volume. This field is + // OPTIONAL. `mount_flags` MAY contain sensitive information. + // Therefore, the CO and the Plugin MUST NOT leak this information + // to untrusted entities. The total size of this repeated field + // SHALL NOT exceed 4 KiB. + MountFlags []string `protobuf:"bytes,2,rep,name=mount_flags,json=mountFlags,proto3" json:"mount_flags,omitempty"` + // If SP has VOLUME_MOUNT_GROUP node capability and CO provides + // this field then SP MUST ensure that the volume_mount_group + // parameter is passed as the group identifier to the underlying + // operating system mount system call, with the understanding + // that the set of available mount call parameters and/or + // mount implementations may vary across operating systems. + // Additionally, new file and/or directory entries written to + // the underlying filesystem SHOULD be permission-labeled in such a + // manner, unless otherwise modified by a workload, that they are + // both readable and writable by said mount group identifier. + // This is an OPTIONAL field. + VolumeMountGroup string `protobuf:"bytes,3,opt,name=volume_mount_group,json=volumeMountGroup,proto3" json:"volume_mount_group,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeCapability_MountVolume) Reset() { *m = VolumeCapability_MountVolume{} } +func (m *VolumeCapability_MountVolume) String() string { return proto.CompactTextString(m) } +func (*VolumeCapability_MountVolume) ProtoMessage() {} +func (*VolumeCapability_MountVolume) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{10, 1} +} + +func (m *VolumeCapability_MountVolume) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VolumeCapability_MountVolume.Unmarshal(m, b) +} +func (m *VolumeCapability_MountVolume) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VolumeCapability_MountVolume.Marshal(b, m, deterministic) +} +func (m *VolumeCapability_MountVolume) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeCapability_MountVolume.Merge(m, src) +} +func (m *VolumeCapability_MountVolume) XXX_Size() int { + return xxx_messageInfo_VolumeCapability_MountVolume.Size(m) +} +func (m *VolumeCapability_MountVolume) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeCapability_MountVolume.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeCapability_MountVolume proto.InternalMessageInfo + +func (m *VolumeCapability_MountVolume) GetFsType() string { + if m != nil { + return m.FsType + } + return "" +} + +func (m *VolumeCapability_MountVolume) GetMountFlags() []string { + if m != nil { + return m.MountFlags + } + return nil +} + +func (m *VolumeCapability_MountVolume) GetVolumeMountGroup() string { + if m != nil { + return m.VolumeMountGroup + } + return "" +} + +// Specify how a volume can be accessed. +type VolumeCapability_AccessMode struct { + // This field is REQUIRED. + Mode VolumeCapability_AccessMode_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=csi.v1.VolumeCapability_AccessMode_Mode" json:"mode,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeCapability_AccessMode) Reset() { *m = VolumeCapability_AccessMode{} } +func (m *VolumeCapability_AccessMode) String() string { return proto.CompactTextString(m) } +func (*VolumeCapability_AccessMode) ProtoMessage() {} +func (*VolumeCapability_AccessMode) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{10, 2} +} + +func (m *VolumeCapability_AccessMode) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VolumeCapability_AccessMode.Unmarshal(m, b) +} +func (m *VolumeCapability_AccessMode) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VolumeCapability_AccessMode.Marshal(b, m, deterministic) +} +func (m *VolumeCapability_AccessMode) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeCapability_AccessMode.Merge(m, src) +} +func (m *VolumeCapability_AccessMode) XXX_Size() int { + return xxx_messageInfo_VolumeCapability_AccessMode.Size(m) +} +func (m *VolumeCapability_AccessMode) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeCapability_AccessMode.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeCapability_AccessMode proto.InternalMessageInfo + +func (m *VolumeCapability_AccessMode) GetMode() VolumeCapability_AccessMode_Mode { + if m != nil { + return m.Mode + } + return VolumeCapability_AccessMode_UNKNOWN +} + +// The capacity of the storage space in bytes. To specify an exact size, +// `required_bytes` and `limit_bytes` SHALL be set to the same value. At +// least one of the these fields MUST be specified. +type CapacityRange struct { + // Volume MUST be at least this big. This field is OPTIONAL. + // A value of 0 is equal to an unspecified field value. + // The value of this field MUST NOT be negative. + RequiredBytes int64 `protobuf:"varint,1,opt,name=required_bytes,json=requiredBytes,proto3" json:"required_bytes,omitempty"` + // Volume MUST not be bigger than this. This field is OPTIONAL. + // A value of 0 is equal to an unspecified field value. + // The value of this field MUST NOT be negative. + LimitBytes int64 `protobuf:"varint,2,opt,name=limit_bytes,json=limitBytes,proto3" json:"limit_bytes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CapacityRange) Reset() { *m = CapacityRange{} } +func (m *CapacityRange) String() string { return proto.CompactTextString(m) } +func (*CapacityRange) ProtoMessage() {} +func (*CapacityRange) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{11} +} + +func (m *CapacityRange) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CapacityRange.Unmarshal(m, b) +} +func (m *CapacityRange) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CapacityRange.Marshal(b, m, deterministic) +} +func (m *CapacityRange) XXX_Merge(src proto.Message) { + xxx_messageInfo_CapacityRange.Merge(m, src) +} +func (m *CapacityRange) XXX_Size() int { + return xxx_messageInfo_CapacityRange.Size(m) +} +func (m *CapacityRange) XXX_DiscardUnknown() { + xxx_messageInfo_CapacityRange.DiscardUnknown(m) +} + +var xxx_messageInfo_CapacityRange proto.InternalMessageInfo + +func (m *CapacityRange) GetRequiredBytes() int64 { + if m != nil { + return m.RequiredBytes + } + return 0 +} + +func (m *CapacityRange) GetLimitBytes() int64 { + if m != nil { + return m.LimitBytes + } + return 0 +} + +// Information about a specific volume. +type Volume struct { + // The capacity of the volume in bytes. This field is OPTIONAL. If not + // set (value of 0), it indicates that the capacity of the volume is + // unknown (e.g., NFS share). + // The value of this field MUST NOT be negative. + CapacityBytes int64 `protobuf:"varint,1,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` + // The identifier for this volume, generated by the plugin. + // This field is REQUIRED. + // This field MUST contain enough information to uniquely identify + // this specific volume vs all other volumes supported by this plugin. + // This field SHALL be used by the CO in subsequent calls to refer to + // this volume. + // The SP is NOT responsible for global uniqueness of volume_id across + // multiple SPs. + VolumeId string `protobuf:"bytes,2,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // Opaque static properties of the volume. SP MAY use this field to + // ensure subsequent volume validation and publishing calls have + // contextual information. + // The contents of this field SHALL be opaque to a CO. + // The contents of this field SHALL NOT be mutable. + // The contents of this field SHALL be safe for the CO to cache. + // The contents of this field SHOULD NOT contain sensitive + // information. + // The contents of this field SHOULD NOT be used for uniquely + // identifying a volume. The `volume_id` alone SHOULD be sufficient to + // identify the volume. + // A volume uniquely identified by `volume_id` SHALL always report the + // same volume_context. + // This field is OPTIONAL and when present MUST be passed to volume + // validation and publishing calls. + VolumeContext map[string]string `protobuf:"bytes,3,rep,name=volume_context,json=volumeContext,proto3" json:"volume_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // If specified, indicates that the volume is not empty and is + // pre-populated with data from the specified source. + // This field is OPTIONAL. + ContentSource *VolumeContentSource `protobuf:"bytes,4,opt,name=content_source,json=contentSource,proto3" json:"content_source,omitempty"` + // Specifies where (regions, zones, racks, etc.) the provisioned + // volume is accessible from. + // A plugin that returns this field MUST also set the + // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability. + // An SP MAY specify multiple topologies to indicate the volume is + // accessible from multiple locations. + // COs MAY use this information along with the topology information + // returned by NodeGetInfo to ensure that a given volume is accessible + // from a given node when scheduling workloads. + // This field is OPTIONAL. If it is not specified, the CO MAY assume + // the volume is equally accessible from all nodes in the cluster and + // MAY schedule workloads referencing the volume on any available + // node. + // + // Example 1: + // + // accessible_topology = {"region": "R1", "zone": "Z2"} + // + // Indicates a volume accessible only from the "region" "R1" and the + // "zone" "Z2". + // + // Example 2: + // + // accessible_topology = + // {"region": "R1", "zone": "Z2"}, + // {"region": "R1", "zone": "Z3"} + // + // Indicates a volume accessible from both "zone" "Z2" and "zone" "Z3" + // in the "region" "R1". + AccessibleTopology []*Topology `protobuf:"bytes,5,rep,name=accessible_topology,json=accessibleTopology,proto3" json:"accessible_topology,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Volume) Reset() { *m = Volume{} } +func (m *Volume) String() string { return proto.CompactTextString(m) } +func (*Volume) ProtoMessage() {} +func (*Volume) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{12} +} + +func (m *Volume) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Volume.Unmarshal(m, b) +} +func (m *Volume) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Volume.Marshal(b, m, deterministic) +} +func (m *Volume) XXX_Merge(src proto.Message) { + xxx_messageInfo_Volume.Merge(m, src) +} +func (m *Volume) XXX_Size() int { + return xxx_messageInfo_Volume.Size(m) +} +func (m *Volume) XXX_DiscardUnknown() { + xxx_messageInfo_Volume.DiscardUnknown(m) +} + +var xxx_messageInfo_Volume proto.InternalMessageInfo + +func (m *Volume) GetCapacityBytes() int64 { + if m != nil { + return m.CapacityBytes + } + return 0 +} + +func (m *Volume) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *Volume) GetVolumeContext() map[string]string { + if m != nil { + return m.VolumeContext + } + return nil +} + +func (m *Volume) GetContentSource() *VolumeContentSource { + if m != nil { + return m.ContentSource + } + return nil +} + +func (m *Volume) GetAccessibleTopology() []*Topology { + if m != nil { + return m.AccessibleTopology + } + return nil +} + +type TopologyRequirement struct { + // Specifies the list of topologies the provisioned volume MUST be + // accessible from. + // This field is OPTIONAL. If TopologyRequirement is specified either + // requisite or preferred or both MUST be specified. + // + // If requisite is specified, the provisioned volume MUST be + // accessible from at least one of the requisite topologies. + // + // Given + // + // x = number of topologies provisioned volume is accessible from + // n = number of requisite topologies + // + // The CO MUST ensure n >= 1. The SP MUST ensure x >= 1 + // If x==n, then the SP MUST make the provisioned volume available to + // all topologies from the list of requisite topologies. If it is + // unable to do so, the SP MUST fail the CreateVolume call. + // For example, if a volume should be accessible from a single zone, + // and requisite = + // + // {"region": "R1", "zone": "Z2"} + // + // then the provisioned volume MUST be accessible from the "region" + // "R1" and the "zone" "Z2". + // Similarly, if a volume should be accessible from two zones, and + // requisite = + // + // {"region": "R1", "zone": "Z2"}, + // {"region": "R1", "zone": "Z3"} + // + // then the provisioned volume MUST be accessible from the "region" + // "R1" and both "zone" "Z2" and "zone" "Z3". + // + // If xn, then the SP MUST make the provisioned volume available from + // all topologies from the list of requisite topologies and MAY choose + // the remaining x-n unique topologies from the list of all possible + // topologies. If it is unable to do so, the SP MUST fail the + // CreateVolume call. + // For example, if a volume should be accessible from two zones, and + // requisite = + // + // {"region": "R1", "zone": "Z2"} + // + // then the provisioned volume MUST be accessible from the "region" + // "R1" and the "zone" "Z2" and the SP may select the second zone + // independently, e.g. "R1/Z4". + Requisite []*Topology `protobuf:"bytes,1,rep,name=requisite,proto3" json:"requisite,omitempty"` + // Specifies the list of topologies the CO would prefer the volume to + // be provisioned in. + // + // This field is OPTIONAL. If TopologyRequirement is specified either + // requisite or preferred or both MUST be specified. + // + // An SP MUST attempt to make the provisioned volume available using + // the preferred topologies in order from first to last. + // + // If requisite is specified, all topologies in preferred list MUST + // also be present in the list of requisite topologies. + // + // If the SP is unable to to make the provisioned volume available + // from any of the preferred topologies, the SP MAY choose a topology + // from the list of requisite topologies. + // If the list of requisite topologies is not specified, then the SP + // MAY choose from the list of all possible topologies. + // If the list of requisite topologies is specified and the SP is + // unable to to make the provisioned volume available from any of the + // requisite topologies it MUST fail the CreateVolume call. + // + // Example 1: + // Given a volume should be accessible from a single zone, and + // requisite = + // + // {"region": "R1", "zone": "Z2"}, + // {"region": "R1", "zone": "Z3"} + // + // preferred = + // + // {"region": "R1", "zone": "Z3"} + // + // then the SP SHOULD first attempt to make the provisioned volume + // available from "zone" "Z3" in the "region" "R1" and fall back to + // "zone" "Z2" in the "region" "R1" if that is not possible. + // + // Example 2: + // Given a volume should be accessible from a single zone, and + // requisite = + // + // {"region": "R1", "zone": "Z2"}, + // {"region": "R1", "zone": "Z3"}, + // {"region": "R1", "zone": "Z4"}, + // {"region": "R1", "zone": "Z5"} + // + // preferred = + // + // {"region": "R1", "zone": "Z4"}, + // {"region": "R1", "zone": "Z2"} + // + // then the SP SHOULD first attempt to make the provisioned volume + // accessible from "zone" "Z4" in the "region" "R1" and fall back to + // "zone" "Z2" in the "region" "R1" if that is not possible. If that + // is not possible, the SP may choose between either the "zone" + // "Z3" or "Z5" in the "region" "R1". + // + // Example 3: + // Given a volume should be accessible from TWO zones (because an + // opaque parameter in CreateVolumeRequest, for example, specifies + // the volume is accessible from two zones, aka synchronously + // replicated), and + // requisite = + // + // {"region": "R1", "zone": "Z2"}, + // {"region": "R1", "zone": "Z3"}, + // {"region": "R1", "zone": "Z4"}, + // {"region": "R1", "zone": "Z5"} + // + // preferred = + // + // {"region": "R1", "zone": "Z5"}, + // {"region": "R1", "zone": "Z3"} + // + // then the SP SHOULD first attempt to make the provisioned volume + // accessible from the combination of the two "zones" "Z5" and "Z3" in + // the "region" "R1". If that's not possible, it should fall back to + // a combination of "Z5" and other possibilities from the list of + // requisite. If that's not possible, it should fall back to a + // combination of "Z3" and other possibilities from the list of + // requisite. If that's not possible, it should fall back to a + // combination of other possibilities from the list of requisite. + Preferred []*Topology `protobuf:"bytes,2,rep,name=preferred,proto3" json:"preferred,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TopologyRequirement) Reset() { *m = TopologyRequirement{} } +func (m *TopologyRequirement) String() string { return proto.CompactTextString(m) } +func (*TopologyRequirement) ProtoMessage() {} +func (*TopologyRequirement) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{13} +} + +func (m *TopologyRequirement) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_TopologyRequirement.Unmarshal(m, b) +} +func (m *TopologyRequirement) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_TopologyRequirement.Marshal(b, m, deterministic) +} +func (m *TopologyRequirement) XXX_Merge(src proto.Message) { + xxx_messageInfo_TopologyRequirement.Merge(m, src) +} +func (m *TopologyRequirement) XXX_Size() int { + return xxx_messageInfo_TopologyRequirement.Size(m) +} +func (m *TopologyRequirement) XXX_DiscardUnknown() { + xxx_messageInfo_TopologyRequirement.DiscardUnknown(m) +} + +var xxx_messageInfo_TopologyRequirement proto.InternalMessageInfo + +func (m *TopologyRequirement) GetRequisite() []*Topology { + if m != nil { + return m.Requisite + } + return nil +} + +func (m *TopologyRequirement) GetPreferred() []*Topology { + if m != nil { + return m.Preferred + } + return nil +} + +// Topology is a map of topological domains to topological segments. +// A topological domain is a sub-division of a cluster, like "region", +// "zone", "rack", etc. +// A topological segment is a specific instance of a topological domain, +// like "zone3", "rack3", etc. +// For example {"com.company/zone": "Z1", "com.company/rack": "R3"} +// Valid keys have two segments: an OPTIONAL prefix and name, separated +// by a slash (/), for example: "com.company.example/zone". +// The key name segment is REQUIRED. The prefix is OPTIONAL. +// The key name MUST be 63 characters or less, begin and end with an +// alphanumeric character ([a-z0-9A-Z]), and contain only dashes (-), +// underscores (_), dots (.), or alphanumerics in between, for example +// "zone". +// The key prefix MUST be 63 characters or less, begin and end with a +// lower-case alphanumeric character ([a-z0-9]), contain only +// dashes (-), dots (.), or lower-case alphanumerics in between, and +// follow domain name notation format +// (https://tools.ietf.org/html/rfc1035#section-2.3.1). +// The key prefix SHOULD include the plugin's host company name and/or +// the plugin name, to minimize the possibility of collisions with keys +// from other plugins. +// If a key prefix is specified, it MUST be identical across all +// topology keys returned by the SP (across all RPCs). +// Keys MUST be case-insensitive. Meaning the keys "Zone" and "zone" +// MUST not both exist. +// Each value (topological segment) MUST contain 1 or more strings. +// Each string MUST be 63 characters or less and begin and end with an +// alphanumeric character with '-', '_', '.', or alphanumerics in +// between. +type Topology struct { + Segments map[string]string `protobuf:"bytes,1,rep,name=segments,proto3" json:"segments,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Topology) Reset() { *m = Topology{} } +func (m *Topology) String() string { return proto.CompactTextString(m) } +func (*Topology) ProtoMessage() {} +func (*Topology) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{14} +} + +func (m *Topology) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Topology.Unmarshal(m, b) +} +func (m *Topology) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Topology.Marshal(b, m, deterministic) +} +func (m *Topology) XXX_Merge(src proto.Message) { + xxx_messageInfo_Topology.Merge(m, src) +} +func (m *Topology) XXX_Size() int { + return xxx_messageInfo_Topology.Size(m) +} +func (m *Topology) XXX_DiscardUnknown() { + xxx_messageInfo_Topology.DiscardUnknown(m) +} + +var xxx_messageInfo_Topology proto.InternalMessageInfo + +func (m *Topology) GetSegments() map[string]string { + if m != nil { + return m.Segments + } + return nil +} + +type DeleteVolumeRequest struct { + // The ID of the volume to be deprovisioned. + // This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // Secrets required by plugin to complete volume deletion request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + Secrets map[string]string `protobuf:"bytes,2,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteVolumeRequest) Reset() { *m = DeleteVolumeRequest{} } +func (m *DeleteVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteVolumeRequest) ProtoMessage() {} +func (*DeleteVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{15} +} + +func (m *DeleteVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteVolumeRequest.Unmarshal(m, b) +} +func (m *DeleteVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteVolumeRequest.Marshal(b, m, deterministic) +} +func (m *DeleteVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteVolumeRequest.Merge(m, src) +} +func (m *DeleteVolumeRequest) XXX_Size() int { + return xxx_messageInfo_DeleteVolumeRequest.Size(m) +} +func (m *DeleteVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteVolumeRequest proto.InternalMessageInfo + +func (m *DeleteVolumeRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *DeleteVolumeRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +type DeleteVolumeResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteVolumeResponse) Reset() { *m = DeleteVolumeResponse{} } +func (m *DeleteVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*DeleteVolumeResponse) ProtoMessage() {} +func (*DeleteVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{16} +} + +func (m *DeleteVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteVolumeResponse.Unmarshal(m, b) +} +func (m *DeleteVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteVolumeResponse.Marshal(b, m, deterministic) +} +func (m *DeleteVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteVolumeResponse.Merge(m, src) +} +func (m *DeleteVolumeResponse) XXX_Size() int { + return xxx_messageInfo_DeleteVolumeResponse.Size(m) +} +func (m *DeleteVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteVolumeResponse proto.InternalMessageInfo + +type ControllerPublishVolumeRequest struct { + // The ID of the volume to be used on a node. + // This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // The ID of the node. This field is REQUIRED. The CO SHALL set this + // field to match the node ID returned by `NodeGetInfo`. + NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` + // Volume capability describing how the CO intends to use this volume. + // SP MUST ensure the CO can use the published volume as described. + // Otherwise SP MUST return the appropriate gRPC error code. + // This is a REQUIRED field. + VolumeCapability *VolumeCapability `protobuf:"bytes,3,opt,name=volume_capability,json=volumeCapability,proto3" json:"volume_capability,omitempty"` + // Indicates SP MUST publish the volume in readonly mode. + // CO MUST set this field to false if SP does not have the + // PUBLISH_READONLY controller capability. + // This is a REQUIRED field. + Readonly bool `protobuf:"varint,4,opt,name=readonly,proto3" json:"readonly,omitempty"` + // Secrets required by plugin to complete controller publish volume + // request. This field is OPTIONAL. Refer to the + // `Secrets Requirements` section on how to use this field. + Secrets map[string]string `protobuf:"bytes,5,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Volume context as returned by SP in + // CreateVolumeResponse.Volume.volume_context. + // This field is OPTIONAL and MUST match the volume_context of the + // volume identified by `volume_id`. + VolumeContext map[string]string `protobuf:"bytes,6,rep,name=volume_context,json=volumeContext,proto3" json:"volume_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerPublishVolumeRequest) Reset() { *m = ControllerPublishVolumeRequest{} } +func (m *ControllerPublishVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*ControllerPublishVolumeRequest) ProtoMessage() {} +func (*ControllerPublishVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{17} +} + +func (m *ControllerPublishVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerPublishVolumeRequest.Unmarshal(m, b) +} +func (m *ControllerPublishVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerPublishVolumeRequest.Marshal(b, m, deterministic) +} +func (m *ControllerPublishVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerPublishVolumeRequest.Merge(m, src) +} +func (m *ControllerPublishVolumeRequest) XXX_Size() int { + return xxx_messageInfo_ControllerPublishVolumeRequest.Size(m) +} +func (m *ControllerPublishVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerPublishVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerPublishVolumeRequest proto.InternalMessageInfo + +func (m *ControllerPublishVolumeRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *ControllerPublishVolumeRequest) GetNodeId() string { + if m != nil { + return m.NodeId + } + return "" +} + +func (m *ControllerPublishVolumeRequest) GetVolumeCapability() *VolumeCapability { + if m != nil { + return m.VolumeCapability + } + return nil +} + +func (m *ControllerPublishVolumeRequest) GetReadonly() bool { + if m != nil { + return m.Readonly + } + return false +} + +func (m *ControllerPublishVolumeRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +func (m *ControllerPublishVolumeRequest) GetVolumeContext() map[string]string { + if m != nil { + return m.VolumeContext + } + return nil +} + +type ControllerPublishVolumeResponse struct { + // Opaque static publish properties of the volume. SP MAY use this + // field to ensure subsequent `NodeStageVolume` or `NodePublishVolume` + // calls calls have contextual information. + // The contents of this field SHALL be opaque to a CO. + // The contents of this field SHALL NOT be mutable. + // The contents of this field SHALL be safe for the CO to cache. + // The contents of this field SHOULD NOT contain sensitive + // information. + // The contents of this field SHOULD NOT be used for uniquely + // identifying a volume. The `volume_id` alone SHOULD be sufficient to + // identify the volume. + // This field is OPTIONAL and when present MUST be passed to + // subsequent `NodeStageVolume` or `NodePublishVolume` calls + PublishContext map[string]string `protobuf:"bytes,1,rep,name=publish_context,json=publishContext,proto3" json:"publish_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerPublishVolumeResponse) Reset() { *m = ControllerPublishVolumeResponse{} } +func (m *ControllerPublishVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*ControllerPublishVolumeResponse) ProtoMessage() {} +func (*ControllerPublishVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{18} +} + +func (m *ControllerPublishVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerPublishVolumeResponse.Unmarshal(m, b) +} +func (m *ControllerPublishVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerPublishVolumeResponse.Marshal(b, m, deterministic) +} +func (m *ControllerPublishVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerPublishVolumeResponse.Merge(m, src) +} +func (m *ControllerPublishVolumeResponse) XXX_Size() int { + return xxx_messageInfo_ControllerPublishVolumeResponse.Size(m) +} +func (m *ControllerPublishVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerPublishVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerPublishVolumeResponse proto.InternalMessageInfo + +func (m *ControllerPublishVolumeResponse) GetPublishContext() map[string]string { + if m != nil { + return m.PublishContext + } + return nil +} + +type ControllerUnpublishVolumeRequest struct { + // The ID of the volume. This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // The ID of the node. This field is OPTIONAL. The CO SHOULD set this + // field to match the node ID returned by `NodeGetInfo` or leave it + // unset. If the value is set, the SP MUST unpublish the volume from + // the specified node. If the value is unset, the SP MUST unpublish + // the volume from all nodes it is published to. + NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` + // Secrets required by plugin to complete controller unpublish volume + // request. This SHOULD be the same secrets passed to the + // ControllerPublishVolume call for the specified volume. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + Secrets map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerUnpublishVolumeRequest) Reset() { *m = ControllerUnpublishVolumeRequest{} } +func (m *ControllerUnpublishVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*ControllerUnpublishVolumeRequest) ProtoMessage() {} +func (*ControllerUnpublishVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{19} +} + +func (m *ControllerUnpublishVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerUnpublishVolumeRequest.Unmarshal(m, b) +} +func (m *ControllerUnpublishVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerUnpublishVolumeRequest.Marshal(b, m, deterministic) +} +func (m *ControllerUnpublishVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerUnpublishVolumeRequest.Merge(m, src) +} +func (m *ControllerUnpublishVolumeRequest) XXX_Size() int { + return xxx_messageInfo_ControllerUnpublishVolumeRequest.Size(m) +} +func (m *ControllerUnpublishVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerUnpublishVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerUnpublishVolumeRequest proto.InternalMessageInfo + +func (m *ControllerUnpublishVolumeRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *ControllerUnpublishVolumeRequest) GetNodeId() string { + if m != nil { + return m.NodeId + } + return "" +} + +func (m *ControllerUnpublishVolumeRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +type ControllerUnpublishVolumeResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerUnpublishVolumeResponse) Reset() { *m = ControllerUnpublishVolumeResponse{} } +func (m *ControllerUnpublishVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*ControllerUnpublishVolumeResponse) ProtoMessage() {} +func (*ControllerUnpublishVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{20} +} + +func (m *ControllerUnpublishVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerUnpublishVolumeResponse.Unmarshal(m, b) +} +func (m *ControllerUnpublishVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerUnpublishVolumeResponse.Marshal(b, m, deterministic) +} +func (m *ControllerUnpublishVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerUnpublishVolumeResponse.Merge(m, src) +} +func (m *ControllerUnpublishVolumeResponse) XXX_Size() int { + return xxx_messageInfo_ControllerUnpublishVolumeResponse.Size(m) +} +func (m *ControllerUnpublishVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerUnpublishVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerUnpublishVolumeResponse proto.InternalMessageInfo + +type ValidateVolumeCapabilitiesRequest struct { + // The ID of the volume to check. This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // Volume context as returned by SP in + // CreateVolumeResponse.Volume.volume_context. + // This field is OPTIONAL and MUST match the volume_context of the + // volume identified by `volume_id`. + VolumeContext map[string]string `protobuf:"bytes,2,rep,name=volume_context,json=volumeContext,proto3" json:"volume_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // The capabilities that the CO wants to check for the volume. This + // call SHALL return "confirmed" only if all the volume capabilities + // specified below are supported. This field is REQUIRED. + VolumeCapabilities []*VolumeCapability `protobuf:"bytes,3,rep,name=volume_capabilities,json=volumeCapabilities,proto3" json:"volume_capabilities,omitempty"` + // See CreateVolumeRequest.parameters. + // This field is OPTIONAL. + Parameters map[string]string `protobuf:"bytes,4,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Secrets required by plugin to complete volume validation request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + Secrets map[string]string `protobuf:"bytes,5,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // See CreateVolumeRequest.mutable_parameters. + // This field is OPTIONAL. + MutableParameters map[string]string `protobuf:"bytes,6,rep,name=mutable_parameters,json=mutableParameters,proto3" json:"mutable_parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ValidateVolumeCapabilitiesRequest) Reset() { *m = ValidateVolumeCapabilitiesRequest{} } +func (m *ValidateVolumeCapabilitiesRequest) String() string { return proto.CompactTextString(m) } +func (*ValidateVolumeCapabilitiesRequest) ProtoMessage() {} +func (*ValidateVolumeCapabilitiesRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{21} +} + +func (m *ValidateVolumeCapabilitiesRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ValidateVolumeCapabilitiesRequest.Unmarshal(m, b) +} +func (m *ValidateVolumeCapabilitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ValidateVolumeCapabilitiesRequest.Marshal(b, m, deterministic) +} +func (m *ValidateVolumeCapabilitiesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValidateVolumeCapabilitiesRequest.Merge(m, src) +} +func (m *ValidateVolumeCapabilitiesRequest) XXX_Size() int { + return xxx_messageInfo_ValidateVolumeCapabilitiesRequest.Size(m) +} +func (m *ValidateVolumeCapabilitiesRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ValidateVolumeCapabilitiesRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ValidateVolumeCapabilitiesRequest proto.InternalMessageInfo + +func (m *ValidateVolumeCapabilitiesRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *ValidateVolumeCapabilitiesRequest) GetVolumeContext() map[string]string { + if m != nil { + return m.VolumeContext + } + return nil +} + +func (m *ValidateVolumeCapabilitiesRequest) GetVolumeCapabilities() []*VolumeCapability { + if m != nil { + return m.VolumeCapabilities + } + return nil +} + +func (m *ValidateVolumeCapabilitiesRequest) GetParameters() map[string]string { + if m != nil { + return m.Parameters + } + return nil +} + +func (m *ValidateVolumeCapabilitiesRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +func (m *ValidateVolumeCapabilitiesRequest) GetMutableParameters() map[string]string { + if m != nil { + return m.MutableParameters + } + return nil +} + +type ValidateVolumeCapabilitiesResponse struct { + // Confirmed indicates to the CO the set of capabilities that the + // plugin has validated. This field SHALL only be set to a non-empty + // value for successful validation responses. + // For successful validation responses, the CO SHALL compare the + // fields of this message to the originally requested capabilities in + // order to guard against an older plugin reporting "valid" for newer + // capability fields that it does not yet understand. + // This field is OPTIONAL. + Confirmed *ValidateVolumeCapabilitiesResponse_Confirmed `protobuf:"bytes,1,opt,name=confirmed,proto3" json:"confirmed,omitempty"` + // Message to the CO if `confirmed` above is empty. This field is + // OPTIONAL. + // An empty string is equal to an unspecified field value. + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ValidateVolumeCapabilitiesResponse) Reset() { *m = ValidateVolumeCapabilitiesResponse{} } +func (m *ValidateVolumeCapabilitiesResponse) String() string { return proto.CompactTextString(m) } +func (*ValidateVolumeCapabilitiesResponse) ProtoMessage() {} +func (*ValidateVolumeCapabilitiesResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{22} +} + +func (m *ValidateVolumeCapabilitiesResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ValidateVolumeCapabilitiesResponse.Unmarshal(m, b) +} +func (m *ValidateVolumeCapabilitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ValidateVolumeCapabilitiesResponse.Marshal(b, m, deterministic) +} +func (m *ValidateVolumeCapabilitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValidateVolumeCapabilitiesResponse.Merge(m, src) +} +func (m *ValidateVolumeCapabilitiesResponse) XXX_Size() int { + return xxx_messageInfo_ValidateVolumeCapabilitiesResponse.Size(m) +} +func (m *ValidateVolumeCapabilitiesResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ValidateVolumeCapabilitiesResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ValidateVolumeCapabilitiesResponse proto.InternalMessageInfo + +func (m *ValidateVolumeCapabilitiesResponse) GetConfirmed() *ValidateVolumeCapabilitiesResponse_Confirmed { + if m != nil { + return m.Confirmed + } + return nil +} + +func (m *ValidateVolumeCapabilitiesResponse) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +type ValidateVolumeCapabilitiesResponse_Confirmed struct { + // Volume context validated by the plugin. + // This field is OPTIONAL. + VolumeContext map[string]string `protobuf:"bytes,1,rep,name=volume_context,json=volumeContext,proto3" json:"volume_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Volume capabilities supported by the plugin. + // This field is REQUIRED. + VolumeCapabilities []*VolumeCapability `protobuf:"bytes,2,rep,name=volume_capabilities,json=volumeCapabilities,proto3" json:"volume_capabilities,omitempty"` + // The volume creation parameters validated by the plugin. + // This field is OPTIONAL. + Parameters map[string]string `protobuf:"bytes,3,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // The volume creation mutable_parameters validated by the plugin. + // This field is OPTIONAL. + MutableParameters map[string]string `protobuf:"bytes,4,rep,name=mutable_parameters,json=mutableParameters,proto3" json:"mutable_parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) Reset() { + *m = ValidateVolumeCapabilitiesResponse_Confirmed{} +} +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) String() string { + return proto.CompactTextString(m) +} +func (*ValidateVolumeCapabilitiesResponse_Confirmed) ProtoMessage() {} +func (*ValidateVolumeCapabilitiesResponse_Confirmed) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{22, 0} +} + +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Unmarshal(m, b) +} +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Marshal(b, m, deterministic) +} +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Merge(m, src) +} +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_Size() int { + return xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.Size(m) +} +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) XXX_DiscardUnknown() { + xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed.DiscardUnknown(m) +} + +var xxx_messageInfo_ValidateVolumeCapabilitiesResponse_Confirmed proto.InternalMessageInfo + +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) GetVolumeContext() map[string]string { + if m != nil { + return m.VolumeContext + } + return nil +} + +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) GetVolumeCapabilities() []*VolumeCapability { + if m != nil { + return m.VolumeCapabilities + } + return nil +} + +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) GetParameters() map[string]string { + if m != nil { + return m.Parameters + } + return nil +} + +func (m *ValidateVolumeCapabilitiesResponse_Confirmed) GetMutableParameters() map[string]string { + if m != nil { + return m.MutableParameters + } + return nil +} + +type ListVolumesRequest struct { + // If specified (non-zero value), the Plugin MUST NOT return more + // entries than this number in the response. If the actual number of + // entries is more than this number, the Plugin MUST set `next_token` + // in the response which can be used to get the next page of entries + // in the subsequent `ListVolumes` call. This field is OPTIONAL. If + // not specified (zero value), it means there is no restriction on the + // number of entries that can be returned. + // The value of this field MUST NOT be negative. + MaxEntries int32 `protobuf:"varint,1,opt,name=max_entries,json=maxEntries,proto3" json:"max_entries,omitempty"` + // A token to specify where to start paginating. Set this field to + // `next_token` returned by a previous `ListVolumes` call to get the + // next page of entries. This field is OPTIONAL. + // An empty string is equal to an unspecified field value. + StartingToken string `protobuf:"bytes,2,opt,name=starting_token,json=startingToken,proto3" json:"starting_token,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListVolumesRequest) Reset() { *m = ListVolumesRequest{} } +func (m *ListVolumesRequest) String() string { return proto.CompactTextString(m) } +func (*ListVolumesRequest) ProtoMessage() {} +func (*ListVolumesRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{23} +} + +func (m *ListVolumesRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListVolumesRequest.Unmarshal(m, b) +} +func (m *ListVolumesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListVolumesRequest.Marshal(b, m, deterministic) +} +func (m *ListVolumesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListVolumesRequest.Merge(m, src) +} +func (m *ListVolumesRequest) XXX_Size() int { + return xxx_messageInfo_ListVolumesRequest.Size(m) +} +func (m *ListVolumesRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ListVolumesRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ListVolumesRequest proto.InternalMessageInfo + +func (m *ListVolumesRequest) GetMaxEntries() int32 { + if m != nil { + return m.MaxEntries + } + return 0 +} + +func (m *ListVolumesRequest) GetStartingToken() string { + if m != nil { + return m.StartingToken + } + return "" +} + +type ListVolumesResponse struct { + Entries []*ListVolumesResponse_Entry `protobuf:"bytes,1,rep,name=entries,proto3" json:"entries,omitempty"` + // This token allows you to get the next page of entries for + // `ListVolumes` request. If the number of entries is larger than + // `max_entries`, use the `next_token` as a value for the + // `starting_token` field in the next `ListVolumes` request. This + // field is OPTIONAL. + // An empty string is equal to an unspecified field value. + NextToken string `protobuf:"bytes,2,opt,name=next_token,json=nextToken,proto3" json:"next_token,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListVolumesResponse) Reset() { *m = ListVolumesResponse{} } +func (m *ListVolumesResponse) String() string { return proto.CompactTextString(m) } +func (*ListVolumesResponse) ProtoMessage() {} +func (*ListVolumesResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{24} +} + +func (m *ListVolumesResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListVolumesResponse.Unmarshal(m, b) +} +func (m *ListVolumesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListVolumesResponse.Marshal(b, m, deterministic) +} +func (m *ListVolumesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListVolumesResponse.Merge(m, src) +} +func (m *ListVolumesResponse) XXX_Size() int { + return xxx_messageInfo_ListVolumesResponse.Size(m) +} +func (m *ListVolumesResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ListVolumesResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ListVolumesResponse proto.InternalMessageInfo + +func (m *ListVolumesResponse) GetEntries() []*ListVolumesResponse_Entry { + if m != nil { + return m.Entries + } + return nil +} + +func (m *ListVolumesResponse) GetNextToken() string { + if m != nil { + return m.NextToken + } + return "" +} + +type ListVolumesResponse_VolumeStatus struct { + // A list of all `node_id` of nodes that the volume in this entry + // is controller published on. + // This field is OPTIONAL. If it is not specified and the SP has + // the LIST_VOLUMES_PUBLISHED_NODES controller capability, the CO + // MAY assume the volume is not controller published to any nodes. + // If the field is not specified and the SP does not have the + // LIST_VOLUMES_PUBLISHED_NODES controller capability, the CO MUST + // not interpret this field. + // published_node_ids MAY include nodes not published to or + // reported by the SP. The CO MUST be resilient to that. + PublishedNodeIds []string `protobuf:"bytes,1,rep,name=published_node_ids,json=publishedNodeIds,proto3" json:"published_node_ids,omitempty"` + // Information about the current condition of the volume. + // This field is OPTIONAL. + // This field MUST be specified if the + // VOLUME_CONDITION controller capability is supported. + VolumeCondition *VolumeCondition `protobuf:"bytes,2,opt,name=volume_condition,json=volumeCondition,proto3" json:"volume_condition,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListVolumesResponse_VolumeStatus) Reset() { *m = ListVolumesResponse_VolumeStatus{} } +func (m *ListVolumesResponse_VolumeStatus) String() string { return proto.CompactTextString(m) } +func (*ListVolumesResponse_VolumeStatus) ProtoMessage() {} +func (*ListVolumesResponse_VolumeStatus) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{24, 0} +} + +func (m *ListVolumesResponse_VolumeStatus) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListVolumesResponse_VolumeStatus.Unmarshal(m, b) +} +func (m *ListVolumesResponse_VolumeStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListVolumesResponse_VolumeStatus.Marshal(b, m, deterministic) +} +func (m *ListVolumesResponse_VolumeStatus) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListVolumesResponse_VolumeStatus.Merge(m, src) +} +func (m *ListVolumesResponse_VolumeStatus) XXX_Size() int { + return xxx_messageInfo_ListVolumesResponse_VolumeStatus.Size(m) +} +func (m *ListVolumesResponse_VolumeStatus) XXX_DiscardUnknown() { + xxx_messageInfo_ListVolumesResponse_VolumeStatus.DiscardUnknown(m) +} + +var xxx_messageInfo_ListVolumesResponse_VolumeStatus proto.InternalMessageInfo + +func (m *ListVolumesResponse_VolumeStatus) GetPublishedNodeIds() []string { + if m != nil { + return m.PublishedNodeIds + } + return nil +} + +func (m *ListVolumesResponse_VolumeStatus) GetVolumeCondition() *VolumeCondition { + if m != nil { + return m.VolumeCondition + } + return nil +} + +type ListVolumesResponse_Entry struct { + // This field is REQUIRED + Volume *Volume `protobuf:"bytes,1,opt,name=volume,proto3" json:"volume,omitempty"` + // This field is OPTIONAL. This field MUST be specified if the + // LIST_VOLUMES_PUBLISHED_NODES controller capability is + // supported. + Status *ListVolumesResponse_VolumeStatus `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListVolumesResponse_Entry) Reset() { *m = ListVolumesResponse_Entry{} } +func (m *ListVolumesResponse_Entry) String() string { return proto.CompactTextString(m) } +func (*ListVolumesResponse_Entry) ProtoMessage() {} +func (*ListVolumesResponse_Entry) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{24, 1} +} + +func (m *ListVolumesResponse_Entry) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListVolumesResponse_Entry.Unmarshal(m, b) +} +func (m *ListVolumesResponse_Entry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListVolumesResponse_Entry.Marshal(b, m, deterministic) +} +func (m *ListVolumesResponse_Entry) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListVolumesResponse_Entry.Merge(m, src) +} +func (m *ListVolumesResponse_Entry) XXX_Size() int { + return xxx_messageInfo_ListVolumesResponse_Entry.Size(m) +} +func (m *ListVolumesResponse_Entry) XXX_DiscardUnknown() { + xxx_messageInfo_ListVolumesResponse_Entry.DiscardUnknown(m) +} + +var xxx_messageInfo_ListVolumesResponse_Entry proto.InternalMessageInfo + +func (m *ListVolumesResponse_Entry) GetVolume() *Volume { + if m != nil { + return m.Volume + } + return nil +} + +func (m *ListVolumesResponse_Entry) GetStatus() *ListVolumesResponse_VolumeStatus { + if m != nil { + return m.Status + } + return nil +} + +type ControllerGetVolumeRequest struct { + // The ID of the volume to fetch current volume information for. + // This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerGetVolumeRequest) Reset() { *m = ControllerGetVolumeRequest{} } +func (m *ControllerGetVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*ControllerGetVolumeRequest) ProtoMessage() {} +func (*ControllerGetVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{25} +} + +func (m *ControllerGetVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerGetVolumeRequest.Unmarshal(m, b) +} +func (m *ControllerGetVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerGetVolumeRequest.Marshal(b, m, deterministic) +} +func (m *ControllerGetVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerGetVolumeRequest.Merge(m, src) +} +func (m *ControllerGetVolumeRequest) XXX_Size() int { + return xxx_messageInfo_ControllerGetVolumeRequest.Size(m) +} +func (m *ControllerGetVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerGetVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerGetVolumeRequest proto.InternalMessageInfo + +func (m *ControllerGetVolumeRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +type ControllerGetVolumeResponse struct { + // This field is REQUIRED + Volume *Volume `protobuf:"bytes,1,opt,name=volume,proto3" json:"volume,omitempty"` + // This field is REQUIRED. + Status *ControllerGetVolumeResponse_VolumeStatus `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerGetVolumeResponse) Reset() { *m = ControllerGetVolumeResponse{} } +func (m *ControllerGetVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*ControllerGetVolumeResponse) ProtoMessage() {} +func (*ControllerGetVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{26} +} + +func (m *ControllerGetVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerGetVolumeResponse.Unmarshal(m, b) +} +func (m *ControllerGetVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerGetVolumeResponse.Marshal(b, m, deterministic) +} +func (m *ControllerGetVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerGetVolumeResponse.Merge(m, src) +} +func (m *ControllerGetVolumeResponse) XXX_Size() int { + return xxx_messageInfo_ControllerGetVolumeResponse.Size(m) +} +func (m *ControllerGetVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerGetVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerGetVolumeResponse proto.InternalMessageInfo + +func (m *ControllerGetVolumeResponse) GetVolume() *Volume { + if m != nil { + return m.Volume + } + return nil +} + +func (m *ControllerGetVolumeResponse) GetStatus() *ControllerGetVolumeResponse_VolumeStatus { + if m != nil { + return m.Status + } + return nil +} + +type ControllerGetVolumeResponse_VolumeStatus struct { + // A list of all the `node_id` of nodes that this volume is + // controller published on. + // This field is OPTIONAL. + // This field MUST be specified if the LIST_VOLUMES_PUBLISHED_NODES + // controller capability is supported. + // published_node_ids MAY include nodes not published to or + // reported by the SP. The CO MUST be resilient to that. + PublishedNodeIds []string `protobuf:"bytes,1,rep,name=published_node_ids,json=publishedNodeIds,proto3" json:"published_node_ids,omitempty"` + // Information about the current condition of the volume. + // This field is OPTIONAL. + // This field MUST be specified if the + // VOLUME_CONDITION controller capability is supported. + VolumeCondition *VolumeCondition `protobuf:"bytes,2,opt,name=volume_condition,json=volumeCondition,proto3" json:"volume_condition,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerGetVolumeResponse_VolumeStatus) Reset() { + *m = ControllerGetVolumeResponse_VolumeStatus{} +} +func (m *ControllerGetVolumeResponse_VolumeStatus) String() string { return proto.CompactTextString(m) } +func (*ControllerGetVolumeResponse_VolumeStatus) ProtoMessage() {} +func (*ControllerGetVolumeResponse_VolumeStatus) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{26, 0} +} + +func (m *ControllerGetVolumeResponse_VolumeStatus) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerGetVolumeResponse_VolumeStatus.Unmarshal(m, b) +} +func (m *ControllerGetVolumeResponse_VolumeStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerGetVolumeResponse_VolumeStatus.Marshal(b, m, deterministic) +} +func (m *ControllerGetVolumeResponse_VolumeStatus) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerGetVolumeResponse_VolumeStatus.Merge(m, src) +} +func (m *ControllerGetVolumeResponse_VolumeStatus) XXX_Size() int { + return xxx_messageInfo_ControllerGetVolumeResponse_VolumeStatus.Size(m) +} +func (m *ControllerGetVolumeResponse_VolumeStatus) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerGetVolumeResponse_VolumeStatus.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerGetVolumeResponse_VolumeStatus proto.InternalMessageInfo + +func (m *ControllerGetVolumeResponse_VolumeStatus) GetPublishedNodeIds() []string { + if m != nil { + return m.PublishedNodeIds + } + return nil +} + +func (m *ControllerGetVolumeResponse_VolumeStatus) GetVolumeCondition() *VolumeCondition { + if m != nil { + return m.VolumeCondition + } + return nil +} + +type ControllerModifyVolumeRequest struct { + // Contains identity information for the existing volume. + // This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // Secrets required by plugin to complete modify volume request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + Secrets map[string]string `protobuf:"bytes,2,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Plugin specific volume attributes to mutate, passed in as + // opaque key-value pairs. + // This field is REQUIRED. The Plugin is responsible for + // parsing and validating these parameters. COs will treat these + // as opaque. The CO SHOULD specify the intended values of all mutable + // parameters it intends to modify. SPs MUST NOT modify volumes based + // on the absence of keys, only keys that are specified should result + // in modifications to the volume. + MutableParameters map[string]string `protobuf:"bytes,3,rep,name=mutable_parameters,json=mutableParameters,proto3" json:"mutable_parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerModifyVolumeRequest) Reset() { *m = ControllerModifyVolumeRequest{} } +func (m *ControllerModifyVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*ControllerModifyVolumeRequest) ProtoMessage() {} +func (*ControllerModifyVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{27} +} + +func (m *ControllerModifyVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerModifyVolumeRequest.Unmarshal(m, b) +} +func (m *ControllerModifyVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerModifyVolumeRequest.Marshal(b, m, deterministic) +} +func (m *ControllerModifyVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerModifyVolumeRequest.Merge(m, src) +} +func (m *ControllerModifyVolumeRequest) XXX_Size() int { + return xxx_messageInfo_ControllerModifyVolumeRequest.Size(m) +} +func (m *ControllerModifyVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerModifyVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerModifyVolumeRequest proto.InternalMessageInfo + +func (m *ControllerModifyVolumeRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *ControllerModifyVolumeRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +func (m *ControllerModifyVolumeRequest) GetMutableParameters() map[string]string { + if m != nil { + return m.MutableParameters + } + return nil +} + +type ControllerModifyVolumeResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerModifyVolumeResponse) Reset() { *m = ControllerModifyVolumeResponse{} } +func (m *ControllerModifyVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*ControllerModifyVolumeResponse) ProtoMessage() {} +func (*ControllerModifyVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{28} +} + +func (m *ControllerModifyVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerModifyVolumeResponse.Unmarshal(m, b) +} +func (m *ControllerModifyVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerModifyVolumeResponse.Marshal(b, m, deterministic) +} +func (m *ControllerModifyVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerModifyVolumeResponse.Merge(m, src) +} +func (m *ControllerModifyVolumeResponse) XXX_Size() int { + return xxx_messageInfo_ControllerModifyVolumeResponse.Size(m) +} +func (m *ControllerModifyVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerModifyVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerModifyVolumeResponse proto.InternalMessageInfo + +type GetCapacityRequest struct { + // If specified, the Plugin SHALL report the capacity of the storage + // that can be used to provision volumes that satisfy ALL of the + // specified `volume_capabilities`. These are the same + // `volume_capabilities` the CO will use in `CreateVolumeRequest`. + // This field is OPTIONAL. + VolumeCapabilities []*VolumeCapability `protobuf:"bytes,1,rep,name=volume_capabilities,json=volumeCapabilities,proto3" json:"volume_capabilities,omitempty"` + // If specified, the Plugin SHALL report the capacity of the storage + // that can be used to provision volumes with the given Plugin + // specific `parameters`. These are the same `parameters` the CO will + // use in `CreateVolumeRequest`. This field is OPTIONAL. + Parameters map[string]string `protobuf:"bytes,2,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // If specified, the Plugin SHALL report the capacity of the storage + // that can be used to provision volumes that in the specified + // `accessible_topology`. This is the same as the + // `accessible_topology` the CO returns in a `CreateVolumeResponse`. + // This field is OPTIONAL. This field SHALL NOT be set unless the + // plugin advertises the VOLUME_ACCESSIBILITY_CONSTRAINTS capability. + AccessibleTopology *Topology `protobuf:"bytes,3,opt,name=accessible_topology,json=accessibleTopology,proto3" json:"accessible_topology,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetCapacityRequest) Reset() { *m = GetCapacityRequest{} } +func (m *GetCapacityRequest) String() string { return proto.CompactTextString(m) } +func (*GetCapacityRequest) ProtoMessage() {} +func (*GetCapacityRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{29} +} + +func (m *GetCapacityRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetCapacityRequest.Unmarshal(m, b) +} +func (m *GetCapacityRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetCapacityRequest.Marshal(b, m, deterministic) +} +func (m *GetCapacityRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetCapacityRequest.Merge(m, src) +} +func (m *GetCapacityRequest) XXX_Size() int { + return xxx_messageInfo_GetCapacityRequest.Size(m) +} +func (m *GetCapacityRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetCapacityRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetCapacityRequest proto.InternalMessageInfo + +func (m *GetCapacityRequest) GetVolumeCapabilities() []*VolumeCapability { + if m != nil { + return m.VolumeCapabilities + } + return nil +} + +func (m *GetCapacityRequest) GetParameters() map[string]string { + if m != nil { + return m.Parameters + } + return nil +} + +func (m *GetCapacityRequest) GetAccessibleTopology() *Topology { + if m != nil { + return m.AccessibleTopology + } + return nil +} + +type GetCapacityResponse struct { + // The available capacity, in bytes, of the storage that can be used + // to provision volumes. If `volume_capabilities` or `parameters` is + // specified in the request, the Plugin SHALL take those into + // consideration when calculating the available capacity of the + // storage. This field is REQUIRED. + // The value of this field MUST NOT be negative. + AvailableCapacity int64 `protobuf:"varint,1,opt,name=available_capacity,json=availableCapacity,proto3" json:"available_capacity,omitempty"` + // The largest size that may be used in a + // CreateVolumeRequest.capacity_range.required_bytes field + // to create a volume with the same parameters as those in + // GetCapacityRequest. + // + // If `volume_capabilities` or `parameters` is + // specified in the request, the Plugin SHALL take those into + // consideration when calculating the minimum volume size of the + // storage. + // + // This field is OPTIONAL. MUST NOT be negative. + // The Plugin SHOULD provide a value for this field if it has + // a maximum size for individual volumes and leave it unset + // otherwise. COs MAY use it to make decision about + // where to create volumes. + MaximumVolumeSize *wrappers.Int64Value `protobuf:"bytes,2,opt,name=maximum_volume_size,json=maximumVolumeSize,proto3" json:"maximum_volume_size,omitempty"` + // The smallest size that may be used in a + // CreateVolumeRequest.capacity_range.limit_bytes field + // to create a volume with the same parameters as those in + // GetCapacityRequest. + // + // If `volume_capabilities` or `parameters` is + // specified in the request, the Plugin SHALL take those into + // consideration when calculating the maximum volume size of the + // storage. + // + // This field is OPTIONAL. MUST NOT be negative. + // The Plugin SHOULD provide a value for this field if it has + // a minimum size for individual volumes and leave it unset + // otherwise. COs MAY use it to make decision about + // where to create volumes. + MinimumVolumeSize *wrappers.Int64Value `protobuf:"bytes,3,opt,name=minimum_volume_size,json=minimumVolumeSize,proto3" json:"minimum_volume_size,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetCapacityResponse) Reset() { *m = GetCapacityResponse{} } +func (m *GetCapacityResponse) String() string { return proto.CompactTextString(m) } +func (*GetCapacityResponse) ProtoMessage() {} +func (*GetCapacityResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{30} +} + +func (m *GetCapacityResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetCapacityResponse.Unmarshal(m, b) +} +func (m *GetCapacityResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetCapacityResponse.Marshal(b, m, deterministic) +} +func (m *GetCapacityResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetCapacityResponse.Merge(m, src) +} +func (m *GetCapacityResponse) XXX_Size() int { + return xxx_messageInfo_GetCapacityResponse.Size(m) +} +func (m *GetCapacityResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetCapacityResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetCapacityResponse proto.InternalMessageInfo + +func (m *GetCapacityResponse) GetAvailableCapacity() int64 { + if m != nil { + return m.AvailableCapacity + } + return 0 +} + +func (m *GetCapacityResponse) GetMaximumVolumeSize() *wrappers.Int64Value { + if m != nil { + return m.MaximumVolumeSize + } + return nil +} + +func (m *GetCapacityResponse) GetMinimumVolumeSize() *wrappers.Int64Value { + if m != nil { + return m.MinimumVolumeSize + } + return nil +} + +type ControllerGetCapabilitiesRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerGetCapabilitiesRequest) Reset() { *m = ControllerGetCapabilitiesRequest{} } +func (m *ControllerGetCapabilitiesRequest) String() string { return proto.CompactTextString(m) } +func (*ControllerGetCapabilitiesRequest) ProtoMessage() {} +func (*ControllerGetCapabilitiesRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{31} +} + +func (m *ControllerGetCapabilitiesRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerGetCapabilitiesRequest.Unmarshal(m, b) +} +func (m *ControllerGetCapabilitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerGetCapabilitiesRequest.Marshal(b, m, deterministic) +} +func (m *ControllerGetCapabilitiesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerGetCapabilitiesRequest.Merge(m, src) +} +func (m *ControllerGetCapabilitiesRequest) XXX_Size() int { + return xxx_messageInfo_ControllerGetCapabilitiesRequest.Size(m) +} +func (m *ControllerGetCapabilitiesRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerGetCapabilitiesRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerGetCapabilitiesRequest proto.InternalMessageInfo + +type ControllerGetCapabilitiesResponse struct { + // All the capabilities that the controller service supports. This + // field is OPTIONAL. + Capabilities []*ControllerServiceCapability `protobuf:"bytes,1,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerGetCapabilitiesResponse) Reset() { *m = ControllerGetCapabilitiesResponse{} } +func (m *ControllerGetCapabilitiesResponse) String() string { return proto.CompactTextString(m) } +func (*ControllerGetCapabilitiesResponse) ProtoMessage() {} +func (*ControllerGetCapabilitiesResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{32} +} + +func (m *ControllerGetCapabilitiesResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerGetCapabilitiesResponse.Unmarshal(m, b) +} +func (m *ControllerGetCapabilitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerGetCapabilitiesResponse.Marshal(b, m, deterministic) +} +func (m *ControllerGetCapabilitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerGetCapabilitiesResponse.Merge(m, src) +} +func (m *ControllerGetCapabilitiesResponse) XXX_Size() int { + return xxx_messageInfo_ControllerGetCapabilitiesResponse.Size(m) +} +func (m *ControllerGetCapabilitiesResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerGetCapabilitiesResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerGetCapabilitiesResponse proto.InternalMessageInfo + +func (m *ControllerGetCapabilitiesResponse) GetCapabilities() []*ControllerServiceCapability { + if m != nil { + return m.Capabilities + } + return nil +} + +// Specifies a capability of the controller service. +type ControllerServiceCapability struct { + // Types that are valid to be assigned to Type: + // + // *ControllerServiceCapability_Rpc + Type isControllerServiceCapability_Type `protobuf_oneof:"type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerServiceCapability) Reset() { *m = ControllerServiceCapability{} } +func (m *ControllerServiceCapability) String() string { return proto.CompactTextString(m) } +func (*ControllerServiceCapability) ProtoMessage() {} +func (*ControllerServiceCapability) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{33} +} + +func (m *ControllerServiceCapability) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerServiceCapability.Unmarshal(m, b) +} +func (m *ControllerServiceCapability) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerServiceCapability.Marshal(b, m, deterministic) +} +func (m *ControllerServiceCapability) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerServiceCapability.Merge(m, src) +} +func (m *ControllerServiceCapability) XXX_Size() int { + return xxx_messageInfo_ControllerServiceCapability.Size(m) +} +func (m *ControllerServiceCapability) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerServiceCapability.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerServiceCapability proto.InternalMessageInfo + +type isControllerServiceCapability_Type interface { + isControllerServiceCapability_Type() +} + +type ControllerServiceCapability_Rpc struct { + Rpc *ControllerServiceCapability_RPC `protobuf:"bytes,1,opt,name=rpc,proto3,oneof"` +} + +func (*ControllerServiceCapability_Rpc) isControllerServiceCapability_Type() {} + +func (m *ControllerServiceCapability) GetType() isControllerServiceCapability_Type { + if m != nil { + return m.Type + } + return nil +} + +func (m *ControllerServiceCapability) GetRpc() *ControllerServiceCapability_RPC { + if x, ok := m.GetType().(*ControllerServiceCapability_Rpc); ok { + return x.Rpc + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*ControllerServiceCapability) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*ControllerServiceCapability_Rpc)(nil), + } +} + +type ControllerServiceCapability_RPC struct { + Type ControllerServiceCapability_RPC_Type `protobuf:"varint,1,opt,name=type,proto3,enum=csi.v1.ControllerServiceCapability_RPC_Type" json:"type,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerServiceCapability_RPC) Reset() { *m = ControllerServiceCapability_RPC{} } +func (m *ControllerServiceCapability_RPC) String() string { return proto.CompactTextString(m) } +func (*ControllerServiceCapability_RPC) ProtoMessage() {} +func (*ControllerServiceCapability_RPC) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{33, 0} +} + +func (m *ControllerServiceCapability_RPC) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerServiceCapability_RPC.Unmarshal(m, b) +} +func (m *ControllerServiceCapability_RPC) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerServiceCapability_RPC.Marshal(b, m, deterministic) +} +func (m *ControllerServiceCapability_RPC) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerServiceCapability_RPC.Merge(m, src) +} +func (m *ControllerServiceCapability_RPC) XXX_Size() int { + return xxx_messageInfo_ControllerServiceCapability_RPC.Size(m) +} +func (m *ControllerServiceCapability_RPC) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerServiceCapability_RPC.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerServiceCapability_RPC proto.InternalMessageInfo + +func (m *ControllerServiceCapability_RPC) GetType() ControllerServiceCapability_RPC_Type { + if m != nil { + return m.Type + } + return ControllerServiceCapability_RPC_UNKNOWN +} + +type CreateSnapshotRequest struct { + // The ID of the source volume to be snapshotted. + // This field is REQUIRED. + SourceVolumeId string `protobuf:"bytes,1,opt,name=source_volume_id,json=sourceVolumeId,proto3" json:"source_volume_id,omitempty"` + // The suggested name for the snapshot. This field is REQUIRED for + // idempotency. + // Any Unicode string that conforms to the length limit is allowed + // except those containing the following banned characters: + // U+0000-U+0008, U+000B, U+000C, U+000E-U+001F, U+007F-U+009F. + // (These are control characters other than commonly used whitespace.) + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + // Secrets required by plugin to complete snapshot creation request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + Secrets map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Plugin specific parameters passed in as opaque key-value pairs. + // This field is OPTIONAL. The Plugin is responsible for parsing and + // validating these parameters. COs will treat these as opaque. + // Use cases for opaque parameters: + // - Specify a policy to automatically clean up the snapshot. + // - Specify an expiration date for the snapshot. + // - Specify whether the snapshot is readonly or read/write. + // - Specify if the snapshot should be replicated to some place. + // - Specify primary or secondary for replication systems that + // support snapshotting only on primary. + Parameters map[string]string `protobuf:"bytes,4,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CreateSnapshotRequest) Reset() { *m = CreateSnapshotRequest{} } +func (m *CreateSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*CreateSnapshotRequest) ProtoMessage() {} +func (*CreateSnapshotRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{34} +} + +func (m *CreateSnapshotRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CreateSnapshotRequest.Unmarshal(m, b) +} +func (m *CreateSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CreateSnapshotRequest.Marshal(b, m, deterministic) +} +func (m *CreateSnapshotRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateSnapshotRequest.Merge(m, src) +} +func (m *CreateSnapshotRequest) XXX_Size() int { + return xxx_messageInfo_CreateSnapshotRequest.Size(m) +} +func (m *CreateSnapshotRequest) XXX_DiscardUnknown() { + xxx_messageInfo_CreateSnapshotRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_CreateSnapshotRequest proto.InternalMessageInfo + +func (m *CreateSnapshotRequest) GetSourceVolumeId() string { + if m != nil { + return m.SourceVolumeId + } + return "" +} + +func (m *CreateSnapshotRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *CreateSnapshotRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +func (m *CreateSnapshotRequest) GetParameters() map[string]string { + if m != nil { + return m.Parameters + } + return nil +} + +type CreateSnapshotResponse struct { + // Contains all attributes of the newly created snapshot that are + // relevant to the CO along with information required by the Plugin + // to uniquely identify the snapshot. This field is REQUIRED. + Snapshot *Snapshot `protobuf:"bytes,1,opt,name=snapshot,proto3" json:"snapshot,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CreateSnapshotResponse) Reset() { *m = CreateSnapshotResponse{} } +func (m *CreateSnapshotResponse) String() string { return proto.CompactTextString(m) } +func (*CreateSnapshotResponse) ProtoMessage() {} +func (*CreateSnapshotResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{35} +} + +func (m *CreateSnapshotResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CreateSnapshotResponse.Unmarshal(m, b) +} +func (m *CreateSnapshotResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CreateSnapshotResponse.Marshal(b, m, deterministic) +} +func (m *CreateSnapshotResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateSnapshotResponse.Merge(m, src) +} +func (m *CreateSnapshotResponse) XXX_Size() int { + return xxx_messageInfo_CreateSnapshotResponse.Size(m) +} +func (m *CreateSnapshotResponse) XXX_DiscardUnknown() { + xxx_messageInfo_CreateSnapshotResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_CreateSnapshotResponse proto.InternalMessageInfo + +func (m *CreateSnapshotResponse) GetSnapshot() *Snapshot { + if m != nil { + return m.Snapshot + } + return nil +} + +// Information about a specific snapshot. +type Snapshot struct { + // This is the complete size of the snapshot in bytes. The purpose of + // this field is to give CO guidance on how much space is needed to + // create a volume from this snapshot. The size of the volume MUST NOT + // be less than the size of the source snapshot. This field is + // OPTIONAL. If this field is not set, it indicates that this size is + // unknown. The value of this field MUST NOT be negative and a size of + // zero means it is unspecified. + SizeBytes int64 `protobuf:"varint,1,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + // The identifier for this snapshot, generated by the plugin. + // This field is REQUIRED. + // This field MUST contain enough information to uniquely identify + // this specific snapshot vs all other snapshots supported by this + // plugin. + // This field SHALL be used by the CO in subsequent calls to refer to + // this snapshot. + // The SP is NOT responsible for global uniqueness of snapshot_id + // across multiple SPs. + SnapshotId string `protobuf:"bytes,2,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + // Identity information for the source volume. Note that creating a + // snapshot from a snapshot is not supported here so the source has to + // be a volume. This field is REQUIRED. + SourceVolumeId string `protobuf:"bytes,3,opt,name=source_volume_id,json=sourceVolumeId,proto3" json:"source_volume_id,omitempty"` + // Timestamp when the point-in-time snapshot is taken on the storage + // system. This field is REQUIRED. + CreationTime *timestamp.Timestamp `protobuf:"bytes,4,opt,name=creation_time,json=creationTime,proto3" json:"creation_time,omitempty"` + // Indicates if a snapshot is ready to use as a + // `volume_content_source` in a `CreateVolumeRequest`. The default + // value is false. This field is REQUIRED. + ReadyToUse bool `protobuf:"varint,5,opt,name=ready_to_use,json=readyToUse,proto3" json:"ready_to_use,omitempty"` + // The ID of the volume group snapshot that this snapshot is part of. + // It uniquely identifies the group snapshot on the storage system. + // This field is OPTIONAL. + // If this snapshot is a member of a volume group snapshot, and it + // MUST NOT be deleted as a stand alone snapshot, then the SP + // MUST provide the ID of the volume group snapshot in this field. + // If provided, CO MUST use this field in subsequent volume group + // snapshot operations to indicate that this snapshot is part of the + // specified group snapshot. + // If not provided, CO SHALL treat the snapshot as independent, + // and SP SHALL allow it to be deleted separately. + // If this message is inside a VolumeGroupSnapshot message, the value + // MUST be the same as the group_snapshot_id in that message. + GroupSnapshotId string `protobuf:"bytes,6,opt,name=group_snapshot_id,json=groupSnapshotId,proto3" json:"group_snapshot_id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Snapshot) Reset() { *m = Snapshot{} } +func (m *Snapshot) String() string { return proto.CompactTextString(m) } +func (*Snapshot) ProtoMessage() {} +func (*Snapshot) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{36} +} + +func (m *Snapshot) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Snapshot.Unmarshal(m, b) +} +func (m *Snapshot) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Snapshot.Marshal(b, m, deterministic) +} +func (m *Snapshot) XXX_Merge(src proto.Message) { + xxx_messageInfo_Snapshot.Merge(m, src) +} +func (m *Snapshot) XXX_Size() int { + return xxx_messageInfo_Snapshot.Size(m) +} +func (m *Snapshot) XXX_DiscardUnknown() { + xxx_messageInfo_Snapshot.DiscardUnknown(m) +} + +var xxx_messageInfo_Snapshot proto.InternalMessageInfo + +func (m *Snapshot) GetSizeBytes() int64 { + if m != nil { + return m.SizeBytes + } + return 0 +} + +func (m *Snapshot) GetSnapshotId() string { + if m != nil { + return m.SnapshotId + } + return "" +} + +func (m *Snapshot) GetSourceVolumeId() string { + if m != nil { + return m.SourceVolumeId + } + return "" +} + +func (m *Snapshot) GetCreationTime() *timestamp.Timestamp { + if m != nil { + return m.CreationTime + } + return nil +} + +func (m *Snapshot) GetReadyToUse() bool { + if m != nil { + return m.ReadyToUse + } + return false +} + +func (m *Snapshot) GetGroupSnapshotId() string { + if m != nil { + return m.GroupSnapshotId + } + return "" +} + +type DeleteSnapshotRequest struct { + // The ID of the snapshot to be deleted. + // This field is REQUIRED. + SnapshotId string `protobuf:"bytes,1,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + // Secrets required by plugin to complete snapshot deletion request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + Secrets map[string]string `protobuf:"bytes,2,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteSnapshotRequest) Reset() { *m = DeleteSnapshotRequest{} } +func (m *DeleteSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteSnapshotRequest) ProtoMessage() {} +func (*DeleteSnapshotRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{37} +} + +func (m *DeleteSnapshotRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteSnapshotRequest.Unmarshal(m, b) +} +func (m *DeleteSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteSnapshotRequest.Marshal(b, m, deterministic) +} +func (m *DeleteSnapshotRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteSnapshotRequest.Merge(m, src) +} +func (m *DeleteSnapshotRequest) XXX_Size() int { + return xxx_messageInfo_DeleteSnapshotRequest.Size(m) +} +func (m *DeleteSnapshotRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteSnapshotRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteSnapshotRequest proto.InternalMessageInfo + +func (m *DeleteSnapshotRequest) GetSnapshotId() string { + if m != nil { + return m.SnapshotId + } + return "" +} + +func (m *DeleteSnapshotRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +type DeleteSnapshotResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteSnapshotResponse) Reset() { *m = DeleteSnapshotResponse{} } +func (m *DeleteSnapshotResponse) String() string { return proto.CompactTextString(m) } +func (*DeleteSnapshotResponse) ProtoMessage() {} +func (*DeleteSnapshotResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{38} +} + +func (m *DeleteSnapshotResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteSnapshotResponse.Unmarshal(m, b) +} +func (m *DeleteSnapshotResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteSnapshotResponse.Marshal(b, m, deterministic) +} +func (m *DeleteSnapshotResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteSnapshotResponse.Merge(m, src) +} +func (m *DeleteSnapshotResponse) XXX_Size() int { + return xxx_messageInfo_DeleteSnapshotResponse.Size(m) +} +func (m *DeleteSnapshotResponse) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteSnapshotResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteSnapshotResponse proto.InternalMessageInfo + +// List all snapshots on the storage system regardless of how they were +// created. +type ListSnapshotsRequest struct { + // If specified (non-zero value), the Plugin MUST NOT return more + // entries than this number in the response. If the actual number of + // entries is more than this number, the Plugin MUST set `next_token` + // in the response which can be used to get the next page of entries + // in the subsequent `ListSnapshots` call. This field is OPTIONAL. If + // not specified (zero value), it means there is no restriction on the + // number of entries that can be returned. + // The value of this field MUST NOT be negative. + MaxEntries int32 `protobuf:"varint,1,opt,name=max_entries,json=maxEntries,proto3" json:"max_entries,omitempty"` + // A token to specify where to start paginating. Set this field to + // `next_token` returned by a previous `ListSnapshots` call to get the + // next page of entries. This field is OPTIONAL. + // An empty string is equal to an unspecified field value. + StartingToken string `protobuf:"bytes,2,opt,name=starting_token,json=startingToken,proto3" json:"starting_token,omitempty"` + // Identity information for the source volume. This field is OPTIONAL. + // It can be used to list snapshots by volume. + SourceVolumeId string `protobuf:"bytes,3,opt,name=source_volume_id,json=sourceVolumeId,proto3" json:"source_volume_id,omitempty"` + // Identity information for a specific snapshot. This field is + // OPTIONAL. It can be used to list only a specific snapshot. + // ListSnapshots will return with current snapshot information + // and will not block if the snapshot is being processed after + // it is cut. + SnapshotId string `protobuf:"bytes,4,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + // Secrets required by plugin to complete ListSnapshot request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + Secrets map[string]string `protobuf:"bytes,5,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListSnapshotsRequest) Reset() { *m = ListSnapshotsRequest{} } +func (m *ListSnapshotsRequest) String() string { return proto.CompactTextString(m) } +func (*ListSnapshotsRequest) ProtoMessage() {} +func (*ListSnapshotsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{39} +} + +func (m *ListSnapshotsRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListSnapshotsRequest.Unmarshal(m, b) +} +func (m *ListSnapshotsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListSnapshotsRequest.Marshal(b, m, deterministic) +} +func (m *ListSnapshotsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListSnapshotsRequest.Merge(m, src) +} +func (m *ListSnapshotsRequest) XXX_Size() int { + return xxx_messageInfo_ListSnapshotsRequest.Size(m) +} +func (m *ListSnapshotsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ListSnapshotsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ListSnapshotsRequest proto.InternalMessageInfo + +func (m *ListSnapshotsRequest) GetMaxEntries() int32 { + if m != nil { + return m.MaxEntries + } + return 0 +} + +func (m *ListSnapshotsRequest) GetStartingToken() string { + if m != nil { + return m.StartingToken + } + return "" +} + +func (m *ListSnapshotsRequest) GetSourceVolumeId() string { + if m != nil { + return m.SourceVolumeId + } + return "" +} + +func (m *ListSnapshotsRequest) GetSnapshotId() string { + if m != nil { + return m.SnapshotId + } + return "" +} + +func (m *ListSnapshotsRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +type ListSnapshotsResponse struct { + Entries []*ListSnapshotsResponse_Entry `protobuf:"bytes,1,rep,name=entries,proto3" json:"entries,omitempty"` + // This token allows you to get the next page of entries for + // `ListSnapshots` request. If the number of entries is larger than + // `max_entries`, use the `next_token` as a value for the + // `starting_token` field in the next `ListSnapshots` request. This + // field is OPTIONAL. + // An empty string is equal to an unspecified field value. + NextToken string `protobuf:"bytes,2,opt,name=next_token,json=nextToken,proto3" json:"next_token,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListSnapshotsResponse) Reset() { *m = ListSnapshotsResponse{} } +func (m *ListSnapshotsResponse) String() string { return proto.CompactTextString(m) } +func (*ListSnapshotsResponse) ProtoMessage() {} +func (*ListSnapshotsResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{40} +} + +func (m *ListSnapshotsResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListSnapshotsResponse.Unmarshal(m, b) +} +func (m *ListSnapshotsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListSnapshotsResponse.Marshal(b, m, deterministic) +} +func (m *ListSnapshotsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListSnapshotsResponse.Merge(m, src) +} +func (m *ListSnapshotsResponse) XXX_Size() int { + return xxx_messageInfo_ListSnapshotsResponse.Size(m) +} +func (m *ListSnapshotsResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ListSnapshotsResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ListSnapshotsResponse proto.InternalMessageInfo + +func (m *ListSnapshotsResponse) GetEntries() []*ListSnapshotsResponse_Entry { + if m != nil { + return m.Entries + } + return nil +} + +func (m *ListSnapshotsResponse) GetNextToken() string { + if m != nil { + return m.NextToken + } + return "" +} + +type ListSnapshotsResponse_Entry struct { + Snapshot *Snapshot `protobuf:"bytes,1,opt,name=snapshot,proto3" json:"snapshot,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ListSnapshotsResponse_Entry) Reset() { *m = ListSnapshotsResponse_Entry{} } +func (m *ListSnapshotsResponse_Entry) String() string { return proto.CompactTextString(m) } +func (*ListSnapshotsResponse_Entry) ProtoMessage() {} +func (*ListSnapshotsResponse_Entry) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{40, 0} +} + +func (m *ListSnapshotsResponse_Entry) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListSnapshotsResponse_Entry.Unmarshal(m, b) +} +func (m *ListSnapshotsResponse_Entry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListSnapshotsResponse_Entry.Marshal(b, m, deterministic) +} +func (m *ListSnapshotsResponse_Entry) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListSnapshotsResponse_Entry.Merge(m, src) +} +func (m *ListSnapshotsResponse_Entry) XXX_Size() int { + return xxx_messageInfo_ListSnapshotsResponse_Entry.Size(m) +} +func (m *ListSnapshotsResponse_Entry) XXX_DiscardUnknown() { + xxx_messageInfo_ListSnapshotsResponse_Entry.DiscardUnknown(m) +} + +var xxx_messageInfo_ListSnapshotsResponse_Entry proto.InternalMessageInfo + +func (m *ListSnapshotsResponse_Entry) GetSnapshot() *Snapshot { + if m != nil { + return m.Snapshot + } + return nil +} + +type ControllerExpandVolumeRequest struct { + // The ID of the volume to expand. This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // This allows CO to specify the capacity requirements of the volume + // after expansion. This field is REQUIRED. + CapacityRange *CapacityRange `protobuf:"bytes,2,opt,name=capacity_range,json=capacityRange,proto3" json:"capacity_range,omitempty"` + // Secrets required by the plugin for expanding the volume. + // This field is OPTIONAL. + Secrets map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Volume capability describing how the CO intends to use this volume. + // This allows SP to determine if volume is being used as a block + // device or mounted file system. For example - if volume is + // being used as a block device - the SP MAY set + // node_expansion_required to false in ControllerExpandVolumeResponse + // to skip invocation of NodeExpandVolume on the node by the CO. + // This is an OPTIONAL field. + VolumeCapability *VolumeCapability `protobuf:"bytes,4,opt,name=volume_capability,json=volumeCapability,proto3" json:"volume_capability,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerExpandVolumeRequest) Reset() { *m = ControllerExpandVolumeRequest{} } +func (m *ControllerExpandVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*ControllerExpandVolumeRequest) ProtoMessage() {} +func (*ControllerExpandVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{41} +} + +func (m *ControllerExpandVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerExpandVolumeRequest.Unmarshal(m, b) +} +func (m *ControllerExpandVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerExpandVolumeRequest.Marshal(b, m, deterministic) +} +func (m *ControllerExpandVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerExpandVolumeRequest.Merge(m, src) +} +func (m *ControllerExpandVolumeRequest) XXX_Size() int { + return xxx_messageInfo_ControllerExpandVolumeRequest.Size(m) +} +func (m *ControllerExpandVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerExpandVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerExpandVolumeRequest proto.InternalMessageInfo + +func (m *ControllerExpandVolumeRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *ControllerExpandVolumeRequest) GetCapacityRange() *CapacityRange { + if m != nil { + return m.CapacityRange + } + return nil +} + +func (m *ControllerExpandVolumeRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +func (m *ControllerExpandVolumeRequest) GetVolumeCapability() *VolumeCapability { + if m != nil { + return m.VolumeCapability + } + return nil +} + +type ControllerExpandVolumeResponse struct { + // Capacity of volume after expansion. This field is REQUIRED. + CapacityBytes int64 `protobuf:"varint,1,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` + // Whether node expansion is required for the volume. When true + // the CO MUST make NodeExpandVolume RPC call on the node. This field + // is REQUIRED. + NodeExpansionRequired bool `protobuf:"varint,2,opt,name=node_expansion_required,json=nodeExpansionRequired,proto3" json:"node_expansion_required,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ControllerExpandVolumeResponse) Reset() { *m = ControllerExpandVolumeResponse{} } +func (m *ControllerExpandVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*ControllerExpandVolumeResponse) ProtoMessage() {} +func (*ControllerExpandVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{42} +} + +func (m *ControllerExpandVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ControllerExpandVolumeResponse.Unmarshal(m, b) +} +func (m *ControllerExpandVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ControllerExpandVolumeResponse.Marshal(b, m, deterministic) +} +func (m *ControllerExpandVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ControllerExpandVolumeResponse.Merge(m, src) +} +func (m *ControllerExpandVolumeResponse) XXX_Size() int { + return xxx_messageInfo_ControllerExpandVolumeResponse.Size(m) +} +func (m *ControllerExpandVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ControllerExpandVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ControllerExpandVolumeResponse proto.InternalMessageInfo + +func (m *ControllerExpandVolumeResponse) GetCapacityBytes() int64 { + if m != nil { + return m.CapacityBytes + } + return 0 +} + +func (m *ControllerExpandVolumeResponse) GetNodeExpansionRequired() bool { + if m != nil { + return m.NodeExpansionRequired + } + return false +} + +type NodeStageVolumeRequest struct { + // The ID of the volume to publish. This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // The CO SHALL set this field to the value returned by + // `ControllerPublishVolume` if the corresponding Controller Plugin + // has `PUBLISH_UNPUBLISH_VOLUME` controller capability, and SHALL be + // left unset if the corresponding Controller Plugin does not have + // this capability. This is an OPTIONAL field. + PublishContext map[string]string `protobuf:"bytes,2,rep,name=publish_context,json=publishContext,proto3" json:"publish_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // The path to which the volume MAY be staged. It MUST be an + // absolute path in the root filesystem of the process serving this + // request, and MUST be a directory. The CO SHALL ensure that there + // is only one `staging_target_path` per volume. The CO SHALL ensure + // that the path is directory and that the process serving the + // request has `read` and `write` permission to that directory. The + // CO SHALL be responsible for creating the directory if it does not + // exist. + // This is a REQUIRED field. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + StagingTargetPath string `protobuf:"bytes,3,opt,name=staging_target_path,json=stagingTargetPath,proto3" json:"staging_target_path,omitempty"` + // Volume capability describing how the CO intends to use this volume. + // SP MUST ensure the CO can use the staged volume as described. + // Otherwise SP MUST return the appropriate gRPC error code. + // This is a REQUIRED field. + VolumeCapability *VolumeCapability `protobuf:"bytes,4,opt,name=volume_capability,json=volumeCapability,proto3" json:"volume_capability,omitempty"` + // Secrets required by plugin to complete node stage volume request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + Secrets map[string]string `protobuf:"bytes,5,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Volume context as returned by SP in + // CreateVolumeResponse.Volume.volume_context. + // This field is OPTIONAL and MUST match the volume_context of the + // volume identified by `volume_id`. + VolumeContext map[string]string `protobuf:"bytes,6,rep,name=volume_context,json=volumeContext,proto3" json:"volume_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeStageVolumeRequest) Reset() { *m = NodeStageVolumeRequest{} } +func (m *NodeStageVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*NodeStageVolumeRequest) ProtoMessage() {} +func (*NodeStageVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{43} +} + +func (m *NodeStageVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeStageVolumeRequest.Unmarshal(m, b) +} +func (m *NodeStageVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeStageVolumeRequest.Marshal(b, m, deterministic) +} +func (m *NodeStageVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeStageVolumeRequest.Merge(m, src) +} +func (m *NodeStageVolumeRequest) XXX_Size() int { + return xxx_messageInfo_NodeStageVolumeRequest.Size(m) +} +func (m *NodeStageVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_NodeStageVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeStageVolumeRequest proto.InternalMessageInfo + +func (m *NodeStageVolumeRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *NodeStageVolumeRequest) GetPublishContext() map[string]string { + if m != nil { + return m.PublishContext + } + return nil +} + +func (m *NodeStageVolumeRequest) GetStagingTargetPath() string { + if m != nil { + return m.StagingTargetPath + } + return "" +} + +func (m *NodeStageVolumeRequest) GetVolumeCapability() *VolumeCapability { + if m != nil { + return m.VolumeCapability + } + return nil +} + +func (m *NodeStageVolumeRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +func (m *NodeStageVolumeRequest) GetVolumeContext() map[string]string { + if m != nil { + return m.VolumeContext + } + return nil +} + +type NodeStageVolumeResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeStageVolumeResponse) Reset() { *m = NodeStageVolumeResponse{} } +func (m *NodeStageVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*NodeStageVolumeResponse) ProtoMessage() {} +func (*NodeStageVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{44} +} + +func (m *NodeStageVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeStageVolumeResponse.Unmarshal(m, b) +} +func (m *NodeStageVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeStageVolumeResponse.Marshal(b, m, deterministic) +} +func (m *NodeStageVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeStageVolumeResponse.Merge(m, src) +} +func (m *NodeStageVolumeResponse) XXX_Size() int { + return xxx_messageInfo_NodeStageVolumeResponse.Size(m) +} +func (m *NodeStageVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_NodeStageVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeStageVolumeResponse proto.InternalMessageInfo + +type NodeUnstageVolumeRequest struct { + // The ID of the volume. This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // The path at which the volume was staged. It MUST be an absolute + // path in the root filesystem of the process serving this request. + // This is a REQUIRED field. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + StagingTargetPath string `protobuf:"bytes,2,opt,name=staging_target_path,json=stagingTargetPath,proto3" json:"staging_target_path,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeUnstageVolumeRequest) Reset() { *m = NodeUnstageVolumeRequest{} } +func (m *NodeUnstageVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*NodeUnstageVolumeRequest) ProtoMessage() {} +func (*NodeUnstageVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{45} +} + +func (m *NodeUnstageVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeUnstageVolumeRequest.Unmarshal(m, b) +} +func (m *NodeUnstageVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeUnstageVolumeRequest.Marshal(b, m, deterministic) +} +func (m *NodeUnstageVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeUnstageVolumeRequest.Merge(m, src) +} +func (m *NodeUnstageVolumeRequest) XXX_Size() int { + return xxx_messageInfo_NodeUnstageVolumeRequest.Size(m) +} +func (m *NodeUnstageVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_NodeUnstageVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeUnstageVolumeRequest proto.InternalMessageInfo + +func (m *NodeUnstageVolumeRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *NodeUnstageVolumeRequest) GetStagingTargetPath() string { + if m != nil { + return m.StagingTargetPath + } + return "" +} + +type NodeUnstageVolumeResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeUnstageVolumeResponse) Reset() { *m = NodeUnstageVolumeResponse{} } +func (m *NodeUnstageVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*NodeUnstageVolumeResponse) ProtoMessage() {} +func (*NodeUnstageVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{46} +} + +func (m *NodeUnstageVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeUnstageVolumeResponse.Unmarshal(m, b) +} +func (m *NodeUnstageVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeUnstageVolumeResponse.Marshal(b, m, deterministic) +} +func (m *NodeUnstageVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeUnstageVolumeResponse.Merge(m, src) +} +func (m *NodeUnstageVolumeResponse) XXX_Size() int { + return xxx_messageInfo_NodeUnstageVolumeResponse.Size(m) +} +func (m *NodeUnstageVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_NodeUnstageVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeUnstageVolumeResponse proto.InternalMessageInfo + +type NodePublishVolumeRequest struct { + // The ID of the volume to publish. This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // The CO SHALL set this field to the value returned by + // `ControllerPublishVolume` if the corresponding Controller Plugin + // has `PUBLISH_UNPUBLISH_VOLUME` controller capability, and SHALL be + // left unset if the corresponding Controller Plugin does not have + // this capability. This is an OPTIONAL field. + PublishContext map[string]string `protobuf:"bytes,2,rep,name=publish_context,json=publishContext,proto3" json:"publish_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // The path to which the volume was staged by `NodeStageVolume`. + // It MUST be an absolute path in the root filesystem of the process + // serving this request. + // It MUST be set if the Node Plugin implements the + // `STAGE_UNSTAGE_VOLUME` node capability. + // This is an OPTIONAL field. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + StagingTargetPath string `protobuf:"bytes,3,opt,name=staging_target_path,json=stagingTargetPath,proto3" json:"staging_target_path,omitempty"` + // The path to which the volume will be published. It MUST be an + // absolute path in the root filesystem of the process serving this + // request. The CO SHALL ensure uniqueness of target_path per volume. + // The CO SHALL ensure that the parent directory of this path exists + // and that the process serving the request has `read` and `write` + // permissions to that parent directory. + // For volumes with an access type of block, the SP SHALL place the + // block device at target_path. + // For volumes with an access type of mount, the SP SHALL place the + // mounted directory at target_path. + // Creation of target_path is the responsibility of the SP. + // This is a REQUIRED field. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + TargetPath string `protobuf:"bytes,4,opt,name=target_path,json=targetPath,proto3" json:"target_path,omitempty"` + // Volume capability describing how the CO intends to use this volume. + // SP MUST ensure the CO can use the published volume as described. + // Otherwise SP MUST return the appropriate gRPC error code. + // This is a REQUIRED field. + VolumeCapability *VolumeCapability `protobuf:"bytes,5,opt,name=volume_capability,json=volumeCapability,proto3" json:"volume_capability,omitempty"` + // Indicates SP MUST publish the volume in readonly mode. + // This field is REQUIRED. + Readonly bool `protobuf:"varint,6,opt,name=readonly,proto3" json:"readonly,omitempty"` + // Secrets required by plugin to complete node publish volume request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + Secrets map[string]string `protobuf:"bytes,7,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Volume context as returned by SP in + // CreateVolumeResponse.Volume.volume_context. + // This field is OPTIONAL and MUST match the volume_context of the + // volume identified by `volume_id`. + VolumeContext map[string]string `protobuf:"bytes,8,rep,name=volume_context,json=volumeContext,proto3" json:"volume_context,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodePublishVolumeRequest) Reset() { *m = NodePublishVolumeRequest{} } +func (m *NodePublishVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*NodePublishVolumeRequest) ProtoMessage() {} +func (*NodePublishVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{47} +} + +func (m *NodePublishVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodePublishVolumeRequest.Unmarshal(m, b) +} +func (m *NodePublishVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodePublishVolumeRequest.Marshal(b, m, deterministic) +} +func (m *NodePublishVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodePublishVolumeRequest.Merge(m, src) +} +func (m *NodePublishVolumeRequest) XXX_Size() int { + return xxx_messageInfo_NodePublishVolumeRequest.Size(m) +} +func (m *NodePublishVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_NodePublishVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_NodePublishVolumeRequest proto.InternalMessageInfo + +func (m *NodePublishVolumeRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *NodePublishVolumeRequest) GetPublishContext() map[string]string { + if m != nil { + return m.PublishContext + } + return nil +} + +func (m *NodePublishVolumeRequest) GetStagingTargetPath() string { + if m != nil { + return m.StagingTargetPath + } + return "" +} + +func (m *NodePublishVolumeRequest) GetTargetPath() string { + if m != nil { + return m.TargetPath + } + return "" +} + +func (m *NodePublishVolumeRequest) GetVolumeCapability() *VolumeCapability { + if m != nil { + return m.VolumeCapability + } + return nil +} + +func (m *NodePublishVolumeRequest) GetReadonly() bool { + if m != nil { + return m.Readonly + } + return false +} + +func (m *NodePublishVolumeRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +func (m *NodePublishVolumeRequest) GetVolumeContext() map[string]string { + if m != nil { + return m.VolumeContext + } + return nil +} + +type NodePublishVolumeResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodePublishVolumeResponse) Reset() { *m = NodePublishVolumeResponse{} } +func (m *NodePublishVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*NodePublishVolumeResponse) ProtoMessage() {} +func (*NodePublishVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{48} +} + +func (m *NodePublishVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodePublishVolumeResponse.Unmarshal(m, b) +} +func (m *NodePublishVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodePublishVolumeResponse.Marshal(b, m, deterministic) +} +func (m *NodePublishVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodePublishVolumeResponse.Merge(m, src) +} +func (m *NodePublishVolumeResponse) XXX_Size() int { + return xxx_messageInfo_NodePublishVolumeResponse.Size(m) +} +func (m *NodePublishVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_NodePublishVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_NodePublishVolumeResponse proto.InternalMessageInfo + +type NodeUnpublishVolumeRequest struct { + // The ID of the volume. This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // The path at which the volume was published. It MUST be an absolute + // path in the root filesystem of the process serving this request. + // The SP MUST delete the file or directory it created at this path. + // This is a REQUIRED field. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + TargetPath string `protobuf:"bytes,2,opt,name=target_path,json=targetPath,proto3" json:"target_path,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeUnpublishVolumeRequest) Reset() { *m = NodeUnpublishVolumeRequest{} } +func (m *NodeUnpublishVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*NodeUnpublishVolumeRequest) ProtoMessage() {} +func (*NodeUnpublishVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{49} +} + +func (m *NodeUnpublishVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeUnpublishVolumeRequest.Unmarshal(m, b) +} +func (m *NodeUnpublishVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeUnpublishVolumeRequest.Marshal(b, m, deterministic) +} +func (m *NodeUnpublishVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeUnpublishVolumeRequest.Merge(m, src) +} +func (m *NodeUnpublishVolumeRequest) XXX_Size() int { + return xxx_messageInfo_NodeUnpublishVolumeRequest.Size(m) +} +func (m *NodeUnpublishVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_NodeUnpublishVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeUnpublishVolumeRequest proto.InternalMessageInfo + +func (m *NodeUnpublishVolumeRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *NodeUnpublishVolumeRequest) GetTargetPath() string { + if m != nil { + return m.TargetPath + } + return "" +} + +type NodeUnpublishVolumeResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeUnpublishVolumeResponse) Reset() { *m = NodeUnpublishVolumeResponse{} } +func (m *NodeUnpublishVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*NodeUnpublishVolumeResponse) ProtoMessage() {} +func (*NodeUnpublishVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{50} +} + +func (m *NodeUnpublishVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeUnpublishVolumeResponse.Unmarshal(m, b) +} +func (m *NodeUnpublishVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeUnpublishVolumeResponse.Marshal(b, m, deterministic) +} +func (m *NodeUnpublishVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeUnpublishVolumeResponse.Merge(m, src) +} +func (m *NodeUnpublishVolumeResponse) XXX_Size() int { + return xxx_messageInfo_NodeUnpublishVolumeResponse.Size(m) +} +func (m *NodeUnpublishVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_NodeUnpublishVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeUnpublishVolumeResponse proto.InternalMessageInfo + +type NodeGetVolumeStatsRequest struct { + // The ID of the volume. This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // It can be any valid path where volume was previously + // staged or published. + // It MUST be an absolute path in the root filesystem of + // the process serving this request. + // This is a REQUIRED field. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + VolumePath string `protobuf:"bytes,2,opt,name=volume_path,json=volumePath,proto3" json:"volume_path,omitempty"` + // The path where the volume is staged, if the plugin has the + // STAGE_UNSTAGE_VOLUME capability, otherwise empty. + // If not empty, it MUST be an absolute path in the root + // filesystem of the process serving this request. + // This field is OPTIONAL. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + StagingTargetPath string `protobuf:"bytes,3,opt,name=staging_target_path,json=stagingTargetPath,proto3" json:"staging_target_path,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeGetVolumeStatsRequest) Reset() { *m = NodeGetVolumeStatsRequest{} } +func (m *NodeGetVolumeStatsRequest) String() string { return proto.CompactTextString(m) } +func (*NodeGetVolumeStatsRequest) ProtoMessage() {} +func (*NodeGetVolumeStatsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{51} +} + +func (m *NodeGetVolumeStatsRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeGetVolumeStatsRequest.Unmarshal(m, b) +} +func (m *NodeGetVolumeStatsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeGetVolumeStatsRequest.Marshal(b, m, deterministic) +} +func (m *NodeGetVolumeStatsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetVolumeStatsRequest.Merge(m, src) +} +func (m *NodeGetVolumeStatsRequest) XXX_Size() int { + return xxx_messageInfo_NodeGetVolumeStatsRequest.Size(m) +} +func (m *NodeGetVolumeStatsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_NodeGetVolumeStatsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeGetVolumeStatsRequest proto.InternalMessageInfo + +func (m *NodeGetVolumeStatsRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *NodeGetVolumeStatsRequest) GetVolumePath() string { + if m != nil { + return m.VolumePath + } + return "" +} + +func (m *NodeGetVolumeStatsRequest) GetStagingTargetPath() string { + if m != nil { + return m.StagingTargetPath + } + return "" +} + +type NodeGetVolumeStatsResponse struct { + // This field is OPTIONAL. + Usage []*VolumeUsage `protobuf:"bytes,1,rep,name=usage,proto3" json:"usage,omitempty"` + // Information about the current condition of the volume. + // This field is OPTIONAL. + // This field MUST be specified if the VOLUME_CONDITION node + // capability is supported. + VolumeCondition *VolumeCondition `protobuf:"bytes,2,opt,name=volume_condition,json=volumeCondition,proto3" json:"volume_condition,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeGetVolumeStatsResponse) Reset() { *m = NodeGetVolumeStatsResponse{} } +func (m *NodeGetVolumeStatsResponse) String() string { return proto.CompactTextString(m) } +func (*NodeGetVolumeStatsResponse) ProtoMessage() {} +func (*NodeGetVolumeStatsResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{52} +} + +func (m *NodeGetVolumeStatsResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeGetVolumeStatsResponse.Unmarshal(m, b) +} +func (m *NodeGetVolumeStatsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeGetVolumeStatsResponse.Marshal(b, m, deterministic) +} +func (m *NodeGetVolumeStatsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetVolumeStatsResponse.Merge(m, src) +} +func (m *NodeGetVolumeStatsResponse) XXX_Size() int { + return xxx_messageInfo_NodeGetVolumeStatsResponse.Size(m) +} +func (m *NodeGetVolumeStatsResponse) XXX_DiscardUnknown() { + xxx_messageInfo_NodeGetVolumeStatsResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeGetVolumeStatsResponse proto.InternalMessageInfo + +func (m *NodeGetVolumeStatsResponse) GetUsage() []*VolumeUsage { + if m != nil { + return m.Usage + } + return nil +} + +func (m *NodeGetVolumeStatsResponse) GetVolumeCondition() *VolumeCondition { + if m != nil { + return m.VolumeCondition + } + return nil +} + +type VolumeUsage struct { + // The available capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + Available int64 `protobuf:"varint,1,opt,name=available,proto3" json:"available,omitempty"` + // The total capacity in specified Unit. This field is REQUIRED. + // The value of this field MUST NOT be negative. + Total int64 `protobuf:"varint,2,opt,name=total,proto3" json:"total,omitempty"` + // The used capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + Used int64 `protobuf:"varint,3,opt,name=used,proto3" json:"used,omitempty"` + // Units by which values are measured. This field is REQUIRED. + Unit VolumeUsage_Unit `protobuf:"varint,4,opt,name=unit,proto3,enum=csi.v1.VolumeUsage_Unit" json:"unit,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeUsage) Reset() { *m = VolumeUsage{} } +func (m *VolumeUsage) String() string { return proto.CompactTextString(m) } +func (*VolumeUsage) ProtoMessage() {} +func (*VolumeUsage) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{53} +} + +func (m *VolumeUsage) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VolumeUsage.Unmarshal(m, b) +} +func (m *VolumeUsage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VolumeUsage.Marshal(b, m, deterministic) +} +func (m *VolumeUsage) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeUsage.Merge(m, src) +} +func (m *VolumeUsage) XXX_Size() int { + return xxx_messageInfo_VolumeUsage.Size(m) +} +func (m *VolumeUsage) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeUsage.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeUsage proto.InternalMessageInfo + +func (m *VolumeUsage) GetAvailable() int64 { + if m != nil { + return m.Available + } + return 0 +} + +func (m *VolumeUsage) GetTotal() int64 { + if m != nil { + return m.Total + } + return 0 +} + +func (m *VolumeUsage) GetUsed() int64 { + if m != nil { + return m.Used + } + return 0 +} + +func (m *VolumeUsage) GetUnit() VolumeUsage_Unit { + if m != nil { + return m.Unit + } + return VolumeUsage_UNKNOWN +} + +// VolumeCondition represents the current condition of a volume. +type VolumeCondition struct { + // Normal volumes are available for use and operating optimally. + // An abnormal volume does not meet these criteria. + // This field is REQUIRED. + Abnormal bool `protobuf:"varint,1,opt,name=abnormal,proto3" json:"abnormal,omitempty"` + // The message describing the condition of the volume. + // This field is REQUIRED. + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeCondition) Reset() { *m = VolumeCondition{} } +func (m *VolumeCondition) String() string { return proto.CompactTextString(m) } +func (*VolumeCondition) ProtoMessage() {} +func (*VolumeCondition) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{54} +} + +func (m *VolumeCondition) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VolumeCondition.Unmarshal(m, b) +} +func (m *VolumeCondition) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VolumeCondition.Marshal(b, m, deterministic) +} +func (m *VolumeCondition) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeCondition.Merge(m, src) +} +func (m *VolumeCondition) XXX_Size() int { + return xxx_messageInfo_VolumeCondition.Size(m) +} +func (m *VolumeCondition) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeCondition.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeCondition proto.InternalMessageInfo + +func (m *VolumeCondition) GetAbnormal() bool { + if m != nil { + return m.Abnormal + } + return false +} + +func (m *VolumeCondition) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +type NodeGetCapabilitiesRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeGetCapabilitiesRequest) Reset() { *m = NodeGetCapabilitiesRequest{} } +func (m *NodeGetCapabilitiesRequest) String() string { return proto.CompactTextString(m) } +func (*NodeGetCapabilitiesRequest) ProtoMessage() {} +func (*NodeGetCapabilitiesRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{55} +} + +func (m *NodeGetCapabilitiesRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeGetCapabilitiesRequest.Unmarshal(m, b) +} +func (m *NodeGetCapabilitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeGetCapabilitiesRequest.Marshal(b, m, deterministic) +} +func (m *NodeGetCapabilitiesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetCapabilitiesRequest.Merge(m, src) +} +func (m *NodeGetCapabilitiesRequest) XXX_Size() int { + return xxx_messageInfo_NodeGetCapabilitiesRequest.Size(m) +} +func (m *NodeGetCapabilitiesRequest) XXX_DiscardUnknown() { + xxx_messageInfo_NodeGetCapabilitiesRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeGetCapabilitiesRequest proto.InternalMessageInfo + +type NodeGetCapabilitiesResponse struct { + // All the capabilities that the node service supports. This field + // is OPTIONAL. + Capabilities []*NodeServiceCapability `protobuf:"bytes,1,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeGetCapabilitiesResponse) Reset() { *m = NodeGetCapabilitiesResponse{} } +func (m *NodeGetCapabilitiesResponse) String() string { return proto.CompactTextString(m) } +func (*NodeGetCapabilitiesResponse) ProtoMessage() {} +func (*NodeGetCapabilitiesResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{56} +} + +func (m *NodeGetCapabilitiesResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeGetCapabilitiesResponse.Unmarshal(m, b) +} +func (m *NodeGetCapabilitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeGetCapabilitiesResponse.Marshal(b, m, deterministic) +} +func (m *NodeGetCapabilitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetCapabilitiesResponse.Merge(m, src) +} +func (m *NodeGetCapabilitiesResponse) XXX_Size() int { + return xxx_messageInfo_NodeGetCapabilitiesResponse.Size(m) +} +func (m *NodeGetCapabilitiesResponse) XXX_DiscardUnknown() { + xxx_messageInfo_NodeGetCapabilitiesResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeGetCapabilitiesResponse proto.InternalMessageInfo + +func (m *NodeGetCapabilitiesResponse) GetCapabilities() []*NodeServiceCapability { + if m != nil { + return m.Capabilities + } + return nil +} + +// Specifies a capability of the node service. +type NodeServiceCapability struct { + // Types that are valid to be assigned to Type: + // + // *NodeServiceCapability_Rpc + Type isNodeServiceCapability_Type `protobuf_oneof:"type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeServiceCapability) Reset() { *m = NodeServiceCapability{} } +func (m *NodeServiceCapability) String() string { return proto.CompactTextString(m) } +func (*NodeServiceCapability) ProtoMessage() {} +func (*NodeServiceCapability) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{57} +} + +func (m *NodeServiceCapability) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeServiceCapability.Unmarshal(m, b) +} +func (m *NodeServiceCapability) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeServiceCapability.Marshal(b, m, deterministic) +} +func (m *NodeServiceCapability) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeServiceCapability.Merge(m, src) +} +func (m *NodeServiceCapability) XXX_Size() int { + return xxx_messageInfo_NodeServiceCapability.Size(m) +} +func (m *NodeServiceCapability) XXX_DiscardUnknown() { + xxx_messageInfo_NodeServiceCapability.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeServiceCapability proto.InternalMessageInfo + +type isNodeServiceCapability_Type interface { + isNodeServiceCapability_Type() +} + +type NodeServiceCapability_Rpc struct { + Rpc *NodeServiceCapability_RPC `protobuf:"bytes,1,opt,name=rpc,proto3,oneof"` +} + +func (*NodeServiceCapability_Rpc) isNodeServiceCapability_Type() {} + +func (m *NodeServiceCapability) GetType() isNodeServiceCapability_Type { + if m != nil { + return m.Type + } + return nil +} + +func (m *NodeServiceCapability) GetRpc() *NodeServiceCapability_RPC { + if x, ok := m.GetType().(*NodeServiceCapability_Rpc); ok { + return x.Rpc + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*NodeServiceCapability) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*NodeServiceCapability_Rpc)(nil), + } +} + +type NodeServiceCapability_RPC struct { + Type NodeServiceCapability_RPC_Type `protobuf:"varint,1,opt,name=type,proto3,enum=csi.v1.NodeServiceCapability_RPC_Type" json:"type,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeServiceCapability_RPC) Reset() { *m = NodeServiceCapability_RPC{} } +func (m *NodeServiceCapability_RPC) String() string { return proto.CompactTextString(m) } +func (*NodeServiceCapability_RPC) ProtoMessage() {} +func (*NodeServiceCapability_RPC) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{57, 0} +} + +func (m *NodeServiceCapability_RPC) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeServiceCapability_RPC.Unmarshal(m, b) +} +func (m *NodeServiceCapability_RPC) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeServiceCapability_RPC.Marshal(b, m, deterministic) +} +func (m *NodeServiceCapability_RPC) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeServiceCapability_RPC.Merge(m, src) +} +func (m *NodeServiceCapability_RPC) XXX_Size() int { + return xxx_messageInfo_NodeServiceCapability_RPC.Size(m) +} +func (m *NodeServiceCapability_RPC) XXX_DiscardUnknown() { + xxx_messageInfo_NodeServiceCapability_RPC.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeServiceCapability_RPC proto.InternalMessageInfo + +func (m *NodeServiceCapability_RPC) GetType() NodeServiceCapability_RPC_Type { + if m != nil { + return m.Type + } + return NodeServiceCapability_RPC_UNKNOWN +} + +type NodeGetInfoRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeGetInfoRequest) Reset() { *m = NodeGetInfoRequest{} } +func (m *NodeGetInfoRequest) String() string { return proto.CompactTextString(m) } +func (*NodeGetInfoRequest) ProtoMessage() {} +func (*NodeGetInfoRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{58} +} + +func (m *NodeGetInfoRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeGetInfoRequest.Unmarshal(m, b) +} +func (m *NodeGetInfoRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeGetInfoRequest.Marshal(b, m, deterministic) +} +func (m *NodeGetInfoRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetInfoRequest.Merge(m, src) +} +func (m *NodeGetInfoRequest) XXX_Size() int { + return xxx_messageInfo_NodeGetInfoRequest.Size(m) +} +func (m *NodeGetInfoRequest) XXX_DiscardUnknown() { + xxx_messageInfo_NodeGetInfoRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeGetInfoRequest proto.InternalMessageInfo + +type NodeGetInfoResponse struct { + // The identifier of the node as understood by the SP. + // This field is REQUIRED. + // This field MUST contain enough information to uniquely identify + // this specific node vs all other nodes supported by this plugin. + // This field SHALL be used by the CO in subsequent calls, including + // `ControllerPublishVolume`, to refer to this node. + // The SP is NOT responsible for global uniqueness of node_id across + // multiple SPs. + // This field overrides the general CSI size limit. + // The size of this field SHALL NOT exceed 256 bytes. The general + // CSI size limit, 128 byte, is RECOMMENDED for best backwards + // compatibility. + NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` + // Maximum number of volumes that controller can publish to the node. + // If value is not set or zero CO SHALL decide how many volumes of + // this type can be published by the controller to the node. The + // plugin MUST NOT set negative values here. + // This field is OPTIONAL. + MaxVolumesPerNode int64 `protobuf:"varint,2,opt,name=max_volumes_per_node,json=maxVolumesPerNode,proto3" json:"max_volumes_per_node,omitempty"` + // Specifies where (regions, zones, racks, etc.) the node is + // accessible from. + // A plugin that returns this field MUST also set the + // VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability. + // COs MAY use this information along with the topology information + // returned in CreateVolumeResponse to ensure that a given volume is + // accessible from a given node when scheduling workloads. + // This field is OPTIONAL. If it is not specified, the CO MAY assume + // the node is not subject to any topological constraint, and MAY + // schedule workloads that reference any volume V, such that there are + // no topological constraints declared for V. + // + // Example 1: + // + // accessible_topology = + // {"region": "R1", "zone": "Z2"} + // + // Indicates the node exists within the "region" "R1" and the "zone" + // "Z2". + AccessibleTopology *Topology `protobuf:"bytes,3,opt,name=accessible_topology,json=accessibleTopology,proto3" json:"accessible_topology,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeGetInfoResponse) Reset() { *m = NodeGetInfoResponse{} } +func (m *NodeGetInfoResponse) String() string { return proto.CompactTextString(m) } +func (*NodeGetInfoResponse) ProtoMessage() {} +func (*NodeGetInfoResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{59} +} + +func (m *NodeGetInfoResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeGetInfoResponse.Unmarshal(m, b) +} +func (m *NodeGetInfoResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeGetInfoResponse.Marshal(b, m, deterministic) +} +func (m *NodeGetInfoResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeGetInfoResponse.Merge(m, src) +} +func (m *NodeGetInfoResponse) XXX_Size() int { + return xxx_messageInfo_NodeGetInfoResponse.Size(m) +} +func (m *NodeGetInfoResponse) XXX_DiscardUnknown() { + xxx_messageInfo_NodeGetInfoResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeGetInfoResponse proto.InternalMessageInfo + +func (m *NodeGetInfoResponse) GetNodeId() string { + if m != nil { + return m.NodeId + } + return "" +} + +func (m *NodeGetInfoResponse) GetMaxVolumesPerNode() int64 { + if m != nil { + return m.MaxVolumesPerNode + } + return 0 +} + +func (m *NodeGetInfoResponse) GetAccessibleTopology() *Topology { + if m != nil { + return m.AccessibleTopology + } + return nil +} + +type NodeExpandVolumeRequest struct { + // The ID of the volume. This field is REQUIRED. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // The path on which volume is available. This field is REQUIRED. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + VolumePath string `protobuf:"bytes,2,opt,name=volume_path,json=volumePath,proto3" json:"volume_path,omitempty"` + // This allows CO to specify the capacity requirements of the volume + // after expansion. If capacity_range is omitted then a plugin MAY + // inspect the file system of the volume to determine the maximum + // capacity to which the volume can be expanded. In such cases a + // plugin MAY expand the volume to its maximum capacity. + // This field is OPTIONAL. + CapacityRange *CapacityRange `protobuf:"bytes,3,opt,name=capacity_range,json=capacityRange,proto3" json:"capacity_range,omitempty"` + // The path where the volume is staged, if the plugin has the + // STAGE_UNSTAGE_VOLUME capability, otherwise empty. + // If not empty, it MUST be an absolute path in the root + // filesystem of the process serving this request. + // This field is OPTIONAL. + // This field overrides the general CSI size limit. + // SP SHOULD support the maximum path length allowed by the operating + // system/filesystem, but, at a minimum, SP MUST accept a max path + // length of at least 128 bytes. + StagingTargetPath string `protobuf:"bytes,4,opt,name=staging_target_path,json=stagingTargetPath,proto3" json:"staging_target_path,omitempty"` + // Volume capability describing how the CO intends to use this volume. + // This allows SP to determine if volume is being used as a block + // device or mounted file system. For example - if volume is being + // used as a block device the SP MAY choose to skip expanding the + // filesystem in NodeExpandVolume implementation but still perform + // rest of the housekeeping needed for expanding the volume. If + // volume_capability is omitted the SP MAY determine + // access_type from given volume_path for the volume and perform + // node expansion. This is an OPTIONAL field. + VolumeCapability *VolumeCapability `protobuf:"bytes,5,opt,name=volume_capability,json=volumeCapability,proto3" json:"volume_capability,omitempty"` + // Secrets required by plugin to complete node expand volume request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + Secrets map[string]string `protobuf:"bytes,6,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeExpandVolumeRequest) Reset() { *m = NodeExpandVolumeRequest{} } +func (m *NodeExpandVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*NodeExpandVolumeRequest) ProtoMessage() {} +func (*NodeExpandVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{60} +} + +func (m *NodeExpandVolumeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeExpandVolumeRequest.Unmarshal(m, b) +} +func (m *NodeExpandVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeExpandVolumeRequest.Marshal(b, m, deterministic) +} +func (m *NodeExpandVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeExpandVolumeRequest.Merge(m, src) +} +func (m *NodeExpandVolumeRequest) XXX_Size() int { + return xxx_messageInfo_NodeExpandVolumeRequest.Size(m) +} +func (m *NodeExpandVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_NodeExpandVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeExpandVolumeRequest proto.InternalMessageInfo + +func (m *NodeExpandVolumeRequest) GetVolumeId() string { + if m != nil { + return m.VolumeId + } + return "" +} + +func (m *NodeExpandVolumeRequest) GetVolumePath() string { + if m != nil { + return m.VolumePath + } + return "" +} + +func (m *NodeExpandVolumeRequest) GetCapacityRange() *CapacityRange { + if m != nil { + return m.CapacityRange + } + return nil +} + +func (m *NodeExpandVolumeRequest) GetStagingTargetPath() string { + if m != nil { + return m.StagingTargetPath + } + return "" +} + +func (m *NodeExpandVolumeRequest) GetVolumeCapability() *VolumeCapability { + if m != nil { + return m.VolumeCapability + } + return nil +} + +func (m *NodeExpandVolumeRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +type NodeExpandVolumeResponse struct { + // The capacity of the volume in bytes. This field is OPTIONAL. + CapacityBytes int64 `protobuf:"varint,1,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *NodeExpandVolumeResponse) Reset() { *m = NodeExpandVolumeResponse{} } +func (m *NodeExpandVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*NodeExpandVolumeResponse) ProtoMessage() {} +func (*NodeExpandVolumeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{61} +} + +func (m *NodeExpandVolumeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NodeExpandVolumeResponse.Unmarshal(m, b) +} +func (m *NodeExpandVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NodeExpandVolumeResponse.Marshal(b, m, deterministic) +} +func (m *NodeExpandVolumeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NodeExpandVolumeResponse.Merge(m, src) +} +func (m *NodeExpandVolumeResponse) XXX_Size() int { + return xxx_messageInfo_NodeExpandVolumeResponse.Size(m) +} +func (m *NodeExpandVolumeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_NodeExpandVolumeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_NodeExpandVolumeResponse proto.InternalMessageInfo + +func (m *NodeExpandVolumeResponse) GetCapacityBytes() int64 { + if m != nil { + return m.CapacityBytes + } + return 0 +} + +type GroupControllerGetCapabilitiesRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GroupControllerGetCapabilitiesRequest) Reset() { *m = GroupControllerGetCapabilitiesRequest{} } +func (m *GroupControllerGetCapabilitiesRequest) String() string { return proto.CompactTextString(m) } +func (*GroupControllerGetCapabilitiesRequest) ProtoMessage() {} +func (*GroupControllerGetCapabilitiesRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{62} +} + +func (m *GroupControllerGetCapabilitiesRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GroupControllerGetCapabilitiesRequest.Unmarshal(m, b) +} +func (m *GroupControllerGetCapabilitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GroupControllerGetCapabilitiesRequest.Marshal(b, m, deterministic) +} +func (m *GroupControllerGetCapabilitiesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GroupControllerGetCapabilitiesRequest.Merge(m, src) +} +func (m *GroupControllerGetCapabilitiesRequest) XXX_Size() int { + return xxx_messageInfo_GroupControllerGetCapabilitiesRequest.Size(m) +} +func (m *GroupControllerGetCapabilitiesRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GroupControllerGetCapabilitiesRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GroupControllerGetCapabilitiesRequest proto.InternalMessageInfo + +type GroupControllerGetCapabilitiesResponse struct { + // All the capabilities that the group controller service supports. + // This field is OPTIONAL. + Capabilities []*GroupControllerServiceCapability `protobuf:"bytes,1,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GroupControllerGetCapabilitiesResponse) Reset() { + *m = GroupControllerGetCapabilitiesResponse{} +} +func (m *GroupControllerGetCapabilitiesResponse) String() string { return proto.CompactTextString(m) } +func (*GroupControllerGetCapabilitiesResponse) ProtoMessage() {} +func (*GroupControllerGetCapabilitiesResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{63} +} + +func (m *GroupControllerGetCapabilitiesResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GroupControllerGetCapabilitiesResponse.Unmarshal(m, b) +} +func (m *GroupControllerGetCapabilitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GroupControllerGetCapabilitiesResponse.Marshal(b, m, deterministic) +} +func (m *GroupControllerGetCapabilitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GroupControllerGetCapabilitiesResponse.Merge(m, src) +} +func (m *GroupControllerGetCapabilitiesResponse) XXX_Size() int { + return xxx_messageInfo_GroupControllerGetCapabilitiesResponse.Size(m) +} +func (m *GroupControllerGetCapabilitiesResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GroupControllerGetCapabilitiesResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GroupControllerGetCapabilitiesResponse proto.InternalMessageInfo + +func (m *GroupControllerGetCapabilitiesResponse) GetCapabilities() []*GroupControllerServiceCapability { + if m != nil { + return m.Capabilities + } + return nil +} + +// Specifies a capability of the group controller service. +type GroupControllerServiceCapability struct { + // Types that are valid to be assigned to Type: + // + // *GroupControllerServiceCapability_Rpc + Type isGroupControllerServiceCapability_Type `protobuf_oneof:"type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GroupControllerServiceCapability) Reset() { *m = GroupControllerServiceCapability{} } +func (m *GroupControllerServiceCapability) String() string { return proto.CompactTextString(m) } +func (*GroupControllerServiceCapability) ProtoMessage() {} +func (*GroupControllerServiceCapability) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{64} +} + +func (m *GroupControllerServiceCapability) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GroupControllerServiceCapability.Unmarshal(m, b) +} +func (m *GroupControllerServiceCapability) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GroupControllerServiceCapability.Marshal(b, m, deterministic) +} +func (m *GroupControllerServiceCapability) XXX_Merge(src proto.Message) { + xxx_messageInfo_GroupControllerServiceCapability.Merge(m, src) +} +func (m *GroupControllerServiceCapability) XXX_Size() int { + return xxx_messageInfo_GroupControllerServiceCapability.Size(m) +} +func (m *GroupControllerServiceCapability) XXX_DiscardUnknown() { + xxx_messageInfo_GroupControllerServiceCapability.DiscardUnknown(m) +} + +var xxx_messageInfo_GroupControllerServiceCapability proto.InternalMessageInfo + +type isGroupControllerServiceCapability_Type interface { + isGroupControllerServiceCapability_Type() +} + +type GroupControllerServiceCapability_Rpc struct { + Rpc *GroupControllerServiceCapability_RPC `protobuf:"bytes,1,opt,name=rpc,proto3,oneof"` +} + +func (*GroupControllerServiceCapability_Rpc) isGroupControllerServiceCapability_Type() {} + +func (m *GroupControllerServiceCapability) GetType() isGroupControllerServiceCapability_Type { + if m != nil { + return m.Type + } + return nil +} + +func (m *GroupControllerServiceCapability) GetRpc() *GroupControllerServiceCapability_RPC { + if x, ok := m.GetType().(*GroupControllerServiceCapability_Rpc); ok { + return x.Rpc + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*GroupControllerServiceCapability) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*GroupControllerServiceCapability_Rpc)(nil), + } +} + +type GroupControllerServiceCapability_RPC struct { + Type GroupControllerServiceCapability_RPC_Type `protobuf:"varint,1,opt,name=type,proto3,enum=csi.v1.GroupControllerServiceCapability_RPC_Type" json:"type,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GroupControllerServiceCapability_RPC) Reset() { *m = GroupControllerServiceCapability_RPC{} } +func (m *GroupControllerServiceCapability_RPC) String() string { return proto.CompactTextString(m) } +func (*GroupControllerServiceCapability_RPC) ProtoMessage() {} +func (*GroupControllerServiceCapability_RPC) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{64, 0} +} + +func (m *GroupControllerServiceCapability_RPC) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GroupControllerServiceCapability_RPC.Unmarshal(m, b) +} +func (m *GroupControllerServiceCapability_RPC) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GroupControllerServiceCapability_RPC.Marshal(b, m, deterministic) +} +func (m *GroupControllerServiceCapability_RPC) XXX_Merge(src proto.Message) { + xxx_messageInfo_GroupControllerServiceCapability_RPC.Merge(m, src) +} +func (m *GroupControllerServiceCapability_RPC) XXX_Size() int { + return xxx_messageInfo_GroupControllerServiceCapability_RPC.Size(m) +} +func (m *GroupControllerServiceCapability_RPC) XXX_DiscardUnknown() { + xxx_messageInfo_GroupControllerServiceCapability_RPC.DiscardUnknown(m) +} + +var xxx_messageInfo_GroupControllerServiceCapability_RPC proto.InternalMessageInfo + +func (m *GroupControllerServiceCapability_RPC) GetType() GroupControllerServiceCapability_RPC_Type { + if m != nil { + return m.Type + } + return GroupControllerServiceCapability_RPC_UNKNOWN +} + +type CreateVolumeGroupSnapshotRequest struct { + // The suggested name for the group snapshot. This field is REQUIRED + // for idempotency. + // Any Unicode string that conforms to the length limit is allowed + // except those containing the following banned characters: + // U+0000-U+0008, U+000B, U+000C, U+000E-U+001F, U+007F-U+009F. + // (These are control characters other than commonly used whitespace.) + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // volume IDs of the source volumes to be snapshotted together. + // This field is REQUIRED. + SourceVolumeIds []string `protobuf:"bytes,2,rep,name=source_volume_ids,json=sourceVolumeIds,proto3" json:"source_volume_ids,omitempty"` + // Secrets required by plugin to complete + // ControllerCreateVolumeGroupSnapshot request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + // The secrets provided in this field SHOULD be the same for + // all group snapshot operations on the same group snapshot. + Secrets map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Plugin specific parameters passed in as opaque key-value pairs. + // This field is OPTIONAL. The Plugin is responsible for parsing and + // validating these parameters. COs will treat these as opaque. + Parameters map[string]string `protobuf:"bytes,4,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CreateVolumeGroupSnapshotRequest) Reset() { *m = CreateVolumeGroupSnapshotRequest{} } +func (m *CreateVolumeGroupSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*CreateVolumeGroupSnapshotRequest) ProtoMessage() {} +func (*CreateVolumeGroupSnapshotRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{65} +} + +func (m *CreateVolumeGroupSnapshotRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CreateVolumeGroupSnapshotRequest.Unmarshal(m, b) +} +func (m *CreateVolumeGroupSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CreateVolumeGroupSnapshotRequest.Marshal(b, m, deterministic) +} +func (m *CreateVolumeGroupSnapshotRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateVolumeGroupSnapshotRequest.Merge(m, src) +} +func (m *CreateVolumeGroupSnapshotRequest) XXX_Size() int { + return xxx_messageInfo_CreateVolumeGroupSnapshotRequest.Size(m) +} +func (m *CreateVolumeGroupSnapshotRequest) XXX_DiscardUnknown() { + xxx_messageInfo_CreateVolumeGroupSnapshotRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_CreateVolumeGroupSnapshotRequest proto.InternalMessageInfo + +func (m *CreateVolumeGroupSnapshotRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *CreateVolumeGroupSnapshotRequest) GetSourceVolumeIds() []string { + if m != nil { + return m.SourceVolumeIds + } + return nil +} + +func (m *CreateVolumeGroupSnapshotRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +func (m *CreateVolumeGroupSnapshotRequest) GetParameters() map[string]string { + if m != nil { + return m.Parameters + } + return nil +} + +type CreateVolumeGroupSnapshotResponse struct { + // Contains all attributes of the newly created group snapshot. + // This field is REQUIRED. + GroupSnapshot *VolumeGroupSnapshot `protobuf:"bytes,1,opt,name=group_snapshot,json=groupSnapshot,proto3" json:"group_snapshot,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CreateVolumeGroupSnapshotResponse) Reset() { *m = CreateVolumeGroupSnapshotResponse{} } +func (m *CreateVolumeGroupSnapshotResponse) String() string { return proto.CompactTextString(m) } +func (*CreateVolumeGroupSnapshotResponse) ProtoMessage() {} +func (*CreateVolumeGroupSnapshotResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{66} +} + +func (m *CreateVolumeGroupSnapshotResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CreateVolumeGroupSnapshotResponse.Unmarshal(m, b) +} +func (m *CreateVolumeGroupSnapshotResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CreateVolumeGroupSnapshotResponse.Marshal(b, m, deterministic) +} +func (m *CreateVolumeGroupSnapshotResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateVolumeGroupSnapshotResponse.Merge(m, src) +} +func (m *CreateVolumeGroupSnapshotResponse) XXX_Size() int { + return xxx_messageInfo_CreateVolumeGroupSnapshotResponse.Size(m) +} +func (m *CreateVolumeGroupSnapshotResponse) XXX_DiscardUnknown() { + xxx_messageInfo_CreateVolumeGroupSnapshotResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_CreateVolumeGroupSnapshotResponse proto.InternalMessageInfo + +func (m *CreateVolumeGroupSnapshotResponse) GetGroupSnapshot() *VolumeGroupSnapshot { + if m != nil { + return m.GroupSnapshot + } + return nil +} + +type VolumeGroupSnapshot struct { + // The identifier for this group snapshot, generated by the plugin. + // This field MUST contain enough information to uniquely identify + // this specific snapshot vs all other group snapshots supported by + // this plugin. + // This field SHALL be used by the CO in subsequent calls to refer to + // this group snapshot. + // The SP is NOT responsible for global uniqueness of + // group_snapshot_id across multiple SPs. + // This field is REQUIRED. + GroupSnapshotId string `protobuf:"bytes,1,opt,name=group_snapshot_id,json=groupSnapshotId,proto3" json:"group_snapshot_id,omitempty"` + // A list of snapshots belonging to this group. + // This field is REQUIRED. + Snapshots []*Snapshot `protobuf:"bytes,2,rep,name=snapshots,proto3" json:"snapshots,omitempty"` + // Timestamp of when the volume group snapshot was taken. + // This field is REQUIRED. + CreationTime *timestamp.Timestamp `protobuf:"bytes,3,opt,name=creation_time,json=creationTime,proto3" json:"creation_time,omitempty"` + // Indicates if all individual snapshots in the group snapshot + // are ready to use as a `volume_content_source` in a + // `CreateVolumeRequest`. The default value is false. + // If any snapshot in the list of snapshots in this message have + // ready_to_use set to false, the SP MUST set this field to false. + // If all of the snapshots in the list of snapshots in this message + // have ready_to_use set to true, the SP SHOULD set this field to + // true. + // This field is REQUIRED. + ReadyToUse bool `protobuf:"varint,4,opt,name=ready_to_use,json=readyToUse,proto3" json:"ready_to_use,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeGroupSnapshot) Reset() { *m = VolumeGroupSnapshot{} } +func (m *VolumeGroupSnapshot) String() string { return proto.CompactTextString(m) } +func (*VolumeGroupSnapshot) ProtoMessage() {} +func (*VolumeGroupSnapshot) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{67} +} + +func (m *VolumeGroupSnapshot) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VolumeGroupSnapshot.Unmarshal(m, b) +} +func (m *VolumeGroupSnapshot) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VolumeGroupSnapshot.Marshal(b, m, deterministic) +} +func (m *VolumeGroupSnapshot) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeGroupSnapshot.Merge(m, src) +} +func (m *VolumeGroupSnapshot) XXX_Size() int { + return xxx_messageInfo_VolumeGroupSnapshot.Size(m) +} +func (m *VolumeGroupSnapshot) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeGroupSnapshot.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeGroupSnapshot proto.InternalMessageInfo + +func (m *VolumeGroupSnapshot) GetGroupSnapshotId() string { + if m != nil { + return m.GroupSnapshotId + } + return "" +} + +func (m *VolumeGroupSnapshot) GetSnapshots() []*Snapshot { + if m != nil { + return m.Snapshots + } + return nil +} + +func (m *VolumeGroupSnapshot) GetCreationTime() *timestamp.Timestamp { + if m != nil { + return m.CreationTime + } + return nil +} + +func (m *VolumeGroupSnapshot) GetReadyToUse() bool { + if m != nil { + return m.ReadyToUse + } + return false +} + +type DeleteVolumeGroupSnapshotRequest struct { + // The ID of the group snapshot to be deleted. + // This field is REQUIRED. + GroupSnapshotId string `protobuf:"bytes,1,opt,name=group_snapshot_id,json=groupSnapshotId,proto3" json:"group_snapshot_id,omitempty"` + // A list of snapshot IDs that are part of this group snapshot. + // If SP does not need to rely on this field to delete the snapshots + // in the group, it SHOULD check this field and report an error + // if it has the ability to detect a mismatch. + // Some SPs require this list to delete the snapshots in the group. + // If SP needs to use this field to delete the snapshots in the + // group, it MUST report an error if it has the ability to detect + // a mismatch. + // This field is REQUIRED. + SnapshotIds []string `protobuf:"bytes,2,rep,name=snapshot_ids,json=snapshotIds,proto3" json:"snapshot_ids,omitempty"` + // Secrets required by plugin to complete group snapshot deletion + // request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + // The secrets provided in this field SHOULD be the same for + // all group snapshot operations on the same group snapshot. + Secrets map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteVolumeGroupSnapshotRequest) Reset() { *m = DeleteVolumeGroupSnapshotRequest{} } +func (m *DeleteVolumeGroupSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteVolumeGroupSnapshotRequest) ProtoMessage() {} +func (*DeleteVolumeGroupSnapshotRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{68} +} + +func (m *DeleteVolumeGroupSnapshotRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteVolumeGroupSnapshotRequest.Unmarshal(m, b) +} +func (m *DeleteVolumeGroupSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteVolumeGroupSnapshotRequest.Marshal(b, m, deterministic) +} +func (m *DeleteVolumeGroupSnapshotRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteVolumeGroupSnapshotRequest.Merge(m, src) +} +func (m *DeleteVolumeGroupSnapshotRequest) XXX_Size() int { + return xxx_messageInfo_DeleteVolumeGroupSnapshotRequest.Size(m) +} +func (m *DeleteVolumeGroupSnapshotRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteVolumeGroupSnapshotRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteVolumeGroupSnapshotRequest proto.InternalMessageInfo + +func (m *DeleteVolumeGroupSnapshotRequest) GetGroupSnapshotId() string { + if m != nil { + return m.GroupSnapshotId + } + return "" +} + +func (m *DeleteVolumeGroupSnapshotRequest) GetSnapshotIds() []string { + if m != nil { + return m.SnapshotIds + } + return nil +} + +func (m *DeleteVolumeGroupSnapshotRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +type DeleteVolumeGroupSnapshotResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteVolumeGroupSnapshotResponse) Reset() { *m = DeleteVolumeGroupSnapshotResponse{} } +func (m *DeleteVolumeGroupSnapshotResponse) String() string { return proto.CompactTextString(m) } +func (*DeleteVolumeGroupSnapshotResponse) ProtoMessage() {} +func (*DeleteVolumeGroupSnapshotResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{69} +} + +func (m *DeleteVolumeGroupSnapshotResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteVolumeGroupSnapshotResponse.Unmarshal(m, b) +} +func (m *DeleteVolumeGroupSnapshotResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteVolumeGroupSnapshotResponse.Marshal(b, m, deterministic) +} +func (m *DeleteVolumeGroupSnapshotResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteVolumeGroupSnapshotResponse.Merge(m, src) +} +func (m *DeleteVolumeGroupSnapshotResponse) XXX_Size() int { + return xxx_messageInfo_DeleteVolumeGroupSnapshotResponse.Size(m) +} +func (m *DeleteVolumeGroupSnapshotResponse) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteVolumeGroupSnapshotResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteVolumeGroupSnapshotResponse proto.InternalMessageInfo + +type GetVolumeGroupSnapshotRequest struct { + // The ID of the group snapshot to fetch current group snapshot + // information for. + // This field is REQUIRED. + GroupSnapshotId string `protobuf:"bytes,1,opt,name=group_snapshot_id,json=groupSnapshotId,proto3" json:"group_snapshot_id,omitempty"` + // A list of snapshot IDs that are part of this group snapshot. + // If SP does not need to rely on this field to get the snapshots + // in the group, it SHOULD check this field and report an error + // if it has the ability to detect a mismatch. + // Some SPs require this list to get the snapshots in the group. + // If SP needs to use this field to get the snapshots in the + // group, it MUST report an error if it has the ability to detect + // a mismatch. + // This field is REQUIRED. + SnapshotIds []string `protobuf:"bytes,2,rep,name=snapshot_ids,json=snapshotIds,proto3" json:"snapshot_ids,omitempty"` + // Secrets required by plugin to complete + // GetVolumeGroupSnapshot request. + // This field is OPTIONAL. Refer to the `Secrets Requirements` + // section on how to use this field. + // The secrets provided in this field SHOULD be the same for + // all group snapshot operations on the same group snapshot. + Secrets map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetVolumeGroupSnapshotRequest) Reset() { *m = GetVolumeGroupSnapshotRequest{} } +func (m *GetVolumeGroupSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*GetVolumeGroupSnapshotRequest) ProtoMessage() {} +func (*GetVolumeGroupSnapshotRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{70} +} + +func (m *GetVolumeGroupSnapshotRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetVolumeGroupSnapshotRequest.Unmarshal(m, b) +} +func (m *GetVolumeGroupSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetVolumeGroupSnapshotRequest.Marshal(b, m, deterministic) +} +func (m *GetVolumeGroupSnapshotRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetVolumeGroupSnapshotRequest.Merge(m, src) +} +func (m *GetVolumeGroupSnapshotRequest) XXX_Size() int { + return xxx_messageInfo_GetVolumeGroupSnapshotRequest.Size(m) +} +func (m *GetVolumeGroupSnapshotRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetVolumeGroupSnapshotRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetVolumeGroupSnapshotRequest proto.InternalMessageInfo + +func (m *GetVolumeGroupSnapshotRequest) GetGroupSnapshotId() string { + if m != nil { + return m.GroupSnapshotId + } + return "" +} + +func (m *GetVolumeGroupSnapshotRequest) GetSnapshotIds() []string { + if m != nil { + return m.SnapshotIds + } + return nil +} + +func (m *GetVolumeGroupSnapshotRequest) GetSecrets() map[string]string { + if m != nil { + return m.Secrets + } + return nil +} + +type GetVolumeGroupSnapshotResponse struct { + // This field is REQUIRED + GroupSnapshot *VolumeGroupSnapshot `protobuf:"bytes,1,opt,name=group_snapshot,json=groupSnapshot,proto3" json:"group_snapshot,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetVolumeGroupSnapshotResponse) Reset() { *m = GetVolumeGroupSnapshotResponse{} } +func (m *GetVolumeGroupSnapshotResponse) String() string { return proto.CompactTextString(m) } +func (*GetVolumeGroupSnapshotResponse) ProtoMessage() {} +func (*GetVolumeGroupSnapshotResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_9cdb00adce470e01, []int{71} +} + +func (m *GetVolumeGroupSnapshotResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetVolumeGroupSnapshotResponse.Unmarshal(m, b) +} +func (m *GetVolumeGroupSnapshotResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetVolumeGroupSnapshotResponse.Marshal(b, m, deterministic) +} +func (m *GetVolumeGroupSnapshotResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetVolumeGroupSnapshotResponse.Merge(m, src) +} +func (m *GetVolumeGroupSnapshotResponse) XXX_Size() int { + return xxx_messageInfo_GetVolumeGroupSnapshotResponse.Size(m) +} +func (m *GetVolumeGroupSnapshotResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetVolumeGroupSnapshotResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetVolumeGroupSnapshotResponse proto.InternalMessageInfo + +func (m *GetVolumeGroupSnapshotResponse) GetGroupSnapshot() *VolumeGroupSnapshot { + if m != nil { + return m.GroupSnapshot + } + return nil +} + +var E_AlphaEnum = &proto.ExtensionDesc{ + ExtendedType: (*descriptor.EnumOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 1060, + Name: "csi.v1.alpha_enum", + Tag: "varint,1060,opt,name=alpha_enum", + Filename: "github.com/container-storage-interface/spec/csi.proto", +} + +var E_AlphaEnumValue = &proto.ExtensionDesc{ + ExtendedType: (*descriptor.EnumValueOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 1060, + Name: "csi.v1.alpha_enum_value", + Tag: "varint,1060,opt,name=alpha_enum_value", + Filename: "github.com/container-storage-interface/spec/csi.proto", +} + +var E_CsiSecret = &proto.ExtensionDesc{ + ExtendedType: (*descriptor.FieldOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 1059, + Name: "csi.v1.csi_secret", + Tag: "varint,1059,opt,name=csi_secret", + Filename: "github.com/container-storage-interface/spec/csi.proto", +} + +var E_AlphaField = &proto.ExtensionDesc{ + ExtendedType: (*descriptor.FieldOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 1060, + Name: "csi.v1.alpha_field", + Tag: "varint,1060,opt,name=alpha_field", + Filename: "github.com/container-storage-interface/spec/csi.proto", +} + +var E_AlphaMessage = &proto.ExtensionDesc{ + ExtendedType: (*descriptor.MessageOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 1060, + Name: "csi.v1.alpha_message", + Tag: "varint,1060,opt,name=alpha_message", + Filename: "github.com/container-storage-interface/spec/csi.proto", +} + +var E_AlphaMethod = &proto.ExtensionDesc{ + ExtendedType: (*descriptor.MethodOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 1060, + Name: "csi.v1.alpha_method", + Tag: "varint,1060,opt,name=alpha_method", + Filename: "github.com/container-storage-interface/spec/csi.proto", +} + +var E_AlphaService = &proto.ExtensionDesc{ + ExtendedType: (*descriptor.ServiceOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 1060, + Name: "csi.v1.alpha_service", + Tag: "varint,1060,opt,name=alpha_service", + Filename: "github.com/container-storage-interface/spec/csi.proto", +} + +func init() { + proto.RegisterEnum("csi.v1.PluginCapability_Service_Type", PluginCapability_Service_Type_name, PluginCapability_Service_Type_value) + proto.RegisterEnum("csi.v1.PluginCapability_VolumeExpansion_Type", PluginCapability_VolumeExpansion_Type_name, PluginCapability_VolumeExpansion_Type_value) + proto.RegisterEnum("csi.v1.VolumeCapability_AccessMode_Mode", VolumeCapability_AccessMode_Mode_name, VolumeCapability_AccessMode_Mode_value) + proto.RegisterEnum("csi.v1.ControllerServiceCapability_RPC_Type", ControllerServiceCapability_RPC_Type_name, ControllerServiceCapability_RPC_Type_value) + proto.RegisterEnum("csi.v1.VolumeUsage_Unit", VolumeUsage_Unit_name, VolumeUsage_Unit_value) + proto.RegisterEnum("csi.v1.NodeServiceCapability_RPC_Type", NodeServiceCapability_RPC_Type_name, NodeServiceCapability_RPC_Type_value) + proto.RegisterEnum("csi.v1.GroupControllerServiceCapability_RPC_Type", GroupControllerServiceCapability_RPC_Type_name, GroupControllerServiceCapability_RPC_Type_value) + proto.RegisterType((*GetPluginInfoRequest)(nil), "csi.v1.GetPluginInfoRequest") + proto.RegisterType((*GetPluginInfoResponse)(nil), "csi.v1.GetPluginInfoResponse") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.GetPluginInfoResponse.ManifestEntry") + proto.RegisterType((*GetPluginCapabilitiesRequest)(nil), "csi.v1.GetPluginCapabilitiesRequest") + proto.RegisterType((*GetPluginCapabilitiesResponse)(nil), "csi.v1.GetPluginCapabilitiesResponse") + proto.RegisterType((*PluginCapability)(nil), "csi.v1.PluginCapability") + proto.RegisterType((*PluginCapability_Service)(nil), "csi.v1.PluginCapability.Service") + proto.RegisterType((*PluginCapability_VolumeExpansion)(nil), "csi.v1.PluginCapability.VolumeExpansion") + proto.RegisterType((*ProbeRequest)(nil), "csi.v1.ProbeRequest") + proto.RegisterType((*ProbeResponse)(nil), "csi.v1.ProbeResponse") + proto.RegisterType((*CreateVolumeRequest)(nil), "csi.v1.CreateVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateVolumeRequest.MutableParametersEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateVolumeRequest.ParametersEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateVolumeRequest.SecretsEntry") + proto.RegisterType((*VolumeContentSource)(nil), "csi.v1.VolumeContentSource") + proto.RegisterType((*VolumeContentSource_SnapshotSource)(nil), "csi.v1.VolumeContentSource.SnapshotSource") + proto.RegisterType((*VolumeContentSource_VolumeSource)(nil), "csi.v1.VolumeContentSource.VolumeSource") + proto.RegisterType((*CreateVolumeResponse)(nil), "csi.v1.CreateVolumeResponse") + proto.RegisterType((*VolumeCapability)(nil), "csi.v1.VolumeCapability") + proto.RegisterType((*VolumeCapability_BlockVolume)(nil), "csi.v1.VolumeCapability.BlockVolume") + proto.RegisterType((*VolumeCapability_MountVolume)(nil), "csi.v1.VolumeCapability.MountVolume") + proto.RegisterType((*VolumeCapability_AccessMode)(nil), "csi.v1.VolumeCapability.AccessMode") + proto.RegisterType((*CapacityRange)(nil), "csi.v1.CapacityRange") + proto.RegisterType((*Volume)(nil), "csi.v1.Volume") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.Volume.VolumeContextEntry") + proto.RegisterType((*TopologyRequirement)(nil), "csi.v1.TopologyRequirement") + proto.RegisterType((*Topology)(nil), "csi.v1.Topology") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.Topology.SegmentsEntry") + proto.RegisterType((*DeleteVolumeRequest)(nil), "csi.v1.DeleteVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.DeleteVolumeRequest.SecretsEntry") + proto.RegisterType((*DeleteVolumeResponse)(nil), "csi.v1.DeleteVolumeResponse") + proto.RegisterType((*ControllerPublishVolumeRequest)(nil), "csi.v1.ControllerPublishVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerPublishVolumeRequest.SecretsEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerPublishVolumeRequest.VolumeContextEntry") + proto.RegisterType((*ControllerPublishVolumeResponse)(nil), "csi.v1.ControllerPublishVolumeResponse") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerPublishVolumeResponse.PublishContextEntry") + proto.RegisterType((*ControllerUnpublishVolumeRequest)(nil), "csi.v1.ControllerUnpublishVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerUnpublishVolumeRequest.SecretsEntry") + proto.RegisterType((*ControllerUnpublishVolumeResponse)(nil), "csi.v1.ControllerUnpublishVolumeResponse") + proto.RegisterType((*ValidateVolumeCapabilitiesRequest)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest.MutableParametersEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest.ParametersEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest.SecretsEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest.VolumeContextEntry") + proto.RegisterType((*ValidateVolumeCapabilitiesResponse)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse") + proto.RegisterType((*ValidateVolumeCapabilitiesResponse_Confirmed)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse.Confirmed") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse.Confirmed.MutableParametersEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse.Confirmed.ParametersEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse.Confirmed.VolumeContextEntry") + proto.RegisterType((*ListVolumesRequest)(nil), "csi.v1.ListVolumesRequest") + proto.RegisterType((*ListVolumesResponse)(nil), "csi.v1.ListVolumesResponse") + proto.RegisterType((*ListVolumesResponse_VolumeStatus)(nil), "csi.v1.ListVolumesResponse.VolumeStatus") + proto.RegisterType((*ListVolumesResponse_Entry)(nil), "csi.v1.ListVolumesResponse.Entry") + proto.RegisterType((*ControllerGetVolumeRequest)(nil), "csi.v1.ControllerGetVolumeRequest") + proto.RegisterType((*ControllerGetVolumeResponse)(nil), "csi.v1.ControllerGetVolumeResponse") + proto.RegisterType((*ControllerGetVolumeResponse_VolumeStatus)(nil), "csi.v1.ControllerGetVolumeResponse.VolumeStatus") + proto.RegisterType((*ControllerModifyVolumeRequest)(nil), "csi.v1.ControllerModifyVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerModifyVolumeRequest.MutableParametersEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerModifyVolumeRequest.SecretsEntry") + proto.RegisterType((*ControllerModifyVolumeResponse)(nil), "csi.v1.ControllerModifyVolumeResponse") + proto.RegisterType((*GetCapacityRequest)(nil), "csi.v1.GetCapacityRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.GetCapacityRequest.ParametersEntry") + proto.RegisterType((*GetCapacityResponse)(nil), "csi.v1.GetCapacityResponse") + proto.RegisterType((*ControllerGetCapabilitiesRequest)(nil), "csi.v1.ControllerGetCapabilitiesRequest") + proto.RegisterType((*ControllerGetCapabilitiesResponse)(nil), "csi.v1.ControllerGetCapabilitiesResponse") + proto.RegisterType((*ControllerServiceCapability)(nil), "csi.v1.ControllerServiceCapability") + proto.RegisterType((*ControllerServiceCapability_RPC)(nil), "csi.v1.ControllerServiceCapability.RPC") + proto.RegisterType((*CreateSnapshotRequest)(nil), "csi.v1.CreateSnapshotRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateSnapshotRequest.ParametersEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateSnapshotRequest.SecretsEntry") + proto.RegisterType((*CreateSnapshotResponse)(nil), "csi.v1.CreateSnapshotResponse") + proto.RegisterType((*Snapshot)(nil), "csi.v1.Snapshot") + proto.RegisterType((*DeleteSnapshotRequest)(nil), "csi.v1.DeleteSnapshotRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.DeleteSnapshotRequest.SecretsEntry") + proto.RegisterType((*DeleteSnapshotResponse)(nil), "csi.v1.DeleteSnapshotResponse") + proto.RegisterType((*ListSnapshotsRequest)(nil), "csi.v1.ListSnapshotsRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ListSnapshotsRequest.SecretsEntry") + proto.RegisterType((*ListSnapshotsResponse)(nil), "csi.v1.ListSnapshotsResponse") + proto.RegisterType((*ListSnapshotsResponse_Entry)(nil), "csi.v1.ListSnapshotsResponse.Entry") + proto.RegisterType((*ControllerExpandVolumeRequest)(nil), "csi.v1.ControllerExpandVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerExpandVolumeRequest.SecretsEntry") + proto.RegisterType((*ControllerExpandVolumeResponse)(nil), "csi.v1.ControllerExpandVolumeResponse") + proto.RegisterType((*NodeStageVolumeRequest)(nil), "csi.v1.NodeStageVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodeStageVolumeRequest.PublishContextEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodeStageVolumeRequest.SecretsEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodeStageVolumeRequest.VolumeContextEntry") + proto.RegisterType((*NodeStageVolumeResponse)(nil), "csi.v1.NodeStageVolumeResponse") + proto.RegisterType((*NodeUnstageVolumeRequest)(nil), "csi.v1.NodeUnstageVolumeRequest") + proto.RegisterType((*NodeUnstageVolumeResponse)(nil), "csi.v1.NodeUnstageVolumeResponse") + proto.RegisterType((*NodePublishVolumeRequest)(nil), "csi.v1.NodePublishVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodePublishVolumeRequest.PublishContextEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodePublishVolumeRequest.SecretsEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodePublishVolumeRequest.VolumeContextEntry") + proto.RegisterType((*NodePublishVolumeResponse)(nil), "csi.v1.NodePublishVolumeResponse") + proto.RegisterType((*NodeUnpublishVolumeRequest)(nil), "csi.v1.NodeUnpublishVolumeRequest") + proto.RegisterType((*NodeUnpublishVolumeResponse)(nil), "csi.v1.NodeUnpublishVolumeResponse") + proto.RegisterType((*NodeGetVolumeStatsRequest)(nil), "csi.v1.NodeGetVolumeStatsRequest") + proto.RegisterType((*NodeGetVolumeStatsResponse)(nil), "csi.v1.NodeGetVolumeStatsResponse") + proto.RegisterType((*VolumeUsage)(nil), "csi.v1.VolumeUsage") + proto.RegisterType((*VolumeCondition)(nil), "csi.v1.VolumeCondition") + proto.RegisterType((*NodeGetCapabilitiesRequest)(nil), "csi.v1.NodeGetCapabilitiesRequest") + proto.RegisterType((*NodeGetCapabilitiesResponse)(nil), "csi.v1.NodeGetCapabilitiesResponse") + proto.RegisterType((*NodeServiceCapability)(nil), "csi.v1.NodeServiceCapability") + proto.RegisterType((*NodeServiceCapability_RPC)(nil), "csi.v1.NodeServiceCapability.RPC") + proto.RegisterType((*NodeGetInfoRequest)(nil), "csi.v1.NodeGetInfoRequest") + proto.RegisterType((*NodeGetInfoResponse)(nil), "csi.v1.NodeGetInfoResponse") + proto.RegisterType((*NodeExpandVolumeRequest)(nil), "csi.v1.NodeExpandVolumeRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodeExpandVolumeRequest.SecretsEntry") + proto.RegisterType((*NodeExpandVolumeResponse)(nil), "csi.v1.NodeExpandVolumeResponse") + proto.RegisterType((*GroupControllerGetCapabilitiesRequest)(nil), "csi.v1.GroupControllerGetCapabilitiesRequest") + proto.RegisterType((*GroupControllerGetCapabilitiesResponse)(nil), "csi.v1.GroupControllerGetCapabilitiesResponse") + proto.RegisterType((*GroupControllerServiceCapability)(nil), "csi.v1.GroupControllerServiceCapability") + proto.RegisterType((*GroupControllerServiceCapability_RPC)(nil), "csi.v1.GroupControllerServiceCapability.RPC") + proto.RegisterType((*CreateVolumeGroupSnapshotRequest)(nil), "csi.v1.CreateVolumeGroupSnapshotRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateVolumeGroupSnapshotRequest.ParametersEntry") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateVolumeGroupSnapshotRequest.SecretsEntry") + proto.RegisterType((*CreateVolumeGroupSnapshotResponse)(nil), "csi.v1.CreateVolumeGroupSnapshotResponse") + proto.RegisterType((*VolumeGroupSnapshot)(nil), "csi.v1.VolumeGroupSnapshot") + proto.RegisterType((*DeleteVolumeGroupSnapshotRequest)(nil), "csi.v1.DeleteVolumeGroupSnapshotRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.DeleteVolumeGroupSnapshotRequest.SecretsEntry") + proto.RegisterType((*DeleteVolumeGroupSnapshotResponse)(nil), "csi.v1.DeleteVolumeGroupSnapshotResponse") + proto.RegisterType((*GetVolumeGroupSnapshotRequest)(nil), "csi.v1.GetVolumeGroupSnapshotRequest") + proto.RegisterMapType((map[string]string)(nil), "csi.v1.GetVolumeGroupSnapshotRequest.SecretsEntry") + proto.RegisterType((*GetVolumeGroupSnapshotResponse)(nil), "csi.v1.GetVolumeGroupSnapshotResponse") + proto.RegisterExtension(E_AlphaEnum) + proto.RegisterExtension(E_AlphaEnumValue) + proto.RegisterExtension(E_CsiSecret) + proto.RegisterExtension(E_AlphaField) + proto.RegisterExtension(E_AlphaMessage) + proto.RegisterExtension(E_AlphaMethod) + proto.RegisterExtension(E_AlphaService) +} + +func init() { + proto.RegisterFile("github.com/container-storage-interface/spec/csi.proto", fileDescriptor_9cdb00adce470e01) +} + +var fileDescriptor_9cdb00adce470e01 = []byte{ + // 4327 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x3c, 0x4d, 0x6c, 0x1c, 0x59, + 0x5a, 0xae, 0xfe, 0xb3, 0xfd, 0xf9, 0x27, 0xed, 0x67, 0xc7, 0xe9, 0x54, 0x12, 0xc7, 0xae, 0x6c, + 0x32, 0x9e, 0x4c, 0xd2, 0x99, 0xf1, 0xce, 0x8c, 0x76, 0x3c, 0x99, 0xdd, 0x74, 0xdb, 0x1d, 0xbb, + 0x37, 0x76, 0xb7, 0xa7, 0xba, 0x9d, 0xd9, 0x04, 0x46, 0x35, 0xe5, 0xee, 0xb2, 0x53, 0x9a, 0xee, + 0xaa, 0x9e, 0xaa, 0x6a, 0x13, 0x2f, 0x87, 0x85, 0x5d, 0x21, 0x76, 0xc5, 0x05, 0x81, 0x04, 0x23, + 0x21, 0xc1, 0x0a, 0x38, 0x2e, 0x5a, 0x21, 0x84, 0x90, 0xe0, 0xc0, 0x19, 0x4e, 0x1c, 0x81, 0x0b, + 0x07, 0x24, 0x90, 0x56, 0x20, 0x2d, 0x12, 0xe2, 0xc0, 0x09, 0xd5, 0x7b, 0xaf, 0xaa, 0x5f, 0x55, + 0xbd, 0xaa, 0xee, 0x8e, 0x1d, 0x96, 0x15, 0xa7, 0xb8, 0xdf, 0xfb, 0xfe, 0xde, 0xcf, 0xf7, 0xff, + 0x2a, 0xf0, 0xde, 0x89, 0xee, 0xbc, 0xe8, 0x1f, 0x15, 0x5b, 0x66, 0xf7, 0x41, 0xcb, 0x34, 0x1c, + 0x55, 0x37, 0x34, 0xeb, 0xbe, 0xed, 0x98, 0x96, 0x7a, 0xa2, 0xdd, 0xd7, 0x0d, 0x47, 0xb3, 0x8e, + 0xd5, 0x96, 0xf6, 0xc0, 0xee, 0x69, 0xad, 0x07, 0x2d, 0x5b, 0x2f, 0xf6, 0x2c, 0xd3, 0x31, 0x51, + 0xce, 0xfd, 0xf3, 0xf4, 0x1d, 0x71, 0xf5, 0xc4, 0x34, 0x4f, 0x3a, 0xda, 0x03, 0x3c, 0x7a, 0xd4, + 0x3f, 0x7e, 0xd0, 0xd6, 0xec, 0x96, 0xa5, 0xf7, 0x1c, 0xd3, 0x22, 0x90, 0xe2, 0xcd, 0x30, 0x84, + 0xa3, 0x77, 0x35, 0xdb, 0x51, 0xbb, 0x3d, 0x0a, 0xb0, 0x12, 0x06, 0xf8, 0x25, 0x4b, 0xed, 0xf5, + 0x34, 0xcb, 0x26, 0xf3, 0xd2, 0x32, 0x2c, 0xed, 0x68, 0xce, 0x41, 0xa7, 0x7f, 0xa2, 0x1b, 0x55, + 0xe3, 0xd8, 0x94, 0xb5, 0x2f, 0xfa, 0x9a, 0xed, 0x48, 0x7f, 0x2f, 0xc0, 0xe5, 0xd0, 0x84, 0xdd, + 0x33, 0x0d, 0x5b, 0x43, 0x08, 0x32, 0x86, 0xda, 0xd5, 0x0a, 0xc2, 0xaa, 0xb0, 0x3e, 0x2d, 0xe3, + 0xbf, 0xd1, 0x6d, 0x98, 0x3f, 0xd5, 0x8c, 0xb6, 0x69, 0x29, 0xa7, 0x9a, 0x65, 0xeb, 0xa6, 0x51, + 0x48, 0xe1, 0xd9, 0x39, 0x32, 0xfa, 0x94, 0x0c, 0xa2, 0x1d, 0x98, 0xea, 0xaa, 0x86, 0x7e, 0xac, + 0xd9, 0x4e, 0x21, 0xbd, 0x9a, 0x5e, 0x9f, 0xd9, 0x78, 0xab, 0x48, 0x96, 0x5a, 0xe4, 0xf2, 0x2a, + 0xee, 0x53, 0xe8, 0x8a, 0xe1, 0x58, 0x67, 0xb2, 0x8f, 0x2c, 0x7e, 0x08, 0x73, 0x81, 0x29, 0x94, + 0x87, 0xf4, 0xe7, 0xda, 0x19, 0x95, 0xc9, 0xfd, 0x13, 0x2d, 0x41, 0xf6, 0x54, 0xed, 0xf4, 0x35, + 0x2a, 0x09, 0xf9, 0xb1, 0x99, 0xfa, 0x9a, 0x20, 0xad, 0xc0, 0x75, 0x9f, 0xdb, 0x96, 0xda, 0x53, + 0x8f, 0xf4, 0x8e, 0xee, 0xe8, 0x9a, 0xed, 0x2d, 0xfd, 0x53, 0xb8, 0x11, 0x33, 0x4f, 0x77, 0xe0, + 0x21, 0xcc, 0xb6, 0x98, 0xf1, 0x82, 0x80, 0x97, 0x52, 0xf0, 0x96, 0x12, 0xc2, 0x3c, 0x93, 0x03, + 0xd0, 0xd2, 0x7f, 0xa4, 0x21, 0x1f, 0x06, 0x41, 0x0f, 0x61, 0xd2, 0xd6, 0xac, 0x53, 0xbd, 0x45, + 0xf6, 0x75, 0x66, 0x63, 0x35, 0x8e, 0x5a, 0xb1, 0x41, 0xe0, 0x76, 0x27, 0x64, 0x0f, 0x05, 0x1d, + 0x42, 0xfe, 0xd4, 0xec, 0xf4, 0xbb, 0x9a, 0xa2, 0xbd, 0xec, 0xa9, 0x86, 0x7f, 0x00, 0x33, 0x1b, + 0xeb, 0xb1, 0x64, 0x9e, 0x62, 0x84, 0x8a, 0x07, 0xbf, 0x3b, 0x21, 0x5f, 0x3a, 0x0d, 0x0e, 0x89, + 0x7f, 0x29, 0xc0, 0x24, 0xe5, 0x86, 0x3e, 0x80, 0x8c, 0x73, 0xd6, 0x23, 0xd2, 0xcd, 0x6f, 0xdc, + 0x1e, 0x26, 0x5d, 0xb1, 0x79, 0xd6, 0xd3, 0x64, 0x8c, 0x22, 0x39, 0x90, 0x71, 0x7f, 0xa1, 0x19, + 0x98, 0x3c, 0xac, 0x3d, 0xa9, 0xd5, 0x3f, 0xa9, 0xe5, 0x27, 0xd0, 0x32, 0xa0, 0xad, 0x7a, 0xad, + 0x29, 0xd7, 0xf7, 0xf6, 0x2a, 0xb2, 0xd2, 0xa8, 0xc8, 0x4f, 0xab, 0x5b, 0x95, 0xbc, 0x80, 0xbe, + 0x02, 0xab, 0x4f, 0xeb, 0x7b, 0x87, 0xfb, 0x15, 0xa5, 0xb4, 0xb5, 0x55, 0x69, 0x34, 0xaa, 0xe5, + 0xea, 0x5e, 0xb5, 0xf9, 0x4c, 0xd9, 0xaa, 0xd7, 0x1a, 0x4d, 0xb9, 0x54, 0xad, 0x35, 0x1b, 0xf9, + 0x14, 0x5a, 0x83, 0xc2, 0x8e, 0x5c, 0x3f, 0x3c, 0x50, 0x38, 0x34, 0xd2, 0x62, 0xfa, 0x87, 0x65, + 0x41, 0xfc, 0xae, 0x00, 0x97, 0x42, 0x6b, 0x44, 0xa5, 0xc0, 0x22, 0xee, 0x8f, 0xba, 0x37, 0xec, + 0x62, 0xee, 0xf1, 0x16, 0x03, 0x90, 0xab, 0xd7, 0xf6, 0xaa, 0x35, 0x77, 0x01, 0x33, 0x30, 0x59, + 0x7f, 0xfc, 0x18, 0xff, 0x48, 0x95, 0x73, 0x84, 0xa1, 0x34, 0x0f, 0xb3, 0x07, 0x96, 0x79, 0xa4, + 0x79, 0x57, 0xac, 0x04, 0x73, 0xf4, 0x37, 0xbd, 0x52, 0x6f, 0x43, 0xd6, 0xd2, 0xd4, 0xf6, 0x19, + 0x3d, 0x7d, 0xb1, 0x48, 0xd4, 0xb6, 0xe8, 0xa9, 0x6d, 0xb1, 0x6c, 0x9a, 0x9d, 0xa7, 0xee, 0x15, + 0x96, 0x09, 0xa0, 0xf4, 0xfb, 0x39, 0x58, 0xdc, 0xb2, 0x34, 0xd5, 0xd1, 0x88, 0xb4, 0x94, 0x34, + 0x57, 0x3d, 0x1f, 0xc2, 0xbc, 0x7b, 0x05, 0x5b, 0xba, 0x73, 0xa6, 0x58, 0xaa, 0x71, 0xa2, 0xd1, + 0xdb, 0x71, 0xd9, 0xdb, 0x81, 0x2d, 0x3a, 0x2b, 0xbb, 0x93, 0xf2, 0x5c, 0x8b, 0xfd, 0x89, 0xaa, + 0xb0, 0x48, 0x6f, 0x57, 0xe0, 0xd6, 0xa7, 0x83, 0xb7, 0x9e, 0x48, 0xc1, 0xdc, 0x7a, 0x74, 0x1a, + 0x1c, 0xd1, 0x35, 0x1b, 0x3d, 0x01, 0xe8, 0xa9, 0x96, 0xda, 0xd5, 0x1c, 0xcd, 0xb2, 0x0b, 0x99, + 0xa0, 0x09, 0xe0, 0xac, 0xa6, 0x78, 0xe0, 0x43, 0x13, 0x13, 0xc0, 0xa0, 0xa3, 0x1d, 0x57, 0x67, + 0x5a, 0x96, 0xe6, 0xd8, 0x85, 0x2c, 0xa6, 0xb4, 0x9e, 0x44, 0xa9, 0x41, 0x40, 0x31, 0x99, 0x72, + 0xfa, 0xcb, 0xb2, 0x20, 0x7b, 0xd8, 0xa8, 0x0e, 0x97, 0xbd, 0x05, 0x9a, 0x86, 0xa3, 0x19, 0x8e, + 0x62, 0x9b, 0x7d, 0xab, 0xa5, 0x15, 0x72, 0x78, 0x97, 0xae, 0x85, 0x96, 0x48, 0x60, 0x1a, 0x18, + 0x44, 0xa6, 0x5b, 0x13, 0x18, 0x44, 0xcf, 0x41, 0x54, 0x5b, 0x2d, 0xcd, 0xb6, 0x75, 0xb2, 0x17, + 0x8a, 0xa5, 0x7d, 0xd1, 0xd7, 0x2d, 0xad, 0xab, 0x19, 0x8e, 0x5d, 0x98, 0x0c, 0x52, 0x6d, 0x9a, + 0x3d, 0xb3, 0x63, 0x9e, 0x9c, 0xc9, 0x03, 0x18, 0xf9, 0x6a, 0x00, 0x9d, 0x99, 0xb1, 0xd1, 0x31, + 0xa0, 0x6e, 0xdf, 0x51, 0x8f, 0x3a, 0x9a, 0xc2, 0x6c, 0xe5, 0x14, 0xde, 0x80, 0x8d, 0xa4, 0x0d, + 0xd8, 0x27, 0x58, 0xa1, 0x1d, 0x2d, 0xbb, 0xca, 0x22, 0x2f, 0x74, 0xc3, 0x93, 0xe2, 0x47, 0x70, + 0x29, 0x04, 0x3a, 0x8e, 0x91, 0x15, 0x37, 0x61, 0x96, 0xdd, 0xf1, 0xb1, 0x70, 0xb7, 0x61, 0x99, + 0x2f, 0xec, 0x58, 0x66, 0xfe, 0x37, 0x52, 0xb0, 0xc8, 0x39, 0x31, 0xb4, 0x0b, 0x53, 0xb6, 0xa1, + 0xf6, 0xec, 0x17, 0xa6, 0x43, 0xb5, 0xed, 0x6e, 0xc2, 0x01, 0x17, 0x1b, 0x14, 0x96, 0xfc, 0xdc, + 0x9d, 0x90, 0x7d, 0x6c, 0x54, 0x86, 0x1c, 0x39, 0xfd, 0xb0, 0xb1, 0xe5, 0xd1, 0x21, 0x63, 0x3e, + 0x15, 0x8a, 0x29, 0xbe, 0x03, 0xf3, 0x41, 0x0e, 0xe8, 0x26, 0xcc, 0x78, 0x1c, 0x14, 0xbd, 0x4d, + 0xd7, 0x0a, 0xde, 0x50, 0xb5, 0x2d, 0xbe, 0x05, 0xb3, 0x2c, 0x31, 0x74, 0x0d, 0xa6, 0xe9, 0xf5, + 0xf5, 0xc1, 0xa7, 0xc8, 0x40, 0xb5, 0xed, 0x5b, 0xa0, 0xaf, 0xc3, 0x52, 0xf0, 0x52, 0x50, 0xc3, + 0x73, 0xc7, 0x5f, 0x03, 0xd9, 0x8b, 0xf9, 0xe0, 0x1a, 0x3c, 0x39, 0xa5, 0x3f, 0xc8, 0x42, 0x3e, + 0xac, 0xe2, 0xe8, 0x21, 0x64, 0x8f, 0x3a, 0x66, 0xeb, 0x73, 0x8a, 0xfb, 0x95, 0x38, 0x5b, 0x50, + 0x2c, 0xbb, 0x50, 0x64, 0x74, 0x77, 0x42, 0x26, 0x48, 0x2e, 0x76, 0xd7, 0xec, 0x1b, 0x0e, 0xdd, + 0xbd, 0x78, 0xec, 0x7d, 0x17, 0x6a, 0x80, 0x8d, 0x91, 0xd0, 0x36, 0xcc, 0x10, 0x25, 0x51, 0xba, + 0x66, 0x5b, 0x2b, 0xa4, 0x31, 0x8d, 0x5b, 0xb1, 0x34, 0x4a, 0x18, 0x76, 0xdf, 0x6c, 0x6b, 0x32, + 0xa8, 0xfe, 0xdf, 0xe2, 0x1c, 0xcc, 0x30, 0xb2, 0x89, 0x7d, 0x98, 0x61, 0x98, 0xa1, 0x2b, 0x30, + 0x79, 0x6c, 0x2b, 0xbe, 0xcb, 0x98, 0x96, 0x73, 0xc7, 0x36, 0xb6, 0xfe, 0x37, 0x61, 0x06, 0x4b, + 0xa1, 0x1c, 0x77, 0xd4, 0x13, 0xbb, 0x90, 0x5a, 0x4d, 0xbb, 0x67, 0x84, 0x87, 0x1e, 0xbb, 0x23, + 0xe8, 0x1e, 0x50, 0xf3, 0xa7, 0x10, 0xb8, 0x13, 0xcb, 0xec, 0xf7, 0xb0, 0x90, 0xd3, 0x32, 0xf5, + 0xd5, 0x98, 0xd1, 0x8e, 0x3b, 0x2e, 0xfe, 0x59, 0x0a, 0x60, 0x20, 0x20, 0x7a, 0x08, 0x19, 0xbc, + 0x26, 0xe2, 0xa6, 0xd6, 0x47, 0x58, 0x53, 0x11, 0x2f, 0x0c, 0x63, 0x49, 0xff, 0x22, 0x40, 0x06, + 0x93, 0x09, 0xfb, 0xdb, 0x46, 0xb5, 0xb6, 0xb3, 0x57, 0x51, 0x6a, 0xf5, 0xed, 0x8a, 0xf2, 0x89, + 0x5c, 0x6d, 0x56, 0xe4, 0xbc, 0x80, 0xae, 0xc1, 0x15, 0x76, 0x5c, 0xae, 0x94, 0xb6, 0x2b, 0xb2, + 0x52, 0xaf, 0xed, 0x3d, 0xcb, 0xa7, 0x90, 0x08, 0xcb, 0xfb, 0x87, 0x7b, 0xcd, 0x6a, 0x74, 0x2e, + 0x8d, 0xae, 0x43, 0x81, 0x99, 0xa3, 0x34, 0x28, 0xd9, 0x8c, 0x4b, 0x96, 0x99, 0x25, 0x7f, 0xd2, + 0xc9, 0x2c, 0x92, 0xe0, 0x2a, 0xcb, 0x33, 0x88, 0x9b, 0xc3, 0xee, 0xdb, 0xf5, 0xf0, 0x2c, 0x4c, + 0x80, 0xc2, 0x24, 0x06, 0x29, 0xcf, 0xf9, 0x37, 0x00, 0xdf, 0xf0, 0x4f, 0x60, 0x2e, 0xe0, 0xc6, + 0xdc, 0xa0, 0x94, 0xda, 0xdd, 0xb6, 0x72, 0x74, 0xe6, 0xe0, 0x40, 0x4d, 0x58, 0x4f, 0xcb, 0x73, + 0xde, 0x68, 0xd9, 0x1d, 0x74, 0xcf, 0xb2, 0xa3, 0x77, 0x75, 0x87, 0xc2, 0xa4, 0x30, 0x0c, 0xe0, + 0x21, 0x0c, 0x20, 0xfd, 0x53, 0x0a, 0x72, 0xf4, 0x42, 0xdc, 0x66, 0x1c, 0x69, 0x80, 0xa4, 0x37, + 0x4a, 0x48, 0x06, 0x34, 0x32, 0x15, 0xd4, 0x48, 0xb4, 0x0b, 0xf3, 0xac, 0xb7, 0x79, 0xe9, 0x85, + 0xc2, 0x6b, 0xc1, 0x73, 0x66, 0x8d, 0xc8, 0x4b, 0x1a, 0x00, 0xcf, 0x9d, 0xb2, 0x63, 0xa8, 0x0c, + 0xf3, 0x21, 0x87, 0x95, 0x19, 0xee, 0xb0, 0xe6, 0x5a, 0x01, 0x6b, 0x58, 0x82, 0x45, 0xcf, 0xd7, + 0x74, 0x34, 0xc5, 0xa1, 0xbe, 0x88, 0x3a, 0xd4, 0x7c, 0xc4, 0x47, 0xa1, 0x01, 0xb0, 0x37, 0x26, + 0x3e, 0x02, 0x14, 0x95, 0x75, 0x2c, 0x53, 0xdd, 0x87, 0x45, 0x8e, 0x17, 0x44, 0x45, 0x98, 0xc6, + 0x47, 0x65, 0xeb, 0x8e, 0x46, 0x83, 0xec, 0xa8, 0x44, 0x03, 0x10, 0x17, 0xbe, 0x67, 0x69, 0xc7, + 0x9a, 0x65, 0x69, 0x6d, 0xac, 0x93, 0x5c, 0x78, 0x1f, 0x44, 0xfa, 0x9e, 0x00, 0x53, 0xde, 0x38, + 0xda, 0x84, 0x29, 0x5b, 0x3b, 0x21, 0x1e, 0x9a, 0xf0, 0x5a, 0x09, 0xe3, 0x16, 0x1b, 0x14, 0x80, + 0xa6, 0x23, 0x1e, 0xbc, 0x9b, 0x8e, 0x04, 0xa6, 0xc6, 0x5a, 0xfc, 0x5f, 0x08, 0xb0, 0xb8, 0xad, + 0x75, 0xb4, 0x70, 0x20, 0x97, 0x64, 0xd6, 0xd9, 0xd8, 0x27, 0x15, 0x8c, 0x7d, 0x38, 0xa4, 0x12, + 0x62, 0x9f, 0xf3, 0xf8, 0x69, 0x37, 0x77, 0x0c, 0x72, 0x23, 0x3e, 0x45, 0xfa, 0xf7, 0x34, 0xac, + 0xb8, 0x77, 0xc1, 0x32, 0x3b, 0x1d, 0xcd, 0x3a, 0xe8, 0x1f, 0x75, 0x74, 0xfb, 0xc5, 0x18, 0x8b, + 0xbb, 0x02, 0x93, 0x86, 0xd9, 0x66, 0x94, 0x27, 0xe7, 0xfe, 0xac, 0xb6, 0x51, 0x05, 0x16, 0xc2, + 0x91, 0xe8, 0x19, 0xb5, 0xfc, 0xf1, 0x71, 0x68, 0xfe, 0x34, 0xec, 0xb6, 0x44, 0x98, 0x72, 0x63, + 0x68, 0xd3, 0xe8, 0x9c, 0x61, 0x8d, 0x99, 0x92, 0xfd, 0xdf, 0x48, 0x0e, 0x07, 0x95, 0x5f, 0xf5, + 0x63, 0xaa, 0xc4, 0x15, 0x25, 0xc5, 0x97, 0x9f, 0x45, 0x34, 0x3e, 0x87, 0x49, 0x7f, 0x30, 0x22, + 0xe9, 0xa1, 0x96, 0xe0, 0x5c, 0xd1, 0xd6, 0xf9, 0xd5, 0xf7, 0x6f, 0x05, 0xb8, 0x19, 0xbb, 0x04, + 0x1a, 0x67, 0xb4, 0xe1, 0x52, 0x8f, 0x4c, 0xf8, 0x9b, 0x40, 0xb4, 0xec, 0xc3, 0xa1, 0x9b, 0x40, + 0x6b, 0x01, 0x74, 0x34, 0xb0, 0x0d, 0xf3, 0xbd, 0xc0, 0xa0, 0x58, 0x82, 0x45, 0x0e, 0xd8, 0x58, + 0x8b, 0xf9, 0x89, 0x00, 0xab, 0x03, 0x51, 0x0e, 0x8d, 0xde, 0xc5, 0x5d, 0xdf, 0xe6, 0xe0, 0x6e, + 0x11, 0x93, 0xff, 0x5e, 0x74, 0xed, 0x7c, 0x86, 0xaf, 0x4b, 0x83, 0x6f, 0xc1, 0x5a, 0x02, 0x6b, + 0xaa, 0xce, 0xbf, 0x97, 0x83, 0xb5, 0xa7, 0x6a, 0x47, 0x6f, 0xfb, 0xd1, 0x23, 0xa7, 0x6a, 0x92, + 0xbc, 0x25, 0xad, 0x88, 0x06, 0x10, 0xab, 0xf5, 0xd0, 0xd7, 0xda, 0x61, 0xf4, 0x47, 0x70, 0x87, + 0x17, 0x98, 0xa7, 0x3e, 0xe3, 0xe4, 0xa9, 0x1f, 0x8c, 0x2e, 0x6b, 0x52, 0xd6, 0x7a, 0x18, 0x36, + 0x30, 0xef, 0x8f, 0x4e, 0x37, 0xc1, 0xc6, 0x38, 0xdc, 0xb4, 0x90, 0xd8, 0x99, 0x47, 0xa3, 0x73, + 0x18, 0x33, 0x49, 0x3c, 0xb7, 0xed, 0xf8, 0xf9, 0x4f, 0x33, 0x7f, 0x90, 0x03, 0x29, 0x69, 0x5f, + 0xa9, 0xfd, 0x93, 0x61, 0xba, 0x65, 0x1a, 0xc7, 0xba, 0xd5, 0xd5, 0xda, 0x34, 0x5d, 0x7a, 0x77, + 0x94, 0x63, 0xa1, 0xc6, 0x6f, 0xcb, 0xc3, 0x95, 0x07, 0x64, 0x50, 0x01, 0x26, 0xbb, 0x9a, 0x6d, + 0xab, 0x27, 0x9e, 0x58, 0xde, 0x4f, 0xf1, 0x3f, 0x33, 0x30, 0xed, 0xa3, 0x20, 0x23, 0xa2, 0x7d, + 0xc4, 0xf4, 0xee, 0xbc, 0x8a, 0x00, 0xaf, 0xae, 0x88, 0xa9, 0x57, 0x50, 0xc4, 0x76, 0x40, 0x11, + 0x89, 0x2a, 0x6f, 0xbf, 0x92, 0xd8, 0x49, 0x3a, 0xf9, 0x1d, 0xae, 0xf2, 0x10, 0xb5, 0x7f, 0xf2, + 0x4a, 0xdc, 0x7e, 0xde, 0xf4, 0xe8, 0x62, 0x74, 0xe1, 0x17, 0x01, 0xed, 0xe9, 0x36, 0xcd, 0x9e, + 0x7d, 0xcf, 0xe0, 0x26, 0xcb, 0xea, 0x4b, 0x45, 0x33, 0x1c, 0x4b, 0xa7, 0x19, 0x53, 0x56, 0x86, + 0xae, 0xfa, 0xb2, 0x42, 0x46, 0xdc, 0xac, 0xca, 0x76, 0x54, 0xcb, 0xd1, 0x8d, 0x13, 0xc5, 0x31, + 0x3f, 0xd7, 0xfc, 0xee, 0x81, 0x37, 0xda, 0x74, 0x07, 0xa5, 0x7f, 0x4b, 0xc1, 0x62, 0x80, 0x3c, + 0x55, 0xad, 0x0f, 0x61, 0x72, 0x40, 0x3b, 0x90, 0x49, 0x71, 0xa0, 0x8b, 0xe4, 0xf4, 0x3d, 0x0c, + 0x74, 0x03, 0xc0, 0xd0, 0x5e, 0x3a, 0x01, 0xbe, 0xd3, 0xee, 0x08, 0xe6, 0x29, 0xfe, 0x9a, 0xe0, + 0x17, 0x5b, 0x1c, 0xd5, 0xe9, 0xe3, 0xc4, 0x9e, 0x7a, 0x49, 0xad, 0xad, 0x50, 0x37, 0x4f, 0xf8, + 0x4e, 0xcb, 0x79, 0x7f, 0xa6, 0x86, 0x1d, 0xbe, 0x8d, 0x76, 0xfc, 0xc2, 0x7c, 0xcb, 0x34, 0xda, + 0xba, 0x33, 0x28, 0xcc, 0x5f, 0x89, 0xe4, 0x68, 0x64, 0x9a, 0x5c, 0x91, 0x4b, 0xa7, 0xc1, 0x51, + 0xf1, 0x0b, 0xc8, 0x92, 0xe3, 0x18, 0xb1, 0x5e, 0x83, 0x1e, 0x41, 0xce, 0xc6, 0x12, 0x87, 0x6b, + 0x53, 0xbc, 0x3d, 0x61, 0x57, 0x28, 0x53, 0x3c, 0xe9, 0xeb, 0x20, 0x0e, 0x62, 0x83, 0x1d, 0xcd, + 0x19, 0x3d, 0x02, 0xda, 0x74, 0xd7, 0x20, 0xfd, 0x6e, 0x0a, 0xae, 0x71, 0x09, 0x8c, 0x57, 0x79, + 0x42, 0xbb, 0xa1, 0x95, 0xbc, 0x1d, 0x0d, 0x9a, 0x22, 0xc4, 0xb9, 0x2b, 0x12, 0x7f, 0xe5, 0x7c, + 0x87, 0x59, 0x1e, 0xfb, 0x30, 0x23, 0xe7, 0x48, 0x76, 0xe6, 0x7b, 0x69, 0xb8, 0x31, 0x10, 0x7e, + 0xdf, 0x6c, 0xeb, 0xc7, 0x67, 0x63, 0xc4, 0x97, 0x1f, 0x87, 0x73, 0xbf, 0x8d, 0xe8, 0x8e, 0x70, + 0x88, 0x26, 0x45, 0x0f, 0x9f, 0x73, 0x0d, 0x60, 0x3a, 0x18, 0xa3, 0x25, 0x53, 0xe7, 0x9b, 0x0f, + 0x9e, 0xb1, 0xfb, 0x99, 0xfb, 0x6c, 0x72, 0x0a, 0xb7, 0xd9, 0x24, 0x35, 0xb8, 0x22, 0x72, 0x89, + 0x08, 0xd8, 0x9f, 0xa4, 0x00, 0xed, 0x68, 0x8e, 0x5f, 0x5a, 0xa2, 0x27, 0x14, 0xe3, 0xe3, 0x84, + 0x57, 0xf0, 0x71, 0xdf, 0x0c, 0xf8, 0x38, 0x72, 0xa4, 0x77, 0x99, 0xbe, 0x68, 0x88, 0x75, 0xa2, + 0x27, 0x8b, 0x29, 0xe7, 0x90, 0x1c, 0x79, 0xb4, 0x72, 0xce, 0xf9, 0x3c, 0x89, 0xf4, 0xcf, 0x02, + 0x2c, 0x06, 0x84, 0xa6, 0xea, 0x7e, 0x1f, 0x90, 0x7a, 0xaa, 0xea, 0x1d, 0x7c, 0xc9, 0xbc, 0x72, + 0x19, 0x2d, 0x9f, 0x2d, 0xf8, 0x33, 0x1e, 0x1a, 0x7a, 0x02, 0x8b, 0x5d, 0xf5, 0xa5, 0xde, 0xed, + 0x77, 0x15, 0xba, 0xcf, 0xb6, 0xfe, 0x6d, 0xaf, 0xd0, 0x7e, 0x2d, 0xd2, 0x1e, 0xab, 0x1a, 0xce, + 0xfb, 0xef, 0x92, 0xfe, 0xd8, 0x02, 0xc5, 0xa3, 0x9a, 0xae, 0x7f, 0x5b, 0x43, 0x07, 0xb0, 0xd8, + 0xd5, 0x8d, 0x08, 0xb1, 0xf4, 0x50, 0x62, 0x9e, 0xc3, 0x26, 0xc8, 0x03, 0x8a, 0x92, 0xc4, 0x26, + 0x89, 0x74, 0xb9, 0xe1, 0x3e, 0x72, 0x87, 0x4d, 0xae, 0x22, 0x30, 0x74, 0x5b, 0x76, 0xb8, 0xbd, + 0xe4, 0x5b, 0x51, 0x9d, 0xa3, 0x8d, 0xd5, 0xd8, 0xb6, 0xf2, 0xef, 0x64, 0x58, 0x73, 0x1b, 0x81, + 0x46, 0x1f, 0x42, 0xda, 0xea, 0xb5, 0xa8, 0xad, 0x7d, 0x63, 0x04, 0xfa, 0x45, 0xf9, 0x60, 0x6b, + 0x77, 0x42, 0x76, 0xb1, 0xc4, 0x3f, 0x4d, 0x43, 0x5a, 0x3e, 0xd8, 0x42, 0x8f, 0x02, 0x0d, 0xd4, + 0x7b, 0x23, 0x52, 0x61, 0xfb, 0xa7, 0xff, 0x9a, 0xe2, 0x35, 0x50, 0x0b, 0xb0, 0xb4, 0x25, 0x57, + 0x4a, 0xcd, 0x8a, 0xb2, 0x5d, 0xd9, 0xab, 0x34, 0x2b, 0x0a, 0xe9, 0x01, 0xe7, 0x05, 0x74, 0x1d, + 0x0a, 0x07, 0x87, 0xe5, 0xbd, 0x6a, 0x63, 0x57, 0x39, 0xac, 0x79, 0x7f, 0xd1, 0xd9, 0x14, 0xca, + 0xc3, 0xec, 0x5e, 0xb5, 0xd1, 0xa4, 0x03, 0x8d, 0x7c, 0xda, 0x1d, 0xd9, 0xa9, 0x34, 0x95, 0xad, + 0xd2, 0x41, 0x69, 0xab, 0xda, 0x7c, 0x96, 0xcf, 0x20, 0x11, 0x96, 0x83, 0xb4, 0x1b, 0xb5, 0xd2, + 0x41, 0x63, 0xb7, 0xde, 0xcc, 0x67, 0x11, 0x82, 0x79, 0x8c, 0xef, 0x0d, 0x35, 0xf2, 0x39, 0x97, + 0xc2, 0xd6, 0x5e, 0xbd, 0xe6, 0xcb, 0x30, 0x89, 0x96, 0x20, 0xef, 0x71, 0x96, 0x2b, 0xa5, 0x6d, + 0x5c, 0x00, 0x9f, 0x42, 0x0b, 0x30, 0x57, 0xf9, 0xd6, 0x41, 0xa9, 0xb6, 0xed, 0x01, 0x4e, 0xa3, + 0x55, 0xb8, 0xce, 0x8a, 0xa3, 0x50, 0xac, 0xca, 0x36, 0x2e, 0x62, 0x37, 0xf2, 0x80, 0xae, 0x42, + 0x9e, 0xb6, 0xb7, 0xb7, 0xea, 0xb5, 0xed, 0x6a, 0xb3, 0x5a, 0xaf, 0xe5, 0x67, 0x48, 0xc5, 0x7b, + 0x11, 0xc0, 0x95, 0x9c, 0x12, 0x9b, 0x1d, 0x5e, 0x06, 0x9f, 0x23, 0x20, 0xcb, 0x30, 0xb7, 0x5f, + 0xdf, 0xae, 0x3e, 0x7e, 0xe6, 0xa1, 0xce, 0x93, 0xf2, 0xb8, 0xd7, 0xf9, 0xf9, 0x49, 0x0a, 0x2e, + 0x93, 0xd6, 0x8f, 0xd7, 0x68, 0xf2, 0x6c, 0xd8, 0x3a, 0xe4, 0x49, 0xdd, 0x58, 0x09, 0x3b, 0x9b, + 0x79, 0x32, 0xfe, 0xd4, 0x73, 0x39, 0x5e, 0x53, 0x39, 0xc5, 0x34, 0x95, 0xab, 0xe1, 0x6a, 0xc6, + 0xdd, 0x60, 0xf7, 0x31, 0xc4, 0x2d, 0xc9, 0xfd, 0xec, 0x73, 0xd2, 0xed, 0xfb, 0xc9, 0xd4, 0x12, + 0x8c, 0xe0, 0xb9, 0x1c, 0xcc, 0x39, 0xad, 0xdf, 0x63, 0x58, 0x0e, 0xcb, 0x4b, 0x15, 0xfd, 0x5e, + 0xa4, 0xed, 0xe8, 0x9b, 0x63, 0x1f, 0xd6, 0x87, 0x90, 0xbe, 0x9f, 0x82, 0x29, 0x6f, 0xd8, 0x8d, + 0x51, 0x5d, 0x7b, 0x15, 0xe8, 0x38, 0x4c, 0xbb, 0x23, 0x7e, 0x03, 0x83, 0x6d, 0x18, 0xa6, 0xc2, + 0x0d, 0x43, 0xee, 0x39, 0xa7, 0xb9, 0xe7, 0xfc, 0x0d, 0x98, 0x6b, 0xb9, 0xe2, 0xeb, 0xa6, 0xa1, + 0x38, 0x7a, 0xd7, 0x6b, 0x28, 0x44, 0x9f, 0x23, 0x34, 0xbd, 0x67, 0x46, 0xf2, 0xac, 0x87, 0xe0, + 0x0e, 0xa1, 0x55, 0x98, 0xc5, 0xcf, 0x13, 0x14, 0xc7, 0x54, 0xfa, 0xb6, 0x56, 0xc8, 0xe2, 0xf2, + 0x2a, 0xe0, 0xb1, 0xa6, 0x79, 0x68, 0x6b, 0xe8, 0x01, 0x2c, 0xe0, 0x66, 0x98, 0xc2, 0xca, 0x9c, + 0x73, 0xa5, 0xa1, 0xa1, 0x2f, 0x9e, 0x6d, 0xf8, 0xd2, 0x4b, 0x7f, 0x2d, 0xc0, 0x65, 0x52, 0x66, + 0x0e, 0xdf, 0xdf, 0x61, 0x9d, 0x52, 0xf6, 0x8a, 0x86, 0xdc, 0x2a, 0x97, 0xe0, 0xeb, 0xaa, 0xb2, + 0x15, 0x60, 0x39, 0xcc, 0x8f, 0x96, 0xd6, 0x7e, 0x9c, 0x82, 0x25, 0x37, 0x20, 0xf7, 0x26, 0x2e, + 0x3a, 0x67, 0x1a, 0xe3, 0xe8, 0x43, 0x9b, 0x99, 0x89, 0x6c, 0xe6, 0x6e, 0xb8, 0x70, 0xf5, 0x26, + 0x9b, 0x52, 0x84, 0x57, 0xf0, 0xba, 0xf6, 0xf2, 0x47, 0x02, 0x5c, 0x0e, 0xf1, 0xa3, 0x0a, 0xf6, + 0x51, 0x38, 0x0d, 0xbc, 0x15, 0x23, 0xdf, 0x2b, 0x25, 0x82, 0xef, 0x79, 0x09, 0xd8, 0x78, 0x7a, + 0xfc, 0x77, 0x29, 0x36, 0xd4, 0xc7, 0x2f, 0x8a, 0xda, 0x63, 0x84, 0xfa, 0xe7, 0x7b, 0xb8, 0xf3, + 0x71, 0xd8, 0x42, 0x73, 0x12, 0x05, 0x8e, 0x48, 0x49, 0x96, 0x9a, 0xdb, 0x81, 0xc9, 0x8c, 0xdb, + 0x81, 0x39, 0xd7, 0x0d, 0xf8, 0x0e, 0x1b, 0xb7, 0x07, 0xc5, 0xa7, 0x37, 0x61, 0xc4, 0x2e, 0xed, + 0xfb, 0x70, 0x05, 0xe7, 0x7c, 0xfe, 0x9b, 0x39, 0xef, 0x99, 0x0e, 0xb1, 0xa1, 0x53, 0xf2, 0x65, + 0x77, 0xda, 0x7f, 0x05, 0x46, 0x3b, 0x93, 0x6d, 0xe9, 0xa7, 0x19, 0x58, 0x76, 0x73, 0xc2, 0x86, + 0xa3, 0x9e, 0x8c, 0xd3, 0xb3, 0xfb, 0x85, 0x68, 0x0b, 0x24, 0x94, 0xbf, 0xf1, 0xa9, 0x8e, 0xd2, + 0xf9, 0x40, 0x45, 0x58, 0xb4, 0x1d, 0xf5, 0x04, 0x9b, 0x03, 0xd5, 0x3a, 0xd1, 0x1c, 0xa5, 0xa7, + 0x3a, 0x2f, 0xa8, 0xae, 0x2f, 0xd0, 0xa9, 0x26, 0x9e, 0x39, 0x50, 0x9d, 0x17, 0x17, 0x74, 0x90, + 0xe8, 0x9b, 0x61, 0xa3, 0xf0, 0xd6, 0x90, 0xb5, 0x24, 0xdc, 0xad, 0x6f, 0xc5, 0xb4, 0xc9, 0xde, + 0x19, 0x42, 0x72, 0x78, 0x7b, 0xec, 0xfc, 0x6d, 0xa1, 0x9f, 0x71, 0x87, 0xed, 0x2a, 0x5c, 0x89, + 0x2c, 0x9e, 0xba, 0x90, 0x13, 0x28, 0xb8, 0x53, 0x87, 0x86, 0x3d, 0xe6, 0x75, 0x8c, 0xb9, 0x31, + 0xa9, 0x98, 0x1b, 0x23, 0x5d, 0x83, 0xab, 0x1c, 0x46, 0x54, 0x8a, 0x3f, 0xcf, 0x12, 0x31, 0xc6, + 0x6f, 0xf6, 0x7e, 0x1a, 0xa7, 0x15, 0xef, 0xb2, 0xc7, 0xce, 0xed, 0x8b, 0xbe, 0x0e, 0xbd, 0xb8, + 0x09, 0x33, 0x2c, 0x1c, 0x75, 0x83, 0xce, 0x10, 0xc5, 0xc9, 0x9e, 0xab, 0x07, 0x9d, 0x0b, 0xf5, + 0xa0, 0xf7, 0x06, 0x4a, 0x35, 0x19, 0x8c, 0x85, 0x63, 0xb7, 0x22, 0x41, 0xad, 0x9e, 0x47, 0xd4, + 0x6a, 0x2a, 0xd8, 0xd8, 0x8e, 0x25, 0xfa, 0xff, 0x40, 0xb1, 0xe8, 0xa5, 0xe6, 0x76, 0x9c, 0xa5, + 0xe7, 0x20, 0x92, 0x1b, 0x3f, 0x7e, 0x0f, 0x38, 0x74, 0x8d, 0x52, 0xe1, 0x6b, 0x24, 0xdd, 0x80, + 0x6b, 0x5c, 0xda, 0x94, 0xf5, 0x0f, 0x04, 0x22, 0x98, 0x5f, 0xd9, 0x6c, 0x38, 0xaa, 0x63, 0x8f, + 0xca, 0x9a, 0x4e, 0xb2, 0xac, 0xc9, 0x10, 0xbe, 0xc1, 0x63, 0xaa, 0x84, 0xf4, 0x9b, 0x02, 0xd9, + 0x87, 0xb0, 0x2c, 0xd4, 0xdb, 0xbe, 0x09, 0xd9, 0x3e, 0xee, 0x41, 0x91, 0xa8, 0x6b, 0x31, 0xa8, + 0x04, 0x87, 0xee, 0x94, 0x4c, 0x20, 0x2e, 0xac, 0x1c, 0x2e, 0xfd, 0x58, 0x80, 0x19, 0x86, 0x3e, + 0xba, 0x0e, 0xd3, 0x7e, 0x09, 0xc9, 0x4b, 0x90, 0xfc, 0x01, 0xf7, 0xf8, 0x1d, 0xd3, 0x51, 0x3b, + 0xf4, 0x6d, 0x17, 0xf9, 0xe1, 0xe6, 0xb4, 0x7d, 0x5b, 0x23, 0xe1, 0x70, 0x5a, 0xc6, 0x7f, 0xa3, + 0x7b, 0x90, 0xe9, 0x1b, 0xba, 0x83, 0xd5, 0x7e, 0x3e, 0xac, 0xcf, 0x98, 0x55, 0xf1, 0xd0, 0xd0, + 0x1d, 0x19, 0x43, 0x49, 0x77, 0x21, 0xe3, 0xfe, 0x0a, 0x96, 0x32, 0xa6, 0x21, 0x5b, 0x7e, 0xd6, + 0xac, 0x34, 0xf2, 0x02, 0x02, 0xc8, 0x55, 0x49, 0xe2, 0x9f, 0x92, 0xf6, 0xbc, 0xd7, 0xe8, 0xfe, + 0x22, 0x5c, 0x13, 0xa0, 0x1e, 0x19, 0xa6, 0xd5, 0x55, 0x3b, 0x58, 0xe6, 0x29, 0xd9, 0xff, 0x1d, + 0xdf, 0xda, 0x23, 0x45, 0xc9, 0xeb, 0xfe, 0x89, 0xf0, 0x0a, 0x4f, 0x9f, 0x91, 0xbb, 0x15, 0x57, + 0x72, 0x2a, 0x71, 0x4b, 0x4e, 0x37, 0x02, 0x5e, 0x76, 0x48, 0xb1, 0xe9, 0x6f, 0x52, 0x70, 0x99, + 0x0b, 0x87, 0xde, 0x63, 0xcb, 0x4c, 0x6b, 0x89, 0x34, 0xd9, 0x02, 0xd3, 0x4f, 0x05, 0x52, 0x60, + 0xda, 0x0c, 0x14, 0x98, 0xee, 0x0c, 0xc5, 0x67, 0x4b, 0x4b, 0x3f, 0x12, 0x62, 0x4a, 0x4b, 0x8d, + 0x66, 0x69, 0xa7, 0xa2, 0x1c, 0xd6, 0xc8, 0xbf, 0x7e, 0x69, 0x69, 0x09, 0xf2, 0x83, 0x82, 0x8b, + 0xd2, 0x68, 0x96, 0xf0, 0xa7, 0x05, 0x91, 0xb2, 0x4e, 0x9a, 0x5b, 0xb4, 0xc9, 0x0c, 0xaf, 0xcf, + 0x64, 0xbd, 0xfa, 0x0c, 0xa2, 0xd8, 0xfb, 0xf5, 0xc3, 0x5a, 0x53, 0xc1, 0x1f, 0x2e, 0xe4, 0x73, + 0x7e, 0x7d, 0x66, 0x09, 0x10, 0x3d, 0x2d, 0xf6, 0xfb, 0x9b, 0x3f, 0x14, 0x60, 0x31, 0x30, 0x4c, + 0x0f, 0x8f, 0x79, 0x5c, 0x22, 0x04, 0x1e, 0x97, 0x3c, 0x80, 0x25, 0x37, 0x63, 0x24, 0x9a, 0x62, + 0x2b, 0x3d, 0xcd, 0xc2, 0x1d, 0x0d, 0x7a, 0xe7, 0x17, 0xba, 0xea, 0x4b, 0xda, 0xf5, 0x39, 0xd0, + 0x2c, 0x97, 0xf0, 0x05, 0x94, 0x8a, 0xa5, 0x2f, 0xd3, 0x24, 0x2e, 0x19, 0x3b, 0xaf, 0x19, 0x6a, + 0xa3, 0xa2, 0x89, 0x4f, 0x7a, 0x8c, 0xc4, 0x27, 0xc6, 0xc2, 0x65, 0xc6, 0x0a, 0x86, 0xc7, 0xf7, + 0xe9, 0xb5, 0x81, 0xdf, 0x26, 0x91, 0xeb, 0x3d, 0xf6, 0xfe, 0x0e, 0xcd, 0xb4, 0x72, 0x5f, 0x96, + 0x85, 0x1f, 0x5e, 0x54, 0x9e, 0x5c, 0x22, 0xf1, 0xd8, 0x39, 0xf2, 0x23, 0xe9, 0x1e, 0xdc, 0xc6, + 0xcf, 0x93, 0x87, 0x15, 0xba, 0x89, 0x49, 0xfa, 0x65, 0xb8, 0x33, 0x0c, 0x9a, 0xb2, 0xdf, 0xe3, + 0xda, 0x1f, 0xbf, 0x41, 0x19, 0xa2, 0x32, 0xc4, 0x14, 0x11, 0xe6, 0xbf, 0x9e, 0x82, 0xd5, 0x61, + 0x78, 0xe8, 0x11, 0x6b, 0x9a, 0xee, 0x8d, 0xca, 0x8e, 0xb5, 0x52, 0xbf, 0x4d, 0xad, 0x54, 0x25, + 0x60, 0xa5, 0xde, 0x19, 0x87, 0x14, 0x6b, 0xb0, 0x2a, 0x3c, 0x7b, 0xf5, 0x36, 0xbc, 0x11, 0x2c, + 0x57, 0x33, 0x36, 0x8a, 0x7c, 0xf3, 0xe4, 0xd7, 0xaf, 0x05, 0x6c, 0x60, 0x36, 0x03, 0xd5, 0xde, + 0xdf, 0x4a, 0xc3, 0x2a, 0xfb, 0xd0, 0x7f, 0x87, 0xad, 0xa6, 0x25, 0x7d, 0x23, 0x74, 0x17, 0x16, + 0xc2, 0x95, 0x22, 0xef, 0x61, 0xfb, 0xa5, 0x60, 0xa9, 0xc8, 0x4e, 0x7a, 0xc8, 0x36, 0x84, 0x75, + 0x72, 0xfe, 0x17, 0xad, 0x02, 0x7f, 0x6d, 0x64, 0xc2, 0xff, 0x37, 0x0b, 0xc2, 0xe4, 0x7a, 0x76, + 0x60, 0x2d, 0x41, 0x7e, 0xaa, 0x16, 0x65, 0x98, 0x0f, 0x16, 0x46, 0xe9, 0x4d, 0x0d, 0xbd, 0xe6, + 0x0e, 0x22, 0xcf, 0x05, 0xaa, 0xa5, 0x84, 0xdb, 0x3f, 0x08, 0xde, 0x87, 0x2f, 0x01, 0x58, 0xf7, + 0x84, 0xa3, 0x95, 0x57, 0xb2, 0x88, 0x70, 0xd1, 0x15, 0x15, 0x61, 0xda, 0x83, 0xb2, 0xc3, 0x4f, + 0xa9, 0x7d, 0xe6, 0x03, 0x90, 0x68, 0xe1, 0x38, 0x7d, 0xce, 0xc2, 0x71, 0x26, 0x5c, 0x38, 0x26, + 0x6b, 0xfb, 0x7e, 0x0a, 0x56, 0xd9, 0x37, 0xc7, 0xdc, 0xeb, 0x3d, 0xce, 0x42, 0xd7, 0x60, 0x96, + 0x81, 0xf2, 0x6e, 0xfc, 0xcc, 0xa0, 0xee, 0x99, 0x74, 0xdb, 0x87, 0x49, 0xf2, 0x9a, 0x8a, 0xa0, + 0x64, 0x2b, 0xd6, 0x61, 0x2d, 0x81, 0x3f, 0xdb, 0xc2, 0xfe, 0x6e, 0x0a, 0x7f, 0xd1, 0xfa, 0xbf, + 0xb7, 0x63, 0xf1, 0x85, 0xc7, 0x44, 0x31, 0x5e, 0xeb, 0x76, 0xe9, 0xb0, 0x12, 0xc7, 0xfc, 0x82, + 0x15, 0x70, 0xe3, 0xbf, 0x05, 0x98, 0xaa, 0xb6, 0x35, 0xc3, 0x21, 0x41, 0xc1, 0x5c, 0xe0, 0xdb, + 0x66, 0x74, 0x3d, 0xe6, 0x93, 0x67, 0xbc, 0x05, 0xe2, 0x8d, 0xc4, 0x0f, 0xa2, 0xa5, 0x09, 0x74, + 0xcc, 0x7c, 0x97, 0x1d, 0x78, 0x46, 0xf0, 0x95, 0x08, 0x26, 0xc7, 0x57, 0x8b, 0xb7, 0x87, 0x40, + 0xf9, 0x7c, 0xde, 0x87, 0x2c, 0xfe, 0x44, 0x15, 0x2d, 0xf9, 0x9f, 0xc9, 0x32, 0x5f, 0xb0, 0x8a, + 0x97, 0x43, 0xa3, 0x1e, 0xde, 0xc6, 0x5f, 0x01, 0xc0, 0xc0, 0x07, 0xa2, 0x27, 0x30, 0xcb, 0x9a, + 0x3e, 0x74, 0x2d, 0xe1, 0x13, 0x45, 0xf1, 0x3a, 0x7f, 0xd2, 0x97, 0xe9, 0x09, 0xcc, 0xb2, 0x57, + 0x7e, 0x40, 0x8c, 0xf3, 0xd1, 0xc3, 0x80, 0x18, 0xf7, 0x1b, 0x85, 0x09, 0xd4, 0x81, 0x2b, 0x31, + 0x4f, 0xce, 0xd1, 0x9d, 0xd1, 0x1e, 0xe6, 0x8b, 0x6f, 0x8c, 0xf8, 0x76, 0x5d, 0x9a, 0x40, 0x16, + 0x5c, 0x8d, 0x7d, 0x69, 0x8d, 0xd6, 0x47, 0x7d, 0x07, 0x2e, 0xbe, 0x39, 0x02, 0xa4, 0xcf, 0xb3, + 0x0f, 0x62, 0xfc, 0xa3, 0x45, 0xf4, 0xe6, 0xc8, 0xaf, 0x82, 0xc5, 0xbb, 0xa3, 0xbf, 0x81, 0x94, + 0x26, 0xd0, 0x2e, 0xcc, 0x30, 0x8f, 0xcc, 0x90, 0xc8, 0x7d, 0x79, 0x46, 0x08, 0x5f, 0x4b, 0x78, + 0x95, 0x46, 0x28, 0x31, 0x4f, 0x49, 0x06, 0x94, 0xa2, 0x8f, 0x62, 0x06, 0x94, 0x38, 0x6f, 0x4f, + 0xc2, 0xdb, 0x1f, 0x0a, 0x4c, 0x79, 0xdb, 0xcf, 0x8f, 0x74, 0x79, 0xdb, 0x1f, 0x13, 0xe5, 0x4a, + 0x13, 0xe8, 0x63, 0x98, 0x0f, 0xf6, 0x82, 0xd1, 0x8d, 0xc4, 0x9e, 0xb6, 0xb8, 0x12, 0x37, 0xcd, + 0x92, 0x0c, 0x76, 0x12, 0x07, 0x24, 0xb9, 0x1d, 0xcd, 0x01, 0xc9, 0x98, 0x06, 0xe4, 0x84, 0x6b, + 0x9f, 0x02, 0xfd, 0xb1, 0x81, 0x7d, 0xe2, 0xb5, 0xf5, 0x06, 0xf6, 0x89, 0xdb, 0x54, 0x93, 0x26, + 0x90, 0x0e, 0xcb, 0xfc, 0xf6, 0x0c, 0xba, 0x3d, 0x52, 0xf7, 0x49, 0xbc, 0x33, 0x0c, 0xcc, 0x67, + 0xd5, 0x82, 0x45, 0xce, 0x1b, 0x40, 0x24, 0x25, 0x3e, 0x10, 0x24, 0x4c, 0x6e, 0x8d, 0xf0, 0x88, + 0x50, 0xc2, 0xe9, 0x7b, 0x87, 0x5d, 0x0f, 0xfb, 0x4c, 0x8c, 0xb7, 0x1e, 0xce, 0xc3, 0x38, 0xde, + 0x7a, 0x78, 0xaf, 0xcd, 0x30, 0xb7, 0x8d, 0xff, 0x4a, 0xc3, 0xa5, 0x50, 0x1a, 0x81, 0x7e, 0x55, + 0x80, 0x95, 0xe4, 0xd4, 0x0a, 0xdd, 0x8f, 0x49, 0x41, 0x62, 0xae, 0x71, 0x71, 0x54, 0x70, 0xc6, + 0x94, 0x5c, 0x8d, 0x8d, 0x60, 0xd1, 0xfa, 0xa8, 0x41, 0x3a, 0xa3, 0x3f, 0xc3, 0xc2, 0x61, 0xb2, + 0xf9, 0x7d, 0xb8, 0x1a, 0x1b, 0xe3, 0xa0, 0xf5, 0x51, 0xc3, 0xb0, 0x01, 0xdb, 0xa1, 0x01, 0x93, + 0x7f, 0xe6, 0xfc, 0x58, 0x01, 0xdd, 0x1e, 0x29, 0x90, 0x19, 0x9c, 0x79, 0x72, 0xc8, 0x81, 0xb9, + 0xe1, 0x24, 0x6e, 0xe3, 0x1f, 0xb3, 0x90, 0xc1, 0x65, 0x99, 0x26, 0x5c, 0x0a, 0xb5, 0x7a, 0xd0, + 0x4a, 0x72, 0x03, 0x4c, 0xbc, 0x19, 0x3b, 0xef, 0x9f, 0xdf, 0x73, 0x58, 0x88, 0x34, 0x6f, 0xd0, + 0x2a, 0x8b, 0xc7, 0x6b, 0x20, 0x89, 0x6b, 0x09, 0x10, 0x61, 0xda, 0x41, 0x17, 0xba, 0x3a, 0xac, + 0xbb, 0x10, 0xa4, 0x1d, 0xe7, 0x36, 0x3f, 0x23, 0x55, 0xb0, 0xb0, 0xc3, 0x94, 0x82, 0x72, 0x71, + 0x5d, 0xe5, 0xad, 0x44, 0x18, 0x9f, 0xc3, 0xa7, 0x7e, 0xf9, 0x8d, 0x29, 0x6e, 0xa3, 0x80, 0x70, + 0xdc, 0x22, 0xbc, 0x28, 0x25, 0x81, 0xf8, 0xe4, 0x3f, 0x81, 0x7c, 0xb8, 0x0e, 0x83, 0x6e, 0x0e, + 0x29, 0x0b, 0x89, 0xab, 0xf1, 0x00, 0xe1, 0x9d, 0x09, 0x5b, 0x82, 0xb0, 0x54, 0x3c, 0xf5, 0xbf, + 0x95, 0x08, 0xc3, 0x7a, 0x5f, 0xa6, 0x02, 0x39, 0xf0, 0xbe, 0xd1, 0x6a, 0xe5, 0xc0, 0xfb, 0x72, + 0x4a, 0x96, 0xd2, 0xc4, 0xe6, 0x43, 0x00, 0xb5, 0xd3, 0x7b, 0xa1, 0x2a, 0x9a, 0xd1, 0xef, 0xa2, + 0xeb, 0x91, 0xa4, 0xb0, 0x62, 0xf4, 0xbb, 0xf5, 0x9e, 0x9b, 0x0b, 0xda, 0x85, 0x3f, 0x9e, 0xc2, + 0x99, 0xdf, 0x34, 0x46, 0x70, 0x27, 0x36, 0xf7, 0x20, 0x3f, 0xc0, 0x56, 0x70, 0x58, 0x8f, 0xd6, + 0xb8, 0x34, 0xf0, 0x9b, 0xcd, 0x10, 0xa1, 0x79, 0x9f, 0x10, 0x9e, 0xdd, 0xfc, 0x08, 0xa0, 0x65, + 0xeb, 0x0a, 0xc9, 0x2b, 0xd0, 0x8d, 0x08, 0x9d, 0xc7, 0xba, 0xd6, 0x69, 0x7b, 0x34, 0xfe, 0x88, + 0x0a, 0xd3, 0xb2, 0x75, 0x92, 0x7d, 0x6c, 0x7e, 0x03, 0x66, 0x88, 0x30, 0xc7, 0x2e, 0xdc, 0x30, + 0x7c, 0x2a, 0x03, 0x59, 0x3d, 0x9e, 0xd9, 0xac, 0xc0, 0x1c, 0x21, 0x40, 0x0b, 0xfa, 0xe8, 0x66, + 0x84, 0xc4, 0x3e, 0x99, 0x09, 0x11, 0x99, 0xc5, 0x68, 0x74, 0x6e, 0xb3, 0x0c, 0xb3, 0x1e, 0x19, + 0xe7, 0x85, 0xd9, 0x46, 0x2b, 0x1c, 0x2a, 0xee, 0x44, 0x88, 0xc8, 0x0c, 0x25, 0xe2, 0x4e, 0x0d, + 0x44, 0xf1, 0xfe, 0x1f, 0xa1, 0xa8, 0x28, 0xb4, 0x86, 0xc5, 0x15, 0x85, 0xce, 0x95, 0xb3, 0xcf, + 0xd3, 0x2d, 0x5b, 0x3f, 0xca, 0x61, 0xa4, 0xaf, 0xfe, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x88, + 0xc2, 0x11, 0x67, 0xf4, 0x4a, 0x00, 0x00, +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// IdentityClient is the client API for Identity service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type IdentityClient interface { + GetPluginInfo(ctx context.Context, in *GetPluginInfoRequest, opts ...grpc.CallOption) (*GetPluginInfoResponse, error) + GetPluginCapabilities(ctx context.Context, in *GetPluginCapabilitiesRequest, opts ...grpc.CallOption) (*GetPluginCapabilitiesResponse, error) + Probe(ctx context.Context, in *ProbeRequest, opts ...grpc.CallOption) (*ProbeResponse, error) +} + +type identityClient struct { + cc *grpc.ClientConn +} + +func NewIdentityClient(cc *grpc.ClientConn) IdentityClient { + return &identityClient{cc} +} + +func (c *identityClient) GetPluginInfo(ctx context.Context, in *GetPluginInfoRequest, opts ...grpc.CallOption) (*GetPluginInfoResponse, error) { + out := new(GetPluginInfoResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Identity/GetPluginInfo", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *identityClient) GetPluginCapabilities(ctx context.Context, in *GetPluginCapabilitiesRequest, opts ...grpc.CallOption) (*GetPluginCapabilitiesResponse, error) { + out := new(GetPluginCapabilitiesResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Identity/GetPluginCapabilities", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *identityClient) Probe(ctx context.Context, in *ProbeRequest, opts ...grpc.CallOption) (*ProbeResponse, error) { + out := new(ProbeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Identity/Probe", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// IdentityServer is the server API for Identity service. +type IdentityServer interface { + GetPluginInfo(context.Context, *GetPluginInfoRequest) (*GetPluginInfoResponse, error) + GetPluginCapabilities(context.Context, *GetPluginCapabilitiesRequest) (*GetPluginCapabilitiesResponse, error) + Probe(context.Context, *ProbeRequest) (*ProbeResponse, error) +} + +// UnimplementedIdentityServer can be embedded to have forward compatible implementations. +type UnimplementedIdentityServer struct { +} + +func (*UnimplementedIdentityServer) GetPluginInfo(ctx context.Context, req *GetPluginInfoRequest) (*GetPluginInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetPluginInfo not implemented") +} +func (*UnimplementedIdentityServer) GetPluginCapabilities(ctx context.Context, req *GetPluginCapabilitiesRequest) (*GetPluginCapabilitiesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetPluginCapabilities not implemented") +} +func (*UnimplementedIdentityServer) Probe(ctx context.Context, req *ProbeRequest) (*ProbeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Probe not implemented") +} + +func RegisterIdentityServer(s *grpc.Server, srv IdentityServer) { + s.RegisterService(&_Identity_serviceDesc, srv) +} + +func _Identity_GetPluginInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetPluginInfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(IdentityServer).GetPluginInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Identity/GetPluginInfo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(IdentityServer).GetPluginInfo(ctx, req.(*GetPluginInfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Identity_GetPluginCapabilities_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetPluginCapabilitiesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(IdentityServer).GetPluginCapabilities(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Identity/GetPluginCapabilities", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(IdentityServer).GetPluginCapabilities(ctx, req.(*GetPluginCapabilitiesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Identity_Probe_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ProbeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(IdentityServer).Probe(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Identity/Probe", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(IdentityServer).Probe(ctx, req.(*ProbeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _Identity_serviceDesc = grpc.ServiceDesc{ + ServiceName: "csi.v1.Identity", + HandlerType: (*IdentityServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetPluginInfo", + Handler: _Identity_GetPluginInfo_Handler, + }, + { + MethodName: "GetPluginCapabilities", + Handler: _Identity_GetPluginCapabilities_Handler, + }, + { + MethodName: "Probe", + Handler: _Identity_Probe_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "github.com/container-storage-interface/spec/csi.proto", +} + +// ControllerClient is the client API for Controller service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type ControllerClient interface { + CreateVolume(ctx context.Context, in *CreateVolumeRequest, opts ...grpc.CallOption) (*CreateVolumeResponse, error) + DeleteVolume(ctx context.Context, in *DeleteVolumeRequest, opts ...grpc.CallOption) (*DeleteVolumeResponse, error) + ControllerPublishVolume(ctx context.Context, in *ControllerPublishVolumeRequest, opts ...grpc.CallOption) (*ControllerPublishVolumeResponse, error) + ControllerUnpublishVolume(ctx context.Context, in *ControllerUnpublishVolumeRequest, opts ...grpc.CallOption) (*ControllerUnpublishVolumeResponse, error) + ValidateVolumeCapabilities(ctx context.Context, in *ValidateVolumeCapabilitiesRequest, opts ...grpc.CallOption) (*ValidateVolumeCapabilitiesResponse, error) + ListVolumes(ctx context.Context, in *ListVolumesRequest, opts ...grpc.CallOption) (*ListVolumesResponse, error) + GetCapacity(ctx context.Context, in *GetCapacityRequest, opts ...grpc.CallOption) (*GetCapacityResponse, error) + ControllerGetCapabilities(ctx context.Context, in *ControllerGetCapabilitiesRequest, opts ...grpc.CallOption) (*ControllerGetCapabilitiesResponse, error) + CreateSnapshot(ctx context.Context, in *CreateSnapshotRequest, opts ...grpc.CallOption) (*CreateSnapshotResponse, error) + DeleteSnapshot(ctx context.Context, in *DeleteSnapshotRequest, opts ...grpc.CallOption) (*DeleteSnapshotResponse, error) + ListSnapshots(ctx context.Context, in *ListSnapshotsRequest, opts ...grpc.CallOption) (*ListSnapshotsResponse, error) + ControllerExpandVolume(ctx context.Context, in *ControllerExpandVolumeRequest, opts ...grpc.CallOption) (*ControllerExpandVolumeResponse, error) + ControllerGetVolume(ctx context.Context, in *ControllerGetVolumeRequest, opts ...grpc.CallOption) (*ControllerGetVolumeResponse, error) + ControllerModifyVolume(ctx context.Context, in *ControllerModifyVolumeRequest, opts ...grpc.CallOption) (*ControllerModifyVolumeResponse, error) +} + +type controllerClient struct { + cc *grpc.ClientConn +} + +func NewControllerClient(cc *grpc.ClientConn) ControllerClient { + return &controllerClient{cc} +} + +func (c *controllerClient) CreateVolume(ctx context.Context, in *CreateVolumeRequest, opts ...grpc.CallOption) (*CreateVolumeResponse, error) { + out := new(CreateVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/CreateVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controllerClient) DeleteVolume(ctx context.Context, in *DeleteVolumeRequest, opts ...grpc.CallOption) (*DeleteVolumeResponse, error) { + out := new(DeleteVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/DeleteVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controllerClient) ControllerPublishVolume(ctx context.Context, in *ControllerPublishVolumeRequest, opts ...grpc.CallOption) (*ControllerPublishVolumeResponse, error) { + out := new(ControllerPublishVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ControllerPublishVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controllerClient) ControllerUnpublishVolume(ctx context.Context, in *ControllerUnpublishVolumeRequest, opts ...grpc.CallOption) (*ControllerUnpublishVolumeResponse, error) { + out := new(ControllerUnpublishVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ControllerUnpublishVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controllerClient) ValidateVolumeCapabilities(ctx context.Context, in *ValidateVolumeCapabilitiesRequest, opts ...grpc.CallOption) (*ValidateVolumeCapabilitiesResponse, error) { + out := new(ValidateVolumeCapabilitiesResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ValidateVolumeCapabilities", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controllerClient) ListVolumes(ctx context.Context, in *ListVolumesRequest, opts ...grpc.CallOption) (*ListVolumesResponse, error) { + out := new(ListVolumesResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ListVolumes", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controllerClient) GetCapacity(ctx context.Context, in *GetCapacityRequest, opts ...grpc.CallOption) (*GetCapacityResponse, error) { + out := new(GetCapacityResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/GetCapacity", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controllerClient) ControllerGetCapabilities(ctx context.Context, in *ControllerGetCapabilitiesRequest, opts ...grpc.CallOption) (*ControllerGetCapabilitiesResponse, error) { + out := new(ControllerGetCapabilitiesResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ControllerGetCapabilities", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controllerClient) CreateSnapshot(ctx context.Context, in *CreateSnapshotRequest, opts ...grpc.CallOption) (*CreateSnapshotResponse, error) { + out := new(CreateSnapshotResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/CreateSnapshot", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controllerClient) DeleteSnapshot(ctx context.Context, in *DeleteSnapshotRequest, opts ...grpc.CallOption) (*DeleteSnapshotResponse, error) { + out := new(DeleteSnapshotResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/DeleteSnapshot", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controllerClient) ListSnapshots(ctx context.Context, in *ListSnapshotsRequest, opts ...grpc.CallOption) (*ListSnapshotsResponse, error) { + out := new(ListSnapshotsResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ListSnapshots", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controllerClient) ControllerExpandVolume(ctx context.Context, in *ControllerExpandVolumeRequest, opts ...grpc.CallOption) (*ControllerExpandVolumeResponse, error) { + out := new(ControllerExpandVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ControllerExpandVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controllerClient) ControllerGetVolume(ctx context.Context, in *ControllerGetVolumeRequest, opts ...grpc.CallOption) (*ControllerGetVolumeResponse, error) { + out := new(ControllerGetVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ControllerGetVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controllerClient) ControllerModifyVolume(ctx context.Context, in *ControllerModifyVolumeRequest, opts ...grpc.CallOption) (*ControllerModifyVolumeResponse, error) { + out := new(ControllerModifyVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Controller/ControllerModifyVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ControllerServer is the server API for Controller service. +type ControllerServer interface { + CreateVolume(context.Context, *CreateVolumeRequest) (*CreateVolumeResponse, error) + DeleteVolume(context.Context, *DeleteVolumeRequest) (*DeleteVolumeResponse, error) + ControllerPublishVolume(context.Context, *ControllerPublishVolumeRequest) (*ControllerPublishVolumeResponse, error) + ControllerUnpublishVolume(context.Context, *ControllerUnpublishVolumeRequest) (*ControllerUnpublishVolumeResponse, error) + ValidateVolumeCapabilities(context.Context, *ValidateVolumeCapabilitiesRequest) (*ValidateVolumeCapabilitiesResponse, error) + ListVolumes(context.Context, *ListVolumesRequest) (*ListVolumesResponse, error) + GetCapacity(context.Context, *GetCapacityRequest) (*GetCapacityResponse, error) + ControllerGetCapabilities(context.Context, *ControllerGetCapabilitiesRequest) (*ControllerGetCapabilitiesResponse, error) + CreateSnapshot(context.Context, *CreateSnapshotRequest) (*CreateSnapshotResponse, error) + DeleteSnapshot(context.Context, *DeleteSnapshotRequest) (*DeleteSnapshotResponse, error) + ListSnapshots(context.Context, *ListSnapshotsRequest) (*ListSnapshotsResponse, error) + ControllerExpandVolume(context.Context, *ControllerExpandVolumeRequest) (*ControllerExpandVolumeResponse, error) + ControllerGetVolume(context.Context, *ControllerGetVolumeRequest) (*ControllerGetVolumeResponse, error) + ControllerModifyVolume(context.Context, *ControllerModifyVolumeRequest) (*ControllerModifyVolumeResponse, error) +} + +// UnimplementedControllerServer can be embedded to have forward compatible implementations. +type UnimplementedControllerServer struct { +} + +func (*UnimplementedControllerServer) CreateVolume(ctx context.Context, req *CreateVolumeRequest) (*CreateVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CreateVolume not implemented") +} +func (*UnimplementedControllerServer) DeleteVolume(ctx context.Context, req *DeleteVolumeRequest) (*DeleteVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteVolume not implemented") +} +func (*UnimplementedControllerServer) ControllerPublishVolume(ctx context.Context, req *ControllerPublishVolumeRequest) (*ControllerPublishVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ControllerPublishVolume not implemented") +} +func (*UnimplementedControllerServer) ControllerUnpublishVolume(ctx context.Context, req *ControllerUnpublishVolumeRequest) (*ControllerUnpublishVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ControllerUnpublishVolume not implemented") +} +func (*UnimplementedControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *ValidateVolumeCapabilitiesRequest) (*ValidateVolumeCapabilitiesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ValidateVolumeCapabilities not implemented") +} +func (*UnimplementedControllerServer) ListVolumes(ctx context.Context, req *ListVolumesRequest) (*ListVolumesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListVolumes not implemented") +} +func (*UnimplementedControllerServer) GetCapacity(ctx context.Context, req *GetCapacityRequest) (*GetCapacityResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetCapacity not implemented") +} +func (*UnimplementedControllerServer) ControllerGetCapabilities(ctx context.Context, req *ControllerGetCapabilitiesRequest) (*ControllerGetCapabilitiesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ControllerGetCapabilities not implemented") +} +func (*UnimplementedControllerServer) CreateSnapshot(ctx context.Context, req *CreateSnapshotRequest) (*CreateSnapshotResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CreateSnapshot not implemented") +} +func (*UnimplementedControllerServer) DeleteSnapshot(ctx context.Context, req *DeleteSnapshotRequest) (*DeleteSnapshotResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteSnapshot not implemented") +} +func (*UnimplementedControllerServer) ListSnapshots(ctx context.Context, req *ListSnapshotsRequest) (*ListSnapshotsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListSnapshots not implemented") +} +func (*UnimplementedControllerServer) ControllerExpandVolume(ctx context.Context, req *ControllerExpandVolumeRequest) (*ControllerExpandVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ControllerExpandVolume not implemented") +} +func (*UnimplementedControllerServer) ControllerGetVolume(ctx context.Context, req *ControllerGetVolumeRequest) (*ControllerGetVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ControllerGetVolume not implemented") +} +func (*UnimplementedControllerServer) ControllerModifyVolume(ctx context.Context, req *ControllerModifyVolumeRequest) (*ControllerModifyVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ControllerModifyVolume not implemented") +} + +func RegisterControllerServer(s *grpc.Server, srv ControllerServer) { + s.RegisterService(&_Controller_serviceDesc, srv) +} + +func _Controller_CreateVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).CreateVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/CreateVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).CreateVolume(ctx, req.(*CreateVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Controller_DeleteVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).DeleteVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/DeleteVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).DeleteVolume(ctx, req.(*DeleteVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Controller_ControllerPublishVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ControllerPublishVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).ControllerPublishVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/ControllerPublishVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).ControllerPublishVolume(ctx, req.(*ControllerPublishVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Controller_ControllerUnpublishVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ControllerUnpublishVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).ControllerUnpublishVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/ControllerUnpublishVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).ControllerUnpublishVolume(ctx, req.(*ControllerUnpublishVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Controller_ValidateVolumeCapabilities_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ValidateVolumeCapabilitiesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).ValidateVolumeCapabilities(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/ValidateVolumeCapabilities", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).ValidateVolumeCapabilities(ctx, req.(*ValidateVolumeCapabilitiesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Controller_ListVolumes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListVolumesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).ListVolumes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/ListVolumes", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).ListVolumes(ctx, req.(*ListVolumesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Controller_GetCapacity_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetCapacityRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).GetCapacity(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/GetCapacity", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).GetCapacity(ctx, req.(*GetCapacityRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Controller_ControllerGetCapabilities_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ControllerGetCapabilitiesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).ControllerGetCapabilities(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/ControllerGetCapabilities", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).ControllerGetCapabilities(ctx, req.(*ControllerGetCapabilitiesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Controller_CreateSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateSnapshotRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).CreateSnapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/CreateSnapshot", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).CreateSnapshot(ctx, req.(*CreateSnapshotRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Controller_DeleteSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteSnapshotRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).DeleteSnapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/DeleteSnapshot", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).DeleteSnapshot(ctx, req.(*DeleteSnapshotRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Controller_ListSnapshots_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListSnapshotsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).ListSnapshots(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/ListSnapshots", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).ListSnapshots(ctx, req.(*ListSnapshotsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Controller_ControllerExpandVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ControllerExpandVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).ControllerExpandVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/ControllerExpandVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).ControllerExpandVolume(ctx, req.(*ControllerExpandVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Controller_ControllerGetVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ControllerGetVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).ControllerGetVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/ControllerGetVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).ControllerGetVolume(ctx, req.(*ControllerGetVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Controller_ControllerModifyVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ControllerModifyVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).ControllerModifyVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Controller/ControllerModifyVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).ControllerModifyVolume(ctx, req.(*ControllerModifyVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _Controller_serviceDesc = grpc.ServiceDesc{ + ServiceName: "csi.v1.Controller", + HandlerType: (*ControllerServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "CreateVolume", + Handler: _Controller_CreateVolume_Handler, + }, + { + MethodName: "DeleteVolume", + Handler: _Controller_DeleteVolume_Handler, + }, + { + MethodName: "ControllerPublishVolume", + Handler: _Controller_ControllerPublishVolume_Handler, + }, + { + MethodName: "ControllerUnpublishVolume", + Handler: _Controller_ControllerUnpublishVolume_Handler, + }, + { + MethodName: "ValidateVolumeCapabilities", + Handler: _Controller_ValidateVolumeCapabilities_Handler, + }, + { + MethodName: "ListVolumes", + Handler: _Controller_ListVolumes_Handler, + }, + { + MethodName: "GetCapacity", + Handler: _Controller_GetCapacity_Handler, + }, + { + MethodName: "ControllerGetCapabilities", + Handler: _Controller_ControllerGetCapabilities_Handler, + }, + { + MethodName: "CreateSnapshot", + Handler: _Controller_CreateSnapshot_Handler, + }, + { + MethodName: "DeleteSnapshot", + Handler: _Controller_DeleteSnapshot_Handler, + }, + { + MethodName: "ListSnapshots", + Handler: _Controller_ListSnapshots_Handler, + }, + { + MethodName: "ControllerExpandVolume", + Handler: _Controller_ControllerExpandVolume_Handler, + }, + { + MethodName: "ControllerGetVolume", + Handler: _Controller_ControllerGetVolume_Handler, + }, + { + MethodName: "ControllerModifyVolume", + Handler: _Controller_ControllerModifyVolume_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "github.com/container-storage-interface/spec/csi.proto", +} + +// GroupControllerClient is the client API for GroupController service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type GroupControllerClient interface { + GroupControllerGetCapabilities(ctx context.Context, in *GroupControllerGetCapabilitiesRequest, opts ...grpc.CallOption) (*GroupControllerGetCapabilitiesResponse, error) + CreateVolumeGroupSnapshot(ctx context.Context, in *CreateVolumeGroupSnapshotRequest, opts ...grpc.CallOption) (*CreateVolumeGroupSnapshotResponse, error) + DeleteVolumeGroupSnapshot(ctx context.Context, in *DeleteVolumeGroupSnapshotRequest, opts ...grpc.CallOption) (*DeleteVolumeGroupSnapshotResponse, error) + GetVolumeGroupSnapshot(ctx context.Context, in *GetVolumeGroupSnapshotRequest, opts ...grpc.CallOption) (*GetVolumeGroupSnapshotResponse, error) +} + +type groupControllerClient struct { + cc *grpc.ClientConn +} + +func NewGroupControllerClient(cc *grpc.ClientConn) GroupControllerClient { + return &groupControllerClient{cc} +} + +func (c *groupControllerClient) GroupControllerGetCapabilities(ctx context.Context, in *GroupControllerGetCapabilitiesRequest, opts ...grpc.CallOption) (*GroupControllerGetCapabilitiesResponse, error) { + out := new(GroupControllerGetCapabilitiesResponse) + err := c.cc.Invoke(ctx, "/csi.v1.GroupController/GroupControllerGetCapabilities", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *groupControllerClient) CreateVolumeGroupSnapshot(ctx context.Context, in *CreateVolumeGroupSnapshotRequest, opts ...grpc.CallOption) (*CreateVolumeGroupSnapshotResponse, error) { + out := new(CreateVolumeGroupSnapshotResponse) + err := c.cc.Invoke(ctx, "/csi.v1.GroupController/CreateVolumeGroupSnapshot", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *groupControllerClient) DeleteVolumeGroupSnapshot(ctx context.Context, in *DeleteVolumeGroupSnapshotRequest, opts ...grpc.CallOption) (*DeleteVolumeGroupSnapshotResponse, error) { + out := new(DeleteVolumeGroupSnapshotResponse) + err := c.cc.Invoke(ctx, "/csi.v1.GroupController/DeleteVolumeGroupSnapshot", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *groupControllerClient) GetVolumeGroupSnapshot(ctx context.Context, in *GetVolumeGroupSnapshotRequest, opts ...grpc.CallOption) (*GetVolumeGroupSnapshotResponse, error) { + out := new(GetVolumeGroupSnapshotResponse) + err := c.cc.Invoke(ctx, "/csi.v1.GroupController/GetVolumeGroupSnapshot", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// GroupControllerServer is the server API for GroupController service. +type GroupControllerServer interface { + GroupControllerGetCapabilities(context.Context, *GroupControllerGetCapabilitiesRequest) (*GroupControllerGetCapabilitiesResponse, error) + CreateVolumeGroupSnapshot(context.Context, *CreateVolumeGroupSnapshotRequest) (*CreateVolumeGroupSnapshotResponse, error) + DeleteVolumeGroupSnapshot(context.Context, *DeleteVolumeGroupSnapshotRequest) (*DeleteVolumeGroupSnapshotResponse, error) + GetVolumeGroupSnapshot(context.Context, *GetVolumeGroupSnapshotRequest) (*GetVolumeGroupSnapshotResponse, error) +} + +// UnimplementedGroupControllerServer can be embedded to have forward compatible implementations. +type UnimplementedGroupControllerServer struct { +} + +func (*UnimplementedGroupControllerServer) GroupControllerGetCapabilities(ctx context.Context, req *GroupControllerGetCapabilitiesRequest) (*GroupControllerGetCapabilitiesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GroupControllerGetCapabilities not implemented") +} +func (*UnimplementedGroupControllerServer) CreateVolumeGroupSnapshot(ctx context.Context, req *CreateVolumeGroupSnapshotRequest) (*CreateVolumeGroupSnapshotResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CreateVolumeGroupSnapshot not implemented") +} +func (*UnimplementedGroupControllerServer) DeleteVolumeGroupSnapshot(ctx context.Context, req *DeleteVolumeGroupSnapshotRequest) (*DeleteVolumeGroupSnapshotResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteVolumeGroupSnapshot not implemented") +} +func (*UnimplementedGroupControllerServer) GetVolumeGroupSnapshot(ctx context.Context, req *GetVolumeGroupSnapshotRequest) (*GetVolumeGroupSnapshotResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetVolumeGroupSnapshot not implemented") +} + +func RegisterGroupControllerServer(s *grpc.Server, srv GroupControllerServer) { + s.RegisterService(&_GroupController_serviceDesc, srv) +} + +func _GroupController_GroupControllerGetCapabilities_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GroupControllerGetCapabilitiesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GroupControllerServer).GroupControllerGetCapabilities(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.GroupController/GroupControllerGetCapabilities", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GroupControllerServer).GroupControllerGetCapabilities(ctx, req.(*GroupControllerGetCapabilitiesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _GroupController_CreateVolumeGroupSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateVolumeGroupSnapshotRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GroupControllerServer).CreateVolumeGroupSnapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.GroupController/CreateVolumeGroupSnapshot", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GroupControllerServer).CreateVolumeGroupSnapshot(ctx, req.(*CreateVolumeGroupSnapshotRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _GroupController_DeleteVolumeGroupSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteVolumeGroupSnapshotRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GroupControllerServer).DeleteVolumeGroupSnapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.GroupController/DeleteVolumeGroupSnapshot", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GroupControllerServer).DeleteVolumeGroupSnapshot(ctx, req.(*DeleteVolumeGroupSnapshotRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _GroupController_GetVolumeGroupSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetVolumeGroupSnapshotRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GroupControllerServer).GetVolumeGroupSnapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.GroupController/GetVolumeGroupSnapshot", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GroupControllerServer).GetVolumeGroupSnapshot(ctx, req.(*GetVolumeGroupSnapshotRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _GroupController_serviceDesc = grpc.ServiceDesc{ + ServiceName: "csi.v1.GroupController", + HandlerType: (*GroupControllerServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GroupControllerGetCapabilities", + Handler: _GroupController_GroupControllerGetCapabilities_Handler, + }, + { + MethodName: "CreateVolumeGroupSnapshot", + Handler: _GroupController_CreateVolumeGroupSnapshot_Handler, + }, + { + MethodName: "DeleteVolumeGroupSnapshot", + Handler: _GroupController_DeleteVolumeGroupSnapshot_Handler, + }, + { + MethodName: "GetVolumeGroupSnapshot", + Handler: _GroupController_GetVolumeGroupSnapshot_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "github.com/container-storage-interface/spec/csi.proto", +} + +// NodeClient is the client API for Node service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type NodeClient interface { + NodeStageVolume(ctx context.Context, in *NodeStageVolumeRequest, opts ...grpc.CallOption) (*NodeStageVolumeResponse, error) + NodeUnstageVolume(ctx context.Context, in *NodeUnstageVolumeRequest, opts ...grpc.CallOption) (*NodeUnstageVolumeResponse, error) + NodePublishVolume(ctx context.Context, in *NodePublishVolumeRequest, opts ...grpc.CallOption) (*NodePublishVolumeResponse, error) + NodeUnpublishVolume(ctx context.Context, in *NodeUnpublishVolumeRequest, opts ...grpc.CallOption) (*NodeUnpublishVolumeResponse, error) + NodeGetVolumeStats(ctx context.Context, in *NodeGetVolumeStatsRequest, opts ...grpc.CallOption) (*NodeGetVolumeStatsResponse, error) + NodeExpandVolume(ctx context.Context, in *NodeExpandVolumeRequest, opts ...grpc.CallOption) (*NodeExpandVolumeResponse, error) + NodeGetCapabilities(ctx context.Context, in *NodeGetCapabilitiesRequest, opts ...grpc.CallOption) (*NodeGetCapabilitiesResponse, error) + NodeGetInfo(ctx context.Context, in *NodeGetInfoRequest, opts ...grpc.CallOption) (*NodeGetInfoResponse, error) +} + +type nodeClient struct { + cc *grpc.ClientConn +} + +func NewNodeClient(cc *grpc.ClientConn) NodeClient { + return &nodeClient{cc} +} + +func (c *nodeClient) NodeStageVolume(ctx context.Context, in *NodeStageVolumeRequest, opts ...grpc.CallOption) (*NodeStageVolumeResponse, error) { + out := new(NodeStageVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeStageVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *nodeClient) NodeUnstageVolume(ctx context.Context, in *NodeUnstageVolumeRequest, opts ...grpc.CallOption) (*NodeUnstageVolumeResponse, error) { + out := new(NodeUnstageVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeUnstageVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *nodeClient) NodePublishVolume(ctx context.Context, in *NodePublishVolumeRequest, opts ...grpc.CallOption) (*NodePublishVolumeResponse, error) { + out := new(NodePublishVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodePublishVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *nodeClient) NodeUnpublishVolume(ctx context.Context, in *NodeUnpublishVolumeRequest, opts ...grpc.CallOption) (*NodeUnpublishVolumeResponse, error) { + out := new(NodeUnpublishVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeUnpublishVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *nodeClient) NodeGetVolumeStats(ctx context.Context, in *NodeGetVolumeStatsRequest, opts ...grpc.CallOption) (*NodeGetVolumeStatsResponse, error) { + out := new(NodeGetVolumeStatsResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeGetVolumeStats", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *nodeClient) NodeExpandVolume(ctx context.Context, in *NodeExpandVolumeRequest, opts ...grpc.CallOption) (*NodeExpandVolumeResponse, error) { + out := new(NodeExpandVolumeResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeExpandVolume", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *nodeClient) NodeGetCapabilities(ctx context.Context, in *NodeGetCapabilitiesRequest, opts ...grpc.CallOption) (*NodeGetCapabilitiesResponse, error) { + out := new(NodeGetCapabilitiesResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeGetCapabilities", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *nodeClient) NodeGetInfo(ctx context.Context, in *NodeGetInfoRequest, opts ...grpc.CallOption) (*NodeGetInfoResponse, error) { + out := new(NodeGetInfoResponse) + err := c.cc.Invoke(ctx, "/csi.v1.Node/NodeGetInfo", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// NodeServer is the server API for Node service. +type NodeServer interface { + NodeStageVolume(context.Context, *NodeStageVolumeRequest) (*NodeStageVolumeResponse, error) + NodeUnstageVolume(context.Context, *NodeUnstageVolumeRequest) (*NodeUnstageVolumeResponse, error) + NodePublishVolume(context.Context, *NodePublishVolumeRequest) (*NodePublishVolumeResponse, error) + NodeUnpublishVolume(context.Context, *NodeUnpublishVolumeRequest) (*NodeUnpublishVolumeResponse, error) + NodeGetVolumeStats(context.Context, *NodeGetVolumeStatsRequest) (*NodeGetVolumeStatsResponse, error) + NodeExpandVolume(context.Context, *NodeExpandVolumeRequest) (*NodeExpandVolumeResponse, error) + NodeGetCapabilities(context.Context, *NodeGetCapabilitiesRequest) (*NodeGetCapabilitiesResponse, error) + NodeGetInfo(context.Context, *NodeGetInfoRequest) (*NodeGetInfoResponse, error) +} + +// UnimplementedNodeServer can be embedded to have forward compatible implementations. +type UnimplementedNodeServer struct { +} + +func (*UnimplementedNodeServer) NodeStageVolume(ctx context.Context, req *NodeStageVolumeRequest) (*NodeStageVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeStageVolume not implemented") +} +func (*UnimplementedNodeServer) NodeUnstageVolume(ctx context.Context, req *NodeUnstageVolumeRequest) (*NodeUnstageVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeUnstageVolume not implemented") +} +func (*UnimplementedNodeServer) NodePublishVolume(ctx context.Context, req *NodePublishVolumeRequest) (*NodePublishVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodePublishVolume not implemented") +} +func (*UnimplementedNodeServer) NodeUnpublishVolume(ctx context.Context, req *NodeUnpublishVolumeRequest) (*NodeUnpublishVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeUnpublishVolume not implemented") +} +func (*UnimplementedNodeServer) NodeGetVolumeStats(ctx context.Context, req *NodeGetVolumeStatsRequest) (*NodeGetVolumeStatsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGetVolumeStats not implemented") +} +func (*UnimplementedNodeServer) NodeExpandVolume(ctx context.Context, req *NodeExpandVolumeRequest) (*NodeExpandVolumeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeExpandVolume not implemented") +} +func (*UnimplementedNodeServer) NodeGetCapabilities(ctx context.Context, req *NodeGetCapabilitiesRequest) (*NodeGetCapabilitiesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGetCapabilities not implemented") +} +func (*UnimplementedNodeServer) NodeGetInfo(ctx context.Context, req *NodeGetInfoRequest) (*NodeGetInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NodeGetInfo not implemented") +} + +func RegisterNodeServer(s *grpc.Server, srv NodeServer) { + s.RegisterService(&_Node_serviceDesc, srv) +} + +func _Node_NodeStageVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeStageVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServer).NodeStageVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Node/NodeStageVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServer).NodeStageVolume(ctx, req.(*NodeStageVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Node_NodeUnstageVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeUnstageVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServer).NodeUnstageVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Node/NodeUnstageVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServer).NodeUnstageVolume(ctx, req.(*NodeUnstageVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Node_NodePublishVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodePublishVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServer).NodePublishVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Node/NodePublishVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServer).NodePublishVolume(ctx, req.(*NodePublishVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Node_NodeUnpublishVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeUnpublishVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServer).NodeUnpublishVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Node/NodeUnpublishVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServer).NodeUnpublishVolume(ctx, req.(*NodeUnpublishVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Node_NodeGetVolumeStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeGetVolumeStatsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServer).NodeGetVolumeStats(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Node/NodeGetVolumeStats", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServer).NodeGetVolumeStats(ctx, req.(*NodeGetVolumeStatsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Node_NodeExpandVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeExpandVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServer).NodeExpandVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Node/NodeExpandVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServer).NodeExpandVolume(ctx, req.(*NodeExpandVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Node_NodeGetCapabilities_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeGetCapabilitiesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServer).NodeGetCapabilities(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Node/NodeGetCapabilities", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServer).NodeGetCapabilities(ctx, req.(*NodeGetCapabilitiesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Node_NodeGetInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NodeGetInfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServer).NodeGetInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/csi.v1.Node/NodeGetInfo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServer).NodeGetInfo(ctx, req.(*NodeGetInfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _Node_serviceDesc = grpc.ServiceDesc{ + ServiceName: "csi.v1.Node", + HandlerType: (*NodeServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "NodeStageVolume", + Handler: _Node_NodeStageVolume_Handler, + }, + { + MethodName: "NodeUnstageVolume", + Handler: _Node_NodeUnstageVolume_Handler, + }, + { + MethodName: "NodePublishVolume", + Handler: _Node_NodePublishVolume_Handler, + }, + { + MethodName: "NodeUnpublishVolume", + Handler: _Node_NodeUnpublishVolume_Handler, + }, + { + MethodName: "NodeGetVolumeStats", + Handler: _Node_NodeGetVolumeStats_Handler, + }, + { + MethodName: "NodeExpandVolume", + Handler: _Node_NodeExpandVolume_Handler, + }, + { + MethodName: "NodeGetCapabilities", + Handler: _Node_NodeGetCapabilities_Handler, + }, + { + MethodName: "NodeGetInfo", + Handler: _Node_NodeGetInfo_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "github.com/container-storage-interface/spec/csi.proto", +} diff --git a/vendor/github.com/container-storage-interface/spec/lib/go/csi/ya.make b/vendor/github.com/container-storage-interface/spec/lib/go/csi/ya.make new file mode 100644 index 00000000000..ed017313a54 --- /dev/null +++ b/vendor/github.com/container-storage-interface/spec/lib/go/csi/ya.make @@ -0,0 +1,9 @@ +GO_LIBRARY() + +LICENSE(Apache-2.0) + +SRCS( + csi.pb.go +) + +END()