diff --git a/NOTICE b/NOTICE index 0c16a260f..643129991 100644 --- a/NOTICE +++ b/NOTICE @@ -21358,4 +21358,250 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + + +**************************************************************************** + +============================================================================ +>>> github.com/cespare/xxhash/v2 +============================================================================== + +Copyright (c) 2016 Caleb Spare + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + + +**************************************************************************** + +============================================================================ +>>> github.com/google/s2a-go +============================================================================== + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + --------------------- END OF THIRD PARTY NOTICE -------------------------------- diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 5bef26312..d1eb90101 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -117,14 +117,18 @@ steps: echo "{" > $cnfFile echo "\"block-acct\"": "\"$(AZTEST_BLOCK_ACC_NAME)\"", >> $cnfFile echo "\"adls-acct\"": "\"$(AZTEST_ADLS_ACC_NAME)\"", >> $cnfFile + echo "\"file-acct\"": "\"$(AZTEST_FILE_ACC_NAME)\"", >> $cnfFile echo "\"block-cont\"": "\"${{ parameters.container }}\"", >> $cnfFile echo "\"adls-cont\"": "\"${{ parameters.container }}\"", >> $cnfFile + echo "\"file-cont\"": "\"${{ parameters.container }}\"", >> $cnfFile echo "\"block-key\"": "\"$(AZTEST_BLOCK_KEY)\"", >> $cnfFile echo "\"adls-key\"": "\"$(AZTEST_ADLS_KEY)\"", >> $cnfFile + echo "\"file-key\"": "\"$(AZTEST_FILE_KEY)\"", >> $cnfFile echo "\"block-sas\"": "\"$(AZTEST_BLOCK_SAS)\"", >> $cnfFile echo "\"block-cont-sas-ubn-18\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_18)\"", >> $cnfFile echo "\"block-cont-sas-ubn-20\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_20)\"", >> $cnfFile echo "\"adls-sas\"": "\"$(AZTEST_ADLS_SAS)\"", >> $cnfFile + echo "\"file-sas\"": "\"$(AZTEST_FILE_SAS)\"", >> $cnfFile echo "\"msi-appid\"": "\"$(AZTEST_APP_ID)\"", >> $cnfFile echo "\"msi-resid\"": "\"$(AZTEST_RES_ID)\"", >> $cnfFile echo "\"msi-objid\"": "\"$(AZTEST_OBJ_ID)\"", >> $cnfFile diff --git a/azure-pipeline-templates/distro-tests.yml b/azure-pipeline-templates/distro-tests.yml index 2b8f7e567..19db3bd8e 100755 --- a/azure-pipeline-templates/distro-tests.yml +++ b/azure-pipeline-templates/distro-tests.yml @@ -23,6 +23,10 @@ parameters: type: string - name: adls_account_key type: string + - name: file_account_name + type: string + - name: file_account_key + type: string - name: distro_name type: string - name: tags @@ -93,6 +97,7 @@ steps: container: $(containerName) idstring: block_blob_key adls: false + fileshare: false account_name: ${{ parameters.blob_account_name }} account_key: ${{ parameters.blob_account_key }} account_type: block @@ -114,6 +119,7 @@ steps: container: $(containerName) idstring: adls_key adls: true + fileshare: false account_name: ${{ parameters.adls_account_name }} account_key: ${{ parameters.adls_account_key }} account_type: adls @@ -122,6 +128,27 @@ steps: quick_test: ${{ parameters.quick_test }} verbose_log: ${{ parameters.verbose_log }} clone: ${{ parameters.clone }} + # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable + temp_dir: $(TEMP_DIR) + mount_dir: $(MOUNT_DIR) + + # File Share Test + - template: 'e2e-tests-spcl.yml' + parameters: + conf_template: azure_key.yaml + config_file: $(BLOBFUSE2_CFG) + container: $(containerName) + idstring: file_key + adls: false + fileshare: true + account_name: ${{ parameters.file_account_name }} + account_key: ${{ parameters.file_account_key }} + account_type: file + account_endpoint: https://${{ parameters.file_account_name }}.file.core.windows.net + distro_name: ${{ parameters.distro_name }} + quick_test: ${{ parameters.quick_test }} + verbose_log: ${{ parameters.verbose_log }} + clone: ${{ parameters.clone }} stream_direct_test: false # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable temp_dir: $(TEMP_DIR) diff --git a/azure-pipeline-templates/e2e-tests-spcl.yml b/azure-pipeline-templates/e2e-tests-spcl.yml index 3aca57687..f208f8a47 100644 --- a/azure-pipeline-templates/e2e-tests-spcl.yml +++ b/azure-pipeline-templates/e2e-tests-spcl.yml @@ -13,6 +13,8 @@ parameters: type: string - name: adls type: boolean + - name: fileshare + type: boolean - name: account_name type: string - name: account_key @@ -58,6 +60,7 @@ steps: mount_dir: ${{ parameters.mount_dir }} temp_dir: ${{ parameters.temp_dir }} adls: ${{ parameters.adls }} + fileshare: ${{ parameters.fileshare }} idstring: '${{ parameters.idstring }}' distro_name: ${{ parameters.distro_name }} quick_test: ${{ parameters.quick_test }} diff --git a/azure-pipeline-templates/e2e-tests.yml b/azure-pipeline-templates/e2e-tests.yml index 39c8766d9..59acdb399 100755 --- a/azure-pipeline-templates/e2e-tests.yml +++ b/azure-pipeline-templates/e2e-tests.yml @@ -13,6 +13,8 @@ parameters: type: step - name: adls type: boolean + - name: fileshare + type: boolean - name: clone type: boolean default: false @@ -51,7 +53,7 @@ steps: - task: Go@0 inputs: command: 'test' - arguments: '-v -timeout=2h ./... -args -mnt-path=${{ parameters.mount_dir }} -adls=${{parameters.adls}} -clone=${{parameters.clone}} -tmp-path=${{parameters.temp_dir}} -quick-test=${{parameters.quick_test}} -stream-direct-test=${{parameters.stream_direct_test}} -distro-name="${{parameters.distro_name}}"' + arguments: '-v -timeout=2h ./... -args -mnt-path=${{ parameters.mount_dir }} -adls=${{parameters.adls}} -fileshare=${{parameters.fileshare}} -clone=${{parameters.clone}} -tmp-path=${{parameters.temp_dir}} -quick-test=${{parameters.quick_test}} -stream-direct-test=${{parameters.stream_direct_test}} -distro-name="${{parameters.distro_name}}"' workingDirectory: ${{ parameters.working_dir }}/test/e2e_tests displayName: 'E2E Test: ${{ parameters.idstring }}' timeoutInMinutes: 120 diff --git a/azure-pipeline-templates/verbose-tests.yml b/azure-pipeline-templates/verbose-tests.yml index 331bd15d2..4569887ac 100644 --- a/azure-pipeline-templates/verbose-tests.yml +++ b/azure-pipeline-templates/verbose-tests.yml @@ -9,6 +9,8 @@ parameters: type: string - name: adls type: boolean + - name: fileshare + type: boolean - name: account_name type: string - name: spn_account_name @@ -90,6 +92,7 @@ steps: - script: cat ${{ parameters.config }} displayName: Print config file + condition: ${{ parameters.test_key_credential }} # Stream e2e - script: | @@ -107,6 +110,7 @@ steps: - script: cat ${{ parameters.stream_config }} displayName: Print Stream config file with Handle Level Caching + condition: ${{ parameters.test_stream }} # Stream e2e filename level caching - script: | @@ -124,6 +128,7 @@ steps: - script: cat ${{ parameters.stream_filename_config }} displayName: Print Stream config file with Filename Caching + condition: ${{ parameters.test_stream }} # Create sas credential config file if we need to test it - script: | @@ -141,6 +146,7 @@ steps: - script: cat ${{ parameters.sas_credential_config }} displayName: Print SAS config file + condition: ${{ parameters.test_sas_credential }} # Create spn credential config file if we need to test it - script: | @@ -160,6 +166,7 @@ steps: - script: cat ${{ parameters.spn_credential_config }} displayName: Print SPN config file + condition: ${{ parameters.test_spn_credential }} # Create azurite config file if we need to test it - script: | @@ -190,6 +197,7 @@ steps: mount_dir: ${{ parameters.mount_dir }} temp_dir: ${{ parameters.temp_dir }} adls: ${{ parameters.adls }} + fileshare: ${{ parameters.fileshare }} idstring: ${{ parameters.service }} with Key Credentials distro_name: ${{ parameters.distro_name }} quick_test: ${{ parameters.quick_test }} @@ -210,6 +218,7 @@ steps: temp_dir: ${{ parameters.temp_dir }} mount_dir: ${{ parameters.mount_dir }} adls: ${{ parameters.adls }} + fileshare: ${{ parameters.fileshare }} idstring: ${{ parameters.service }} with Streaming distro_name: ${{ parameters.distro_name }} quick_test: ${{ parameters.quick_test }} @@ -230,6 +239,7 @@ steps: temp_dir: ${{ parameters.temp_dir }} mount_dir: ${{ parameters.mount_dir }} adls: ${{ parameters.adls }} + fileshare: ${{ parameters.fileshare }} idstring: ${{ parameters.service }} with Streaming with filename distro_name: ${{ parameters.distro_name }} quick_test: ${{ parameters.quick_test }} @@ -250,6 +260,7 @@ steps: mount_dir: ${{ parameters.mount_dir }} temp_dir: ${{ parameters.temp_dir }} adls: ${{ parameters.adls }} + fileshare: ${{ parameters.fileshare }} idstring: ${{ parameters.service }} with SAS Credentials distro_name: ${{ parameters.distro_name }} artifact_name: '${{ parameters.distro_name }}_${{ parameters.service }}_sas.txt' @@ -269,6 +280,7 @@ steps: mount_dir: ${{ parameters.mount_dir }} temp_dir: ${{ parameters.temp_dir }} adls: ${{ parameters.adls }} + fileshare: ${{ parameters.fileshare }} idstring: ${{ parameters.service }} with SPN Credentials distro_name: ${{ parameters.distro_name }} artifact_name: '${{ parameters.distro_name }}_${{ parameters.service }}_spn.txt' @@ -296,6 +308,7 @@ steps: mount_dir: ${{ parameters.mount_dir }} temp_dir: ${{ parameters.temp_dir }} adls: ${{ parameters.adls }} + fileshare: ${{ parameters.fileshare }} idstring: ${{ parameters.service }} with Azurite distro_name: ${{ parameters.distro_name }} quick_test: ${{ parameters.quick_test }} @@ -319,6 +332,7 @@ steps: temp_dir: ${{ parameters.temp_dir }} mount_dir: ${{ parameters.mount_dir }} adls: ${{ parameters.adls }} + fileshare: ${{ parameters.fileshare }} account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} account_type: ${{ parameters.account_type }} @@ -335,6 +349,7 @@ steps: temp_dir: ${{ parameters.temp_dir }} mount_dir: ${{ parameters.mount_dir }} adls: ${{ parameters.adls }} + fileshare: ${{ parameters.fileshare }} account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} account_type: ${{ parameters.account_type }} @@ -351,6 +366,7 @@ steps: temp_dir: ${{ parameters.temp_dir }} mount_dir: ${{ parameters.mount_dir }} adls: ${{ parameters.adls }} + fileshare: ${{ parameters.fileshare }} account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} account_type: ${{ parameters.account_type }} diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 808acc9a6..62a52041c 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -9,6 +9,7 @@ pr: jobs: # Ubuntu based test suite - job: test + timeoutInMinutes: 150 displayName: Build and Test on strategy: matrix: @@ -65,14 +66,18 @@ jobs: echo "{" > $cnfFile echo "\"block-acct\"": "\"$(AZTEST_BLOCK_ACC_NAME)\"", >> $cnfFile echo "\"adls-acct\"": "\"$(AZTEST_ADLS_ACC_NAME)\"", >> $cnfFile + echo "\"file-acct\"": "\"$(AZTEST_FILE_ACC_NAME)\"", >> $cnfFile echo "\"block-cont\"": "\"$(containerName)\"", >> $cnfFile echo "\"adls-cont\"": "\"$(containerName)\"", >> $cnfFile + cho "\"file-cont\"": "\"$(containerName)\"", >> $cnfFile echo "\"block-key\"": "\"$(AZTEST_BLOCK_KEY)\"", >> $cnfFile echo "\"adls-key\"": "\"$(AZTEST_ADLS_KEY)\"", >> $cnfFile + echo "\"file-key\"": "\"$(AZTEST_FILE_KEY)\"", >> $cnfFile echo "\"block-sas\"": "\"$(AZTEST_BLOCK_SAS)\"", >> $cnfFile + echo "\"adls-sas\"": "\"$(adlsSas)\"", >> $cnfFile + echo "\"file-sas\"": "\"$(AZTEST_FILE_SAS)\"", >> $cnfFile echo "\"block-cont-sas-ubn-18\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_18)\"", >> $cnfFile echo "\"block-cont-sas-ubn-20\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_20)\"", >> $cnfFile - echo "\"adls-sas\"": "\"$(adlsSas)\"", >> $cnfFile echo "\"msi-appid\"": "\"$(AZTEST_APP_ID)\"", >> $cnfFile echo "\"msi-resid\"": "\"$(AZTEST_RES_ID)\"", >> $cnfFile echo "\"msi-objid\"": "\"$(AZTEST_OBJ_ID)\"", >> $cnfFile diff --git a/blobfuse2-code-coverage.yaml b/blobfuse2-code-coverage.yaml index 95f6b4837..6ba89baa8 100644 --- a/blobfuse2-code-coverage.yaml +++ b/blobfuse2-code-coverage.yaml @@ -776,7 +776,7 @@ stages: - task: Go@0 inputs: command: 'test' - arguments: '-timeout 120m -v test/accoutcleanup/accountcleanup_test.go' + arguments: '-timeout 120m -v test/accountcleanup_container/accountcleanup_container_test.go' workingDirectory: $(WORK_DIR) displayName: 'Block Blob cleanup' timeoutInMinutes: 120 @@ -789,7 +789,7 @@ stages: - task: Go@0 inputs: command: 'test' - arguments: '-timeout 120m -v test/accoutcleanup/accountcleanup_test.go' + arguments: '-timeout 120m -v test/accountcleanup_container/accountcleanup_container_test.go' workingDirectory: $(WORK_DIR) displayName: 'Gen2 cleanup' timeoutInMinutes: 120 @@ -798,3 +798,16 @@ stages: STORAGE_ACCOUNT_NAME: $(AZTEST_ADLS_ACC_NAME) STORAGE_ACCOUNT_Key: $(AZTEST_ADLS_KEY) + # Run the file share cleanup + - task: Go@0 + inputs: + command: 'test' + arguments: '-timeout 120m -v test/accountcleanup_share/accountcleanup_share_test.go' + workingDirectory: $(WORK_DIR) + displayName: 'File Share cleanup' + timeoutInMinutes: 120 + continueOnError: true + env: + STORAGE_ACCOUNT_NAME: $(AZTEST_FILE_ACC_NAME) + STORAGE_ACCOUNT_KEY: $(AZTEST_FILE_KEY) + diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index ab0501eba..fa6cffc9b 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -162,6 +162,7 @@ stages: account_type: 'block' account_endpoint: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' adls: false + fileshare: false account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) account_sas: $(NIGHTLY_STO_ACC_SAS) @@ -294,6 +295,7 @@ stages: account_type: 'adls' account_endpoint: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' adls: true + fileshare: false account_name: $(AZTEST_ADLS_ACC_NAME) account_key: $(AZTEST_ADLS_KEY) account_sas: $(adlsSas) @@ -330,10 +332,143 @@ stages: mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) + # Ubuntu Tests + - job: Set_3 + timeoutInMinutes: 300 + + strategy: + matrix: + Ubuntu-20-FileShare: + imageName: 'ubuntu-20.04' + containerName: 'test-cnt-ubn-20' + adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) + fuselib: 'libfuse-dev' + tags: 'fuse2' + Ubuntu-22-FileShare: + imageName: 'ubuntu-22.04' + containerName: 'test-cnt-ubn-22' + adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) + fuselib: 'libfuse3-dev' + tags: 'fuse3' + pool: + vmImage: $(imageName) + + variables: + - group: NightlyBlobFuse + - name: MOUNT_DIR + value: '$(Pipeline.Workspace)/blob_mnt' + - name: TEMP_DIR + value: '$(Pipeline.Workspace)/blobfuse2_tmp' + - name: BLOBFUSE2_CFG + value: '$(Pipeline.Workspace)/blobfuse2.yaml' + - name: BLOBFUSE2_SAS_CFG + value: '$(Pipeline.Workspace)/blobfuse2_sas_config.yaml' + - name: BLOBFUSE2_SPN_CFG + value: '$(Pipeline.Workspace)/blobfuse2_spn_config.yaml' + - name: BLOBFUSE2_STREAM_CFG + value: '$(Pipeline.Workspace)/blobfuse2_stream.yaml' + - name: BLOBFUSE2_STREAM_FILENAME_CFG + value: '$(Pipeline.Workspace)/blobfuse2_stream_filename.yaml' + - name: BLOBFUSE2_ADLS_CFG + value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' + - name: BLOBFUSE2_GTEST_CFG + value: '$(Pipeline.Workspace)/connection.yaml' + - name: BLOBFUSE2_AZURITE_CFG + value: '$(Pipeline.Workspace)/blobfuse2_azurite_config.yaml' + - name: BLOBFUSE2_STRESS_DIR + value: '$(Pipeline.Workspace)/blobfuse2_stress' + - name: DECODE_PERCENTS + value: false + - name: GOPATH + value: '$(Pipeline.Workspace)/go' + - name: ROOT_DIR + value: '$(System.DefaultWorkingDirectory)' + - name: WORK_DIR + value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + + steps: + - checkout: none + + # Clone the repo + - script: | + git clone https://github.com/Azure/azure-storage-fuse + displayName: 'Checkout Code' + workingDirectory: $(ROOT_DIR) + + # Checkout the branch + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout Branch' + workingDirectory: $(WORK_DIR) + + - script: | + sudo apt-get update --fix-missing + sudo apt-get install $(fuselib) -y + displayName: 'Install libfuse' + + # ------------------------------------------------------- + # Pull and build the code + - template: 'azure-pipeline-templates/build.yml' + parameters: + working_directory: $(WORK_DIR) + root_dir: $(Pipeline.Workspace) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) + gopath: $(GOPATH) + container: $(containerName) + tags: $(tags) + fuselib: $(fuselib) + skip_ut: true # Skip UT because Block Blob set runs it + + # ------------------------------------------------------- + - ${{ if eq(parameters.exhaustive_test, true) }}: + - template: 'azure-pipeline-templates/verbose-tests.yml' + parameters: + service: 'FileShare' + account_type: 'file' + account_endpoint: 'https://$(AZTEST_FILE_ACC_NAME).file.core.windows.net' + adls: false + fileshare: true + account_name: $(AZTEST_FILE_ACC_NAME) + account_key: $(AZTEST_FILE_KEY) + account_sas: $(AZTEST_FILE_SAS) + spn_account_name: "" + spn_account_endpoint: "" + client_id: "" + tenant_id: "" + client_secret: "" + container: $(containerName) + config: $(BLOBFUSE2_CFG) + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) + stress_dir: $(BLOBFUSE2_STRESS_DIR) + huge_container: 'testcnt' + quick_stress: ${{ parameters.quick_stress }} + test_key_credential: true + test_sas_credential: true + test_spn_credential: false + test_stream: true + test_azurite: false + stream_config: $(BLOBFUSE2_STREAM_CFG) + stream_filename_config: $(BLOBFUSE2_STREAM_FILENAME_CFG) + sas_credential_config: $(BLOBFUSE2_SAS_CFG) + spn_credential_config: $(BLOBFUSE2_SPN_CFG) + azurite_config: $(BLOBFUSE2_AZURITE_CFG) + distro_name: $(imageName) + verbose_log: ${{ parameters.verbose_log }} + tags: $(tags) + + - template: azure-pipeline-templates/cleanup.yml + parameters: + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) + - ${{ if eq(parameters.proxy_test, true) }}: # ----------------------------------------------------------- # Ubuntu-20.04 Proxy tests - - job: Set_3 + - job: Set_4 timeoutInMinutes: 300 strategy: matrix: @@ -441,6 +576,7 @@ stages: idstring: 'BlockBlob with Proxy and Key Credentials' distro_name: $(imageName) adls: false + fileshare: false artifact_name: 'blockblob_proxy_key.txt' verbose_log: ${{ parameters.verbose_log }} mountStep: @@ -475,6 +611,7 @@ stages: idstring: 'ADLS with Proxy and Key Credentials' distro_name: $(imageName) adls: true + fileshare: false artifact_name: 'adls_proxy_key.txt' verbose_log: ${{ parameters.verbose_log }} mountStep: @@ -558,7 +695,7 @@ stages: - ${{ if eq(parameters.exhaustive_test, true) }}: # RHEL Tests - - job: Set_4 + - job: Set_5 timeoutInMinutes: 60 strategy: matrix: @@ -623,6 +760,8 @@ stages: blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) + file_account_name: $(AZTEST_FILE_ACC_NAME) + file_account_key: $(AZTEST_FILE_KEY) distro_name: $(AgentName) gopath: $(GOPATH) tags: $(tags) @@ -641,7 +780,7 @@ stages: verbose_log: ${{ parameters.verbose_log }} # Centos Tests - - job: Set_5 + - job: Set_6 timeoutInMinutes: 60 strategy: matrix: @@ -705,6 +844,8 @@ stages: blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) + file_account_name: $(AZTEST_FILE_ACC_NAME) + file_account_key: $(AZTEST_FILE_KEY) distro_name: $(AgentName) gopath: $(GOPATH) installStep: @@ -719,7 +860,7 @@ stages: verbose_log: ${{ parameters.verbose_log }} # Oracle Tests - - job: Set_6 + - job: Set_7 timeoutInMinutes: 60 strategy: matrix: @@ -772,6 +913,8 @@ stages: blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) + file_account_name: $(AZTEST_FILE_ACC_NAME) + file_account_key: $(AZTEST_FILE_KEY) distro_name: $(AgentName) gopath: $(GOPATH) installStep: @@ -844,6 +987,8 @@ stages: blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) + file_account_name: $(AZTEST_FILE_ACC_NAME) + file_account_key: $(AZTEST_FILE_KEY) distro_name: $(AgentName) tags: $(tags) fuselib: $(fuselib) @@ -911,6 +1056,8 @@ stages: blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) + file_account_name: $(AZTEST_FILE_ACC_NAME) + file_account_key: $(AZTEST_FILE_KEY) distro_name: $(AgentName) gopath: $(GOPATH) installStep: @@ -980,6 +1127,8 @@ stages: blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) + file_account_name: $(AZTEST_FILE_ACC_NAME) + file_account_key: $(AZTEST_FILE_KEY) distro_name: $(AgentName) tags: $(tags) fuselib: $(fuselib) @@ -1208,6 +1357,7 @@ stages: container: $(containerName) idstring: Block_Blob adls: false + fileshare: false account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) account_type: block @@ -1278,6 +1428,7 @@ stages: container: $(containerName) idstring: ADLS adls: true + fileshare: false account_name: $(AZTEST_ADLS_ACC_NAME) account_key: $(AZTEST_ADLS_KEY) account_type: adls @@ -1291,6 +1442,77 @@ stages: temp_dir: $(TEMP_DIR) mount_dir: $(MOUNT_DIR) + - ${{ if eq(parameters.data_validation, true) }}: + - stage: DataValidationFileShare + jobs: + # Ubuntu Tests + - job: Set_1 + timeoutInMinutes: 300 + strategy: + matrix: + Ubuntu-20: + imageName: 'ubuntu-20.04' + containerName: 'test-cnt-ubn-20' + fuselib: 'libfuse-dev' + tags: 'fuse2' + Ubuntu-22: + imageName: 'ubuntu-22.04' + containerName: 'test-cnt-ubn-22' + fuselib: 'libfuse3-dev' + tags: 'fuse3' + pool: + vmImage: $(imageName) + + variables: + - group: NightlyBlobFuse + - name: ROOT_DIR + value: "/usr/pipeline/workv2" + - name: WORK_DIR + value: "/usr/pipeline/workv2/go/src/azure-storage-fuse" + - name: skipComponentGovernanceDetection + value: true + - name: MOUNT_DIR + value: "/usr/pipeline/workv2/blob_mnt" + - name: TEMP_DIR + value: "/usr/pipeline/workv2/temp" + - name: BLOBFUSE2_CFG + value: "/usr/pipeline/workv2/blobfuse2.yaml" + - name: GOPATH + value: "/usr/pipeline/workv2/go" + + steps: + - checkout: none + + - template: 'azure-pipeline-templates/setup.yml' + parameters: + tags: $(tags) + installStep: + script: | + sudo apt-get update --fix-missing + sudo apt-get install $(fuselib) -y + displayName: 'Install fuse' + + - template: 'azure-pipeline-templates/e2e-tests-spcl.yml' + parameters: + conf_template: azure_key.yaml + config_file: $(BLOBFUSE2_CFG) + container: $(containerName) + idstring: FILE + adls: false + fileshare: true + account_name: $(AZTEST_FILE_ACC_NAME) + account_key: $(AZTEST_FILE_KEY) + account_type: file + account_endpoint: https://$(AZTEST_FILE_ACC_NAME).file.core.windows.net + distro_name: $(imageName) + quick_test: false + verbose_log: ${{ parameters.verbose_log }} + clone: false # This test case takes too long with file share accounts. + stream_direct_test: false + # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable + temp_dir: $(TEMP_DIR) + mount_dir: $(MOUNT_DIR) + - ${{ if eq(parameters.data_validation, true) }}: - stage: DataValidationStreamFileHandle jobs: @@ -1348,6 +1570,7 @@ stages: container: $(containerName) idstring: Stream_File_Handle adls: false + fileshare: false account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) account_type: block @@ -1418,6 +1641,7 @@ stages: container: $(containerName) idstring: Stream_File_Handle_Direct adls: false + fileshare: false account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) account_type: block @@ -1488,6 +1712,7 @@ stages: container: $(containerName) idstring: Stream_File_Name adls: false + fileshare: false account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) account_type: block @@ -1558,6 +1783,7 @@ stages: container: $(containerName) idstring: Stream_File_Name_Direct adls: false + fileshare: false account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) account_type: block diff --git a/component/azstorage/azauth.go b/component/azstorage/azauth.go index 02ee22f8d..697d2e5ad 100644 --- a/component/azstorage/azauth.go +++ b/component/azstorage/azauth.go @@ -91,6 +91,8 @@ func getAzAuth(config azAuthConfig) azAuth { return getAzAuthBlob(config) } else if EAccountType.ADLS() == config.AccountType { return getAzAuthBfs(config) + } else if EAccountType.FILE() == config.AccountType { + return getAzAuthFile(config) } return nil } @@ -159,6 +161,26 @@ func getAzAuthBfs(config azAuthConfig) azAuth { return nil } +func getAzAuthFile(config azAuthConfig) azAuth { + base := azAuthBase{config: config} + if config.AuthMode == EAuthType.KEY() { + return &azAuthFileKey{ + azAuthKey{ + azAuthBase: base, + }, + } + } else if config.AuthMode == EAuthType.SAS() { + return &azAuthFileSAS{ + azAuthSAS{ + azAuthBase: base, + }, + } + } else { + log.Crit("azAuth::getAzAuthBfs : Auth type %s not supported. Failed to create Auth object", config.AuthMode) + } + return nil +} + type azAuthBase struct { config azAuthConfig } diff --git a/component/azstorage/azauth_test.go b/component/azstorage/azauth_test.go index 82e4142c6..4cd9e6b83 100644 --- a/component/azstorage/azauth_test.go +++ b/component/azstorage/azauth_test.go @@ -54,17 +54,22 @@ type storageTestConfiguration struct { // Get the mount path from command line argument BlockAccount string `json:"block-acct"` AdlsAccount string `json:"adls-acct"` + FileAccount string `json:"file-acct"` BlockContainer string `json:"block-cont"` AdlsContainer string `json:"adls-cont"` + FileContainer string `json:"file-cont"` // AdlsDirectory string `json:"adls-dir"` BlockContainerHuge string `json:"block-cont-huge"` AdlsContainerHuge string `json:"adls-cont-huge"` + FileContainerHuge string `json:"file-cont-huge"` BlockKey string `json:"block-key"` AdlsKey string `json:"adls-key"` + FileKey string `json:"file-key"` BlockSas string `json:"block-sas"` BlockContSasUbn18 string `json:"block-cont-sas-ubn-18"` BlockContSasUbn20 string `json:"block-cont-sas-ubn-20"` AdlsSas string `json:"adls-sas"` + FileSas string `json:"file-sas"` // AdlsDirSasUbn18 string `json:"adls-dir-sas-ubn-18"` // AdlsDirSasUbn20 string `json:"adls-dir-sas-ubn-20"` MsiAppId string `json:"msi-appid"` @@ -144,6 +149,8 @@ func generateEndpoint(useHttp bool, accountName string, accountType AccountType) endpoint += ".dfs." } else if accountType == EAccountType.BLOCK() { endpoint += ".blob." + } else if accountType == EAccountType.FILE() { + endpoint += ".file." } endpoint += "core.windows.net/" return endpoint @@ -339,6 +346,37 @@ func (suite *authTestSuite) TestHttpAdlsSharedKey() { suite.validateStorageTest("TestHttpAdlsSharedKey", stgConfig) } +func (suite *authTestSuite) TestFileSharedKey() { + defer suite.cleanupTest() + stgConfig := AzStorageConfig{ + container: storageTestConfigurationParameters.FileContainer, + authConfig: azAuthConfig{ + AuthMode: EAuthType.KEY(), + AccountType: EAccountType.FILE(), + AccountName: storageTestConfigurationParameters.FileAccount, + AccountKey: storageTestConfigurationParameters.FileKey, + Endpoint: generateEndpoint(false, storageTestConfigurationParameters.FileAccount, EAccountType.FILE()), + }, + } + suite.validateStorageTest("TestFileSharedKey", stgConfig) +} + +func (suite *authTestSuite) TestHttpFileSharedKey() { + defer suite.cleanupTest() + stgConfig := AzStorageConfig{ + container: storageTestConfigurationParameters.FileContainer, + authConfig: azAuthConfig{ + AuthMode: EAuthType.KEY(), + AccountType: EAccountType.FILE(), + AccountName: storageTestConfigurationParameters.FileAccount, + AccountKey: storageTestConfigurationParameters.FileKey, + UseHTTP: true, + Endpoint: generateEndpoint(true, storageTestConfigurationParameters.FileAccount, EAccountType.FILE()), + }, + } + suite.validateStorageTest("TestHttpFileSharedKey", stgConfig) +} + func (suite *authTestSuite) TestBlockInvalidSasKey() { defer suite.cleanupTest() stgConfig := AzStorageConfig{ @@ -522,6 +560,37 @@ func (suite *authTestSuite) TestHttpAdlsSasKey() { suite.validateStorageTest("TestHttpAdlsSasKey", stgConfig) } +func (suite *authTestSuite) TestFileSasKey() { + defer suite.cleanupTest() + stgConfig := AzStorageConfig{ + container: storageTestConfigurationParameters.FileContainer, + authConfig: azAuthConfig{ + AuthMode: EAuthType.SAS(), + AccountType: EAccountType.FILE(), + AccountName: storageTestConfigurationParameters.FileAccount, + SASKey: storageTestConfigurationParameters.FileSas, + Endpoint: generateEndpoint(false, storageTestConfigurationParameters.FileAccount, EAccountType.FILE()), + }, + } + suite.validateStorageTest("TestFileSasKey", stgConfig) +} + +func (suite *authTestSuite) TestHttpFileSasKey() { + defer suite.cleanupTest() + stgConfig := AzStorageConfig{ + container: storageTestConfigurationParameters.FileContainer, + authConfig: azAuthConfig{ + AuthMode: EAuthType.SAS(), + AccountType: EAccountType.FILE(), + AccountName: storageTestConfigurationParameters.FileAccount, + SASKey: storageTestConfigurationParameters.FileSas, + UseHTTP: true, + Endpoint: generateEndpoint(true, storageTestConfigurationParameters.FileAccount, EAccountType.FILE()), + }, + } + suite.validateStorageTest("TestHttpFileSasKey", stgConfig) +} + // func (suite *authTestSuite) TestAdlsDirSasKey() { // defer suite.cleanupTest() // assert := assert.New(suite.T()) diff --git a/component/azstorage/azauthkey.go b/component/azstorage/azauthkey.go index 7427d7183..46434abd9 100644 --- a/component/azstorage/azauthkey.go +++ b/component/azstorage/azauthkey.go @@ -38,6 +38,7 @@ import ( "github.com/Azure/azure-storage-azcopy/v10/azbfs" "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-storage-file-go/azfile" ) // Verify that the Auth implement the correct AzAuth interfaces @@ -87,3 +88,25 @@ func (azkey *azAuthBfsKey) getCredential() interface{} { return credential } + +type azAuthFileKey struct { + azAuthKey +} + +// GetCredential : Gets shared key based storage credentials for datalake +func (azkey *azAuthFileKey) getCredential() interface{} { + if azkey.config.AccountKey == "" { + log.Err("azAuthFileKey::getCredential : Shared key for account is empty, cannot authenticate user") + return nil + } + + credential, err := azfile.NewSharedKeyCredential( + azkey.config.AccountName, + azkey.config.AccountKey) + if err != nil { + log.Err("azAuthFileKey::getCredential : Failed to create shared key credentials") + return nil + } + + return credential +} diff --git a/component/azstorage/azauthsas.go b/component/azstorage/azauthsas.go index 70b41f5e7..d9db1f34c 100644 --- a/component/azstorage/azauthsas.go +++ b/component/azstorage/azauthsas.go @@ -40,6 +40,7 @@ import ( "github.com/Azure/azure-storage-azcopy/v10/azbfs" "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-storage-file-go/azfile" ) // Verify that the Auth implement the correct AzAuth interfaces @@ -91,3 +92,17 @@ func (azsas *azAuthBfsSAS) getCredential() interface{} { return azbfs.NewAnonymousCredential() } + +type azAuthFileSAS struct { + azAuthSAS +} + +// GetCredential : Gets SAS based credentials for blob +func (azsas *azAuthFileSAS) getCredential() interface{} { + if azsas.config.SASKey == "" { + log.Err("azAuthFileSAS::getCredential : SAS key for account is empty, cannot authenticate user") + return nil + } + + return azfile.NewAnonymousCredential() +} diff --git a/component/azstorage/block_blob.go b/component/azstorage/block_blob.go index d3bcd0c71..7e7d123f5 100644 --- a/component/azstorage/block_blob.go +++ b/component/azstorage/block_blob.go @@ -851,6 +851,7 @@ func (bb *BlockBlob) WriteFromFile(name string, metadata map[string]string, fi * // based on file-size calculate block size blockSize, err = bb.calculateBlockSize(name, stat.Size()) if err != nil { + log.Err("BlockBlob::WriteFromFile : Failed to get block size %s (%s)", name, err.Error()) return err } } diff --git a/component/azstorage/config.go b/component/azstorage/config.go index 305897df7..c2bd5a425 100644 --- a/component/azstorage/config.go +++ b/component/azstorage/config.go @@ -100,6 +100,10 @@ func (AccountType) ADLS() AccountType { return AccountType(2) } +func (AccountType) FILE() AccountType { + return AccountType(3) +} + func (f AccountType) String() string { return enum.StringInt(f, reflect.TypeOf(f)) } @@ -334,6 +338,8 @@ func ParseAndValidateConfig(az *AzStorage, opt AzStorageOptions) error { opt.Endpoint = fmt.Sprintf("%s.blob.core.windows.net", opt.AccountName) } else if az.stConfig.authConfig.AccountType == EAccountType.ADLS() { opt.Endpoint = fmt.Sprintf("%s.dfs.core.windows.net", opt.AccountName) + } else if az.stConfig.authConfig.AccountType == EAccountType.FILE() { + opt.Endpoint = fmt.Sprintf("%s.file.core.windows.net", opt.AccountName) } } az.stConfig.authConfig.Endpoint = opt.Endpoint diff --git a/component/azstorage/connection.go b/component/azstorage/connection.go index b7b3919f0..6b3e16d56 100644 --- a/component/azstorage/connection.go +++ b/component/azstorage/connection.go @@ -141,6 +141,10 @@ func NewAzStorageConnection(cfg AzStorageConfig) AzConnection { stg := &Datalake{} _ = stg.Configure(cfg) return stg + } else if cfg.authConfig.AccountType == EAccountType.FILE() { + stg := &FileShare{} + _ = stg.Configure(cfg) + return stg } return nil diff --git a/component/azstorage/file_share.go b/component/azstorage/file_share.go new file mode 100644 index 000000000..511eb50a7 --- /dev/null +++ b/component/azstorage/file_share.go @@ -0,0 +1,1003 @@ +/* + _____ _____ _____ ____ ______ _____ ------ + | | | | | | | | | | | | | + | | | | | | | | | | | | | + | --- | | | | |-----| |---- | | |-----| |----- ------ + | | | | | | | | | | | | | + | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ + + + Licensed under the MIT License . + + Copyright © 2020-2023 Microsoft Corporation. All rights reserved. + Author : + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package azstorage + +import ( + "bytes" + "context" + "errors" + "math" + "net/url" + "os" + "path/filepath" + "reflect" + "strings" + "syscall" + "time" + + "github.com/Azure/azure-pipeline-go/pipeline" + "github.com/Azure/azure-storage-azcopy/v10/ste" + "github.com/Azure/azure-storage-fuse/v2/common" + "github.com/Azure/azure-storage-fuse/v2/common/log" + "github.com/Azure/azure-storage-fuse/v2/internal" + "github.com/Azure/azure-storage-fuse/v2/internal/stats_manager" + + "github.com/Azure/azure-storage-file-go/azfile" +) + +const ( + // FileMaxSizeInBytes indicates the maximum size of a file + FileMaxSizeInBytes = 4 * 1024 * 1024 * 1024 * 1024 // 4TiB + + // max number of ranges = max file size / max size for one range + FileShareMaxRanges = FileMaxSizeInBytes / azfile.FileMaxUploadRangeBytes +) + +type FileShare struct { + AzStorageConnection + Auth azAuth + Service azfile.ServiceURL + Share azfile.ShareURL + downloadOptions azfile.DownloadFromAzureFileOptions + rangeLocks common.KeyedMutex +} + +// Verify that FileShare implements AzConnection interface +var _ AzConnection = &FileShare{} + +func (fs *FileShare) Configure(cfg AzStorageConfig) error { + fs.Config = cfg + + fs.downloadOptions = azfile.DownloadFromAzureFileOptions{ + RangeSize: fs.Config.blockSize, + Parallelism: fs.Config.maxConcurrency, + // This is also not set in Blobs, so first investigation needs to go into how this param is used + // TODO: MaxRetryRequestsPerRange: int(fs.Config.maxRetries) + } + + return nil +} + +// For dynamic config update the config here +func (fs *FileShare) UpdateConfig(cfg AzStorageConfig) error { + fs.Config.blockSize = cfg.blockSize + fs.Config.maxConcurrency = cfg.maxConcurrency + fs.Config.defaultTier = cfg.defaultTier + fs.Config.ignoreAccessModifiers = cfg.ignoreAccessModifiers + return nil +} + +// NewCredentialKey : Update the credential key specified by the user +func (fs *FileShare) NewCredentialKey(key, value string) (err error) { + if key == "saskey" { + fs.Auth.setOption(key, value) + // Update the endpoint url from the credential + fs.Endpoint, err = url.Parse(fs.Auth.getEndpoint()) + if err != nil { + log.Err("FileShare::NewCredentialKey : Failed to form base endpoint url (%s)", err.Error()) + return errors.New("failed to form base endpoint url") + } + + // Update the service url + fs.Service = azfile.NewServiceURL(*fs.Endpoint, fs.Pipeline) + + // Update the share url + fs.Share = fs.Service.NewShareURL(fs.Config.container) + } + return nil +} + +// getCredential : Create the credential object +func (fs *FileShare) getCredential() azfile.Credential { + log.Trace("FileShare::getCredential : Getting credential") + + fs.Auth = getAzAuth(fs.Config.authConfig) + if fs.Auth == nil { + log.Err("FileShare::getCredential : Failed to retrieve auth object") + return nil + } + + cred := fs.Auth.getCredential() + if cred == nil { + log.Err("FileShare::getCredential : Failed to get credential") + return nil + } + + return cred.(azfile.Credential) +} + +// NewPipeline creates a Pipeline using the specified credentials and options. +func NewFilePipeline(c azfile.Credential, o azfile.PipelineOptions, ro azfile.RetryOptions) pipeline.Pipeline { + return ste.NewFilePipeline( + c, o, ro, + nil, + ste.NewAzcopyHTTPClient(100), + nil, + ) +} + +// SetupPipeline : Based on the config setup the ***URLs +func (fs *FileShare) SetupPipeline() error { + log.Trace("FileShare::SetupPipeline : Setting up") + var err error + + // Get the credential + cred := fs.getCredential() + if cred == nil { + log.Err("FileShare::SetupPipeline : Failed to get credential") + return errors.New("failed to get credential") + } + + // Create a new pipeline + options, retryOptions := getAzFilePipelineOptions(fs.Config) + fs.Pipeline = NewFilePipeline(cred, options, retryOptions) + if fs.Pipeline == nil { + log.Err("FileShare::SetupPipeline : Failed to create pipeline object") + return errors.New("failed to create pipeline object") + } + + // Get the endpoint url from the credential + fs.Endpoint, err = url.Parse(fs.Auth.getEndpoint()) + if err != nil { + log.Err("FileShare::SetupPipeline : Failed to form base end point url (%s)", err.Error()) + return errors.New("failed to form base end point url") + } + + // Create the service url + fs.Service = azfile.NewServiceURL(*fs.Endpoint, fs.Pipeline) + + // Create the share url + fs.Share = fs.Service.NewShareURL(fs.Config.container) + + return nil +} + +// TestPipeline : Validate the credentials specified in the auth config +func (fs *FileShare) TestPipeline() error { + log.Trace("FileShare::TestPipeline : Validating") + + if fs.Config.mountAllContainers { + return nil + } + + if fs.Share.String() == "" { + log.Err("FileShare::TestPipeline : Share URL is not built, check your credentials") + return nil + } + + marker := (azfile.Marker{}) + listFile, err := fs.Share.NewRootDirectoryURL().ListFilesAndDirectoriesSegment(context.Background(), marker, + azfile.ListFilesAndDirectoriesOptions{MaxResults: 2}) + + if err != nil { + log.Err("FileShare::TestPipeline : Failed to validate account with given auth %s", err.Error()) + return err + } + + if listFile == nil { + log.Info("FileShare::TestPipeline : Share is empty") + } + return nil +} + +func (fs *FileShare) ListContainers() ([]string, error) { + log.Trace("FileShare::ListContainers : Listing containers") + cntList := make([]string, 0) + + marker := azfile.Marker{} + for marker.NotDone() { + resp, err := fs.Service.ListSharesSegment(context.Background(), marker, azfile.ListSharesOptions{}) + if err != nil { + log.Err("FileShare::ListContainers : Failed to get container list %s", err.Error()) + return cntList, err + } + + for _, v := range resp.ShareItems { + cntList = append(cntList, v.Name) + } + + marker = resp.NextMarker + } + + return cntList, nil +} + +// This is just for test, shall not be used otherwise +func (fs *FileShare) SetPrefixPath(path string) error { + log.Trace("FileShare::SetPrefixPath : path %s", path) + fs.Config.prefixPath = path + return nil +} + +// CreateFile : Create a new file in the share/directory +func (fs *FileShare) CreateFile(name string, mode os.FileMode) error { + log.Trace("FileShare::CreateFile : name %s", name) + + fileName, dirPath := getFileAndDirFromPath(filepath.Join(fs.Config.prefixPath, name)) + fileURL := fs.Share.NewDirectoryURL(dirPath).NewFileURL(fileName) + + _, err := fileURL.Create(context.Background(), 0, azfile.FileHTTPHeaders{ + ContentType: getContentType(name), + }, + nil) + + if err != nil { + log.Err("FileShare::CreateFile : Failed to create file %s %s", name, err.Error()) + return err + } + + return nil +} + +func (fs *FileShare) CreateDirectory(name string) error { + log.Trace("FileShare::CreateDirectory : name %s", name) + + metadata := make(azfile.Metadata) + metadata[folderKey] = "true" + + dirURL := fs.Share.NewDirectoryURL(filepath.Join(fs.Config.prefixPath, name)) + + _, err := dirURL.Create(context.Background(), metadata, azfile.SMBProperties{}) + + if err != nil { + log.Err("FileShare::CreateDirectory : Failed to create directory %s %s", name, err.Error()) + return err + } + return nil +} + +// CreateLink : Create a symlink in the share/virtual directory +func (fs *FileShare) CreateLink(source string, target string) error { + log.Trace("FileShare::CreateLink : %s -> %s", source, target) + data := []byte(target) + metadata := make(azfile.Metadata) + metadata[symlinkKey] = "true" + return fs.WriteFromBuffer(source, metadata, data) +} + +// DeleteFile : Delete a file in the share/virtual directory +func (fs *FileShare) DeleteFile(name string) (err error) { + log.Trace("FileShare::DeleteFile : name %s", name) + + fileName, dirPath := getFileAndDirFromPath(filepath.Join(fs.Config.prefixPath, name)) + + fileURL := fs.Share.NewDirectoryURL(dirPath).NewFileURL(fileName) + _, err = fileURL.Delete(context.Background()) + if err != nil { + serr := storeFileErrToErr(err) + if serr == ErrFileNotFound { + log.Err("FileShare::DeleteFile : %s does not exist %s", name, err.Error()) + return syscall.ENOENT + } else { + log.Err("FileShare::DeleteFile : Failed to delete file %s (%s)", name, err.Error()) + return err + } + } + + return nil +} + +// DeleteDirectory : Delete a directory in the share +func (fs *FileShare) DeleteDirectory(name string) (err error) { + log.Trace("FileShare::DeleteDirectory : name %s", name) + + dirURL := fs.Share.NewDirectoryURL(filepath.Join(fs.Config.prefixPath, name)) + + for marker := (azfile.Marker{}); marker.NotDone(); { + listFile, err := dirURL.ListFilesAndDirectoriesSegment(context.Background(), marker, + azfile.ListFilesAndDirectoriesOptions{ + MaxResults: common.MaxDirListCount, + }) + if err != nil { + log.Err("FileShare::DeleteDirectory : Failed to get list of files and directories %s", err.Error()) + return err + } + marker = listFile.NextMarker + + // Process the files returned in this result segment (if the segment is empty, the loop body won't execute) + for _, fileInfo := range listFile.FileItems { + err = fs.DeleteFile(filepath.Join(name, fileInfo.Name)) + if err != nil { + log.Err("FileShare::DeleteDirectory : Failed to delete file %s [%s]", fileInfo.Name, err.Error()) + return err + } + } + + for _, dirInfo := range listFile.DirectoryItems { + err = fs.DeleteDirectory(filepath.Join(filepath.Join(fs.Config.prefixPath, name), dirInfo.Name)) + if err != nil { + log.Err("FileShare::DeleteDirectory : Failed delete subdirectory %s [%s]", dirInfo.Name, err.Error()) + return err + } + } + } + + _, err = dirURL.Delete(context.Background()) + if err != nil { + serr := storeFileErrToErr(err) + if serr == ErrFileNotFound { + log.Err("FileShare::DeleteDirectory : %s does not exist", name) + return syscall.ENOENT + } else { + log.Err("FileShare::DeleteDirectory : Failed to delete directory %s (%s)", name, err.Error()) + return err + } + } + + return nil +} + +// RenameFile : Rename a file +func (fs *FileShare) RenameFile(source string, target string) error { + log.Trace("FileShare::RenameFile : %s -> %s", source, target) + + srcFileName, srcDirPath := getFileAndDirFromPath(filepath.Join(fs.Config.prefixPath, source)) + srcFileURL := fs.Share.NewDirectoryURL(srcDirPath).NewFileURL(srcFileName) + + prop, err := srcFileURL.GetProperties(context.Background()) + if err != nil { + serr := storeFileErrToErr(err) + if serr == ErrFileNotFound { + log.Err("FileShare::RenameFile : Source file %s does not exist", source) + return syscall.ENOENT + } else { + log.Err("FileShare::RenameFile : Failed to get file properties for %s (%s)", source, err.Error()) + return err + } + } + + contentType := prop.ContentType() + replaceIfExists := true + _, err = srcFileURL.Rename(context.Background(), filepath.Join(fs.Config.prefixPath, target), &replaceIfExists, prop.NewMetadata(), &contentType) + + return err +} + +// RenameDirectory : Rename a directory +func (fs *FileShare) RenameDirectory(source string, target string) error { + log.Trace("FileShare::RenameDirectory : %s -> %s", source, target) + + srcDir := fs.Share.NewDirectoryURL(filepath.Join(fs.Config.prefixPath, source)) + prop, err := srcDir.GetProperties(context.Background()) + if err != nil { + serr := storeFileErrToErr(err) + if serr == ErrFileNotFound { + log.Err("FileShare::RenameDirectory : Source directory %s does not exist", source) + return err + } else { + log.Err("FileShare::RenameDirectory : Failed to get directory properties for %s (%s)", source, err.Error()) + return err + } + } + + replaceIfExists := true + _, err = srcDir.Rename(context.Background(), filepath.Join(fs.Config.prefixPath, target), &replaceIfExists, prop.NewMetadata()) + + return err +} + +// GetAttr : Retrieve attributes of a file or directory +func (fs *FileShare) GetAttr(name string) (attr *internal.ObjAttr, err error) { + log.Trace("FileShare::GetAttr : name %s", name) + + fileName, dirPath := getFileAndDirFromPath(filepath.Join(fs.Config.prefixPath, name)) + + fileURL := fs.Share.NewDirectoryURL(dirPath).NewFileURL(fileName) + prop, fileerr := fileURL.GetProperties(context.Background()) + + if fileerr == nil { // file + ctime, err := time.Parse(time.RFC1123, prop.FileChangeTime()) + if err != nil { + ctime = prop.LastModified() + } + crtime, err := time.Parse(time.RFC1123, prop.FileCreationTime()) + if err != nil { + crtime = prop.LastModified() + } + attr = &internal.ObjAttr{ + Path: name, // We don't need to strip the prefixPath here since we pass the input name + Name: filepath.Base(name), + Size: prop.ContentLength(), + Mode: 0, + Mtime: prop.LastModified(), + Atime: prop.LastModified(), + Ctime: ctime, + Crtime: crtime, + Flags: internal.NewFileBitMap(), + MD5: prop.ContentMD5(), + } + parseMetadata(attr, prop.NewMetadata()) + attr.Flags.Set(internal.PropFlagMetadataRetrieved) + attr.Flags.Set(internal.PropFlagModeDefault) + + return attr, nil + } else if storeFileErrToErr(fileerr) == ErrFileNotFound { // directory + dirURL := fs.Share.NewDirectoryURL(filepath.Join(fs.Config.prefixPath, name)) + prop, direrr := dirURL.GetProperties(context.Background()) + + if direrr == nil { + ctime, err := time.Parse(time.RFC1123, prop.FileChangeTime()) + if err != nil { + ctime = prop.LastModified() + } + crtime, err := time.Parse(time.RFC1123, prop.FileCreationTime()) + if err != nil { + crtime = prop.LastModified() + } + attr = &internal.ObjAttr{ + Path: name, + Name: filepath.Base(name), + Size: 4096, + Mode: 0, + Mtime: prop.LastModified(), + Atime: prop.LastModified(), + Ctime: ctime, + Crtime: crtime, + Flags: internal.NewDirBitMap(), + } + parseMetadata(attr, prop.NewMetadata()) + attr.Flags.Set(internal.PropFlagMetadataRetrieved) + attr.Flags.Set(internal.PropFlagModeDefault) + + return attr, nil + } + return attr, syscall.ENOENT + } + // error + log.Err("FileShare::GetAttr : Failed to get file/directory properties for %s (%s)", name, fileerr.Error()) + return attr, fileerr +} + +// List : Get a list of files/directories matching the given prefix +// This fetches the list using a marker so the caller code should handle marker logic +// If count=0 - fetch max entries +func (fs *FileShare) List(prefix string, marker *string, count int32) ([]*internal.ObjAttr, *string, error) { + log.Trace("FileShare::List : prefix %s, marker %s", prefix, func(marker *string) string { + if marker != nil { + return *marker + } else { + return "" + } + }(marker)) + + fileList := make([]*internal.ObjAttr, 0) + + if count == 0 { + count = common.MaxDirListCount + } + + listPath := filepath.Join(fs.Config.prefixPath, prefix) + + listFile, err := fs.Share.NewDirectoryURL(listPath).ListFilesAndDirectoriesSegment(context.Background(), azfile.Marker{Val: marker}, + azfile.ListFilesAndDirectoriesOptions{MaxResults: count}) + + if err != nil { + log.Err("FileShare::List : Failed to list the container with the prefix %s", err.Error()) + return fileList, nil, err + } + + // Process the files returned in this result segment (if the segment is empty, the loop body won't execute) + for _, fileInfo := range listFile.FileItems { + attr := &internal.ObjAttr{ + Path: split(fs.Config.prefixPath, filepath.Join(listPath, fileInfo.Name)), + Name: filepath.Base(fileInfo.Name), + Size: fileInfo.Properties.ContentLength, + Mode: 0, + // Azure file SDK supports 2019.02.02 but time and metadata are only supported by 2020.x.x onwards + // TODO: support times when Azure SDK is updated + Mtime: time.Now(), + Atime: time.Now(), + Ctime: time.Now(), + Crtime: time.Now(), + Flags: internal.NewFileBitMap(), + // Note : List does not return MD5 so we can not populate it. This is fine since MD5 is retrieved via get properties on read + } + + attr.Flags.Set(internal.PropFlagModeDefault) + fileList = append(fileList, attr) + + if attr.IsDir() { + attr.Size = 4096 + } + } + + for _, dirInfo := range listFile.DirectoryItems { + attr := &internal.ObjAttr{ + Path: split(fs.Config.prefixPath, filepath.Join(listPath, dirInfo.Name)), + Name: filepath.Base(dirInfo.Name), + Size: 4096, + Mode: os.ModeDir, + // Azure file SDK supports 2019.02.02 but time, metadata, and dir size are only supported by 2020.x.x onwards + // TODO: support times when Azure SDK is updated + Mtime: time.Now(), + Atime: time.Now(), + Ctime: time.Now(), + Crtime: time.Now(), + Flags: internal.NewDirBitMap(), + } + + attr.Flags.Set(internal.PropFlagModeDefault) + fileList = append(fileList, attr) + } + + return fileList, listFile.NextMarker.Val, nil +} + +// ReadToFile : Download an Azure file to a local file +func (fs *FileShare) ReadToFile(name string, offset int64, count int64, fi *os.File) error { + log.Trace("FileShare::ReadToFile : name %s, offset : %d, count %d", name, offset, count) + //defer exectime.StatTimeCurrentBlock("FileShare::ReadToFile")() + + if offset != 0 { + log.Err("FileShare::ReadToFile : offset is not 0") + return errors.New("offset is not 0") + } + + fileName, dirPath := getFileAndDirFromPath(filepath.Join(fs.Config.prefixPath, name)) + fileURL := fs.Share.NewDirectoryURL(dirPath).NewFileURL(fileName) + + var downloadPtr *int64 = new(int64) + *downloadPtr = 1 + + if common.MonitorBfs() { + fs.downloadOptions.Progress = func(bytesTransferred int64) { + trackDownload(name, bytesTransferred, count, downloadPtr) + } + } + + defer log.TimeTrack(time.Now(), "FileShare::ReadToFile", name) + _, err := azfile.DownloadAzureFileToFile(context.Background(), fileURL, fi, fs.downloadOptions) + + if err != nil { + e := storeFileErrToErr(err) + if e == ErrFileNotFound { + return syscall.ENOENT + } else { + log.Err("FileShare::ReadToFile : Failed to download file %s (%s)", name, err.Error()) + return err + } + } else { + log.Debug("FileShare::ReadToFile : Download complete of file %v", name) + + // store total bytes downloaded so far + azStatsCollector.UpdateStats(stats_manager.Increment, bytesDownloaded, count) + } + + if fs.Config.validateMD5 { + // Compute md5 of local file + localFileMD5, err := getMD5(fi) + if err != nil { + log.Warn("FileShare::ReadToFile : Failed to generate MD5 Sum for %s", name) + } else { + // Get latest properties from container to get the md5 of file + prop, err := fileURL.GetProperties(context.Background()) + if err != nil { + log.Warn("FileShare::ReadToFile : Failed to get properties of file %s [%s]", name, err.Error()) + } else { + remoteFileMD5 := prop.ContentMD5() + if remoteFileMD5 == nil { + log.Warn("FileShare::ReadToFile : Failed to get MD5 Sum for file %s", name) + } else { + // compare md5 and fail is not match + if !reflect.DeepEqual(localFileMD5, remoteFileMD5) { + log.Err("FileShare::ReadToFile : MD5 Sum mismatch %s", name) + return errors.New("md5 sum mismatch on download") + } + } + } + } + } + + return nil +} + +// ReadBuffer : Downloads a file to a buffer +func (fs *FileShare) ReadBuffer(name string, offset int64, len int64) ([]byte, error) { + log.Trace("FileShare::ReadBuffer : name %s", name) + var buff []byte + + if offset != 0 { + log.Err("FileShare::ReadBuffer : offset is not 0") + return buff, errors.New("offset is not 0") + } + + if len == 0 { + attr, err := fs.GetAttr(name) + if err != nil { + return buff, err + } + buff = make([]byte, attr.Size) + } else { + buff = make([]byte, len) + } + + fileName, dirPath := getFileAndDirFromPath(filepath.Join(fs.Config.prefixPath, name)) + fileURL := fs.Share.NewDirectoryURL(dirPath).NewFileURL(fileName) + + _, err := azfile.DownloadAzureFileToBuffer(context.Background(), fileURL, buff, fs.downloadOptions) + + if err != nil { + e := storeFileErrToErr(err) + if e == ErrFileNotFound { + return buff, syscall.ENOENT + } else if e == InvalidRange { + return buff, syscall.ERANGE + } + + log.Err("FileShare::ReadBuffer : Failed to download file %s (%s)", name, err.Error()) + return buff, err + } + + return buff, nil +} + +// ReadInBuffer : Download specific range from a file to a user provided buffer +func (fs *FileShare) ReadInBuffer(name string, offset int64, len int64, data []byte) error { + log.Trace("FileShare::ReadInBuffer : name %s", name) + + if offset != 0 { + log.Err("FileShare::ReadInBuffer : offset is not 0") + return errors.New("offset is not 0") + } + + fileName, dirPath := getFileAndDirFromPath(filepath.Join(fs.Config.prefixPath, name)) + fileURL := fs.Share.NewDirectoryURL(dirPath).NewFileURL(fileName) + + _, err := azfile.DownloadAzureFileToBuffer(context.Background(), fileURL, data, fs.downloadOptions) + + if err != nil { + e := storeFileErrToErr(err) + if e == ErrFileNotFound { + return syscall.ENOENT + } else if e == InvalidRange { + return syscall.ERANGE + } + + log.Err("FileShare::ReadInBuffer : Failed to download file %s (%s)", name, err.Error()) + return err + } + + return nil +} + +// WriteFromFile : Upload local file to Azure file +func (fs *FileShare) WriteFromFile(name string, metadata map[string]string, fi *os.File) error { + log.Trace("FileShare::WriteFromFile : name %s", name) + //defer exectime.StatTimeCurrentBlock("WriteFromFile::WriteFromFile")() + + fileName, dirPath := getFileAndDirFromPath(filepath.Join(fs.Config.prefixPath, name)) + fileURL := fs.Share.NewDirectoryURL(dirPath).NewFileURL(fileName) + + defer log.TimeTrack(time.Now(), "FileShare::WriteFromFile", name) + + var uploadPtr *int64 = new(int64) + *uploadPtr = 1 + + rangeSize := fs.Config.blockSize + // get the size of the file + stat, err := fi.Stat() + if err != nil { + log.Err("FileShare::WriteFromFile : Failed to get file size %s (%s)", name, err.Error()) + return err + } + // if the range size is not set then we configure it based on file size + if rangeSize == 0 { + // based on file-size calculate range size + rangeSize, err = fs.calculateRangeSize(name, stat.Size()) + if err != nil { + log.Err("FileShare::WriteFromFile : Failed to get range size %s (%s)", name, err.Error()) + return err + } + } + + // Compute md5 of this file is requested by user + var md5sum []byte = nil + if fs.Config.updateMD5 { + md5sum, err = getMD5(fi) + if err != nil { + // Md5 sum generation failed so set nil while uploading + log.Warn("FileShare::WriteFromFile : Failed to generate md5 of %s", name) + md5sum = nil + } + } + + uploadOptions := azfile.UploadToAzureFileOptions{ + RangeSize: rangeSize, + Parallelism: fs.Config.maxConcurrency, + Metadata: metadata, + FileHTTPHeaders: azfile.FileHTTPHeaders{ + ContentType: getContentType(name), + ContentMD5: md5sum, + }, + } + + if common.MonitorBfs() && stat.Size() > 0 { + uploadOptions.Progress = func(bytesTransferred int64) { + trackUpload(name, bytesTransferred, stat.Size(), uploadPtr) + } + } + + err = azfile.UploadFileToAzureFile(context.Background(), fi, fileURL, uploadOptions) + + if err != nil { + serr := storeFileErrToErr(err) + if serr == ErrFileAlreadyExists { + log.Err("FileShare::WriteFromFile : %s already exists (%s)", name, err.Error()) + return syscall.EIO + } else { + log.Err("FileShare::WriteFromFile : Failed to upload file %s (%s)", name, err.Error()) + } + return err + } else { + log.Debug("BlockBlob::WriteFromFile : Upload complete of file %v", name) + + // store total bytes uploaded so far + if stat.Size() > 0 { + azStatsCollector.UpdateStats(stats_manager.Increment, bytesUploaded, stat.Size()) + } + } + return nil +} + +// WriteFromBuffer : Upload from a buffer to a file +func (fs *FileShare) WriteFromBuffer(name string, metadata map[string]string, data []byte) (err error) { + log.Trace("FileShare::WriteFromBuffer : name %s", name) + + fileName, dirPath := getFileAndDirFromPath(filepath.Join(fs.Config.prefixPath, name)) + fileURL := fs.Share.NewDirectoryURL(dirPath).NewFileURL(fileName) + + defer log.TimeTrack(time.Now(), "FileShare::WriteFromBuffer", name) + err = azfile.UploadBufferToAzureFile(context.Background(), data, fileURL, azfile.UploadToAzureFileOptions{ + RangeSize: fs.Config.blockSize, + Parallelism: fs.Config.maxConcurrency, + Metadata: metadata, + FileHTTPHeaders: azfile.FileHTTPHeaders{ + ContentType: getContentType(name), + }, + }) + + if err != nil { + log.Err("FileShare::WriteFromBuffer : Failed to upload file %s (%s)", name, err.Error()) + return err + } + + return nil +} + +// ChangeMod : Change mode of a file +func (fs *FileShare) ChangeMod(name string, _ os.FileMode) error { + log.Trace("FileShare::ChangeMod : name %s", name) + + if fs.Config.ignoreAccessModifiers { + // for operations like git clone where transaction fails if chmod is not successful + // return success instead of ENOSYS + return nil + } + + // This is not currently supported for a fileshare account + return syscall.ENOTSUP +} + +// ChangeOwner : Change owner of a file +func (fs *FileShare) ChangeOwner(name string, _ int, _ int) error { + log.Trace("FileShare::ChangeOwner : name %s", name) + + if fs.Config.ignoreAccessModifiers { + // for operations like git clone where transaction fails if chown is not successful + // return success instead of ENOSYS + return nil + } + + // This is not currently supported for a fileshare account + return syscall.ENOTSUP +} + +// StageAndCommit : write data to an Azure file given a list of ranges +func (fs *FileShare) StageAndCommit(name string, bol *common.BlockOffsetList) error { + // lock on the file name so that no stage and commit race condition occur causing failure + fileMtx := fs.rangeLocks.GetLock(name) + fileMtx.Lock() + defer fileMtx.Unlock() + log.Trace("FileShare::StageAndCommit : name %s", name) + + fileName, dirPath := getFileAndDirFromPath(filepath.Join(fs.Config.prefixPath, name)) + fileURL := fs.Share.NewDirectoryURL(dirPath).NewFileURL(fileName) + + var data []byte + + for _, rng := range bol.BlockList { + if rng.Truncated() { + data = make([]byte, rng.EndIndex-rng.StartIndex) + rng.Flags.Clear(common.TruncatedBlock) + } else { + data = rng.Data + } + if rng.Dirty() { + _, err := fileURL.UploadRange(context.Background(), + rng.StartIndex, + bytes.NewReader(data), + nil, + ) + if err != nil { + log.Err("FileShare::StageAndCommit : Failed to upload range to file %s at index %v (%s)", name, rng.StartIndex, err.Error()) + return err + } + rng.Flags.Clear(common.DirtyBlock) + } + } + return nil +} + +// Write : write data at given offset to an Azure file +func (fs *FileShare) Write(options internal.WriteFileOptions) (err error) { + name := options.Handle.Path + offset := options.Offset + data := options.Data + length := int64(len(options.Data)) + + defer log.TimeTrack(time.Now(), "FileShare::Write", options.Handle.Path) + log.Trace("FileShare::Write : name %s offset %v", name, offset) + + if len(data) == 0 { + return nil + } + + fileOffsets, err := fs.GetFileBlockOffsets(name) + if err != nil { + log.Err("FileShare::Write : Failed to get file range offsets %s", err.Error()) + return err + } + + _, _, exceedsFileBlocks, _ := fileOffsets.FindBlocksToModify(offset, length) + if exceedsFileBlocks { + err = fs.TruncateFile(name, offset+length) + if err != nil { + log.Err("FileShare::Write : Failed to truncate Azure file %s", err.Error()) + return err + } + } + + fileName, dirPath := getFileAndDirFromPath(filepath.Join(fs.Config.prefixPath, name)) + fileURL := fs.Share.NewDirectoryURL(dirPath).NewFileURL(fileName) + + _, err = fileURL.UploadRange(context.Background(), options.Offset, bytes.NewReader(data), nil) + if err != nil { + log.Err("FileShare::Write : Failed to write data to Azure file %s", err.Error()) + return err + } + + return nil +} + +// GetFileBlockOffsets : store file range list and corresponding offsets +func (fs *FileShare) GetFileBlockOffsets(name string) (shareFileRangeList *common.BlockOffsetList, err error) { + log.Trace("FileShare::GetFileBlockOffsets : name %s", name) + rangeList := common.BlockOffsetList{} + + fileName, dirPath := getFileAndDirFromPath(filepath.Join(fs.Config.prefixPath, name)) + fileURL := fs.Share.NewDirectoryURL(dirPath).NewFileURL(fileName) + + storageRangeList, err := fileURL.GetRangeList( + context.Background(), 0, 0) + if err != nil { + log.Err("FileShare::GetFileBlockOffsets : Failed to get range list %s ", name, err.Error()) + return &common.BlockOffsetList{}, err + } + + if len(storageRangeList.Ranges) == 0 { + rangeList.Flags.Set(common.SmallFile) + return &rangeList, nil + } + for _, rng := range storageRangeList.Ranges { + fileRng := &common.Block{ + StartIndex: rng.Start, + EndIndex: rng.End, + } + rangeList.BlockList = append(rangeList.BlockList, fileRng) + } + + return &rangeList, nil +} + +// TruncateFile : resize the file to a smaller, equal, or bigger size +func (fs *FileShare) TruncateFile(name string, size int64) (err error) { + log.Trace("FileShare::TruncateFile : name=%s, size=%d", name, size) + + fileName, dirPath := getFileAndDirFromPath(filepath.Join(fs.Config.prefixPath, name)) + fileURL := fs.Share.NewDirectoryURL(dirPath).NewFileURL(fileName) + + _, err = fileURL.Resize(context.Background(), size) + if err != nil { + log.Err("FileShare::TruncateFile : failed to resize file %s", name) + return err + } + return nil +} + +// getFileAndDirFromPath : Helper that separates directory/directories and file name of a given file/directory path +// Covers case where name param includes subdirectories and not just the file name +// Only call when path includes file +// Assumes files don't have "/" at the end whereas directories do +func getFileAndDirFromPath(completePath string) (fileName string, dirPath string) { + if completePath == "" { + return "", "" + } + + splitPath := strings.Split(completePath, "/") + + dirArray := splitPath[:len(splitPath)-1] + dirPath = strings.Join(dirArray, "/") // doesn't end with "/" + + fileName = filepath.Base(completePath) + + return fileName, dirPath +} + +// calculateRangeSize : calculates range size of the file based on file size +func (fs *FileShare) calculateRangeSize(name string, fileSize int64) (rangeSize int64, err error) { + if fileSize > FileMaxSizeInBytes { + log.Err("FileShare::calculateRangeSize : buffer is too large to upload to an Azure file %s", name) + err = errors.New("buffer is too large to upload to an Azure file") + return 0, err + } + + if fileSize <= azfile.FileMaxUploadRangeBytes { + // Files up to 4MB can be uploaded as a single range + rangeSize = azfile.FileMaxUploadRangeBytes + } else { + // buffer / max number of file ranges = range size to use for all ranges + rangeSize = int64(math.Ceil(float64(fileSize) / float64(FileShareMaxRanges))) + + if rangeSize < azfile.FileMaxUploadRangeBytes { + // Range size is smaller than 4MB then consider 4MB as default + rangeSize = azfile.FileMaxUploadRangeBytes + } else { + if (rangeSize & (-8)) != 0 { + // EXTRA : round off the range size to next higher multiple of 8. + // No reason to do so; assuming odd numbers in range size will not be good on server end + rangeSize = (rangeSize + 7) & (-8) + } + + if rangeSize > azfile.FileMaxUploadRangeBytes { + // After rounding off the rangeSize has become bigger then max allowed range size. + log.Err("FileShare::calculateRangeSize : rangeSize exceeds max allowed range size for %s", name) + err = errors.New("range size is too large to upload to a file") + return 0, err + } + } + } + + log.Info("FileShare::calculateRangeSize : %s size %lu, blockSize %lu", name, fileSize, rangeSize) + return rangeSize, nil +} diff --git a/component/azstorage/file_share_test.go b/component/azstorage/file_share_test.go new file mode 100644 index 000000000..339603a81 --- /dev/null +++ b/component/azstorage/file_share_test.go @@ -0,0 +1,2105 @@ +//go:build !authtest +// +build !authtest + +/* + _____ _____ _____ ____ ______ _____ ------ + | | | | | | | | | | | | | + | | | | | | | | | | | | | + | --- | | | | |-----| |---- | | |-----| |----- ------ + | | | | | | | | | | | | | + | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ + + + Licensed under the MIT License . + + Copyright © 2020-2023 Microsoft Corporation. All rights reserved. + Author : + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package azstorage + +import ( + "container/list" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "os" + "strings" + "syscall" + "testing" + "time" + + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-storage-fuse/v2/common" + "github.com/Azure/azure-storage-fuse/v2/common/log" + "github.com/Azure/azure-storage-fuse/v2/internal" + "github.com/Azure/azure-storage-fuse/v2/internal/handlemap" + + "github.com/Azure/azure-storage-file-go/azfile" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type fileTestSuite struct { + suite.Suite + assert *assert.Assertions + az *AzStorage + serviceUrl azfile.ServiceURL + shareUrl azfile.ShareURL + config string + container string +} + +func (s *fileTestSuite) SetupTest() { + // Logging config + cfg := common.LogConfig{ + FilePath: "./logfile.txt", + MaxFileSize: 10, + FileCount: 10, + Level: common.ELogLevel.LOG_DEBUG(), + } + log.SetDefaultLogger("base", cfg) + + homeDir, err := os.UserHomeDir() + if err != nil { + fmt.Println("Unable to get home directory") + os.Exit(1) + } + cfgFile, err := os.Open(homeDir + "/azuretest.json") + if err != nil { + fmt.Println("Unable to open config file") + os.Exit(1) + } + + cfgData, _ := ioutil.ReadAll(cfgFile) + err = json.Unmarshal(cfgData, &storageTestConfigurationParameters) + if err != nil { + fmt.Println("Failed to parse the config file") + os.Exit(1) + } + + cfgFile.Close() + s.setupTestHelper("", "", true) +} + +func (s *fileTestSuite) setupTestHelper(configuration string, container string, create bool) { + if container == "" { + container = generateContainerName() + } + s.container = container + if configuration == "" { + configuration = fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + } + s.config = configuration + + s.assert = assert.New(s.T()) + + s.az, _ = newTestAzStorage(configuration) + s.az.Start(ctx) // Note: Start->TestValidation will fail but it doesn't matter. We are creating the container a few lines below anyway. + // We could create the container before but that requires rewriting the code to new up a service client. + + s.serviceUrl = s.az.storage.(*FileShare).Service // Grab the service client to do some validation + s.shareUrl = s.serviceUrl.NewShareURL(s.container) + if create { + s.shareUrl.Create(ctx, azfile.Metadata{}, 0) + } +} + +func (s *fileTestSuite) tearDownTestHelper(delete bool) { + s.az.Stop() + if delete { + s.shareUrl.Delete(ctx, azfile.DeleteSnapshotsOptionNone) + } +} + +func (s *fileTestSuite) cleanupTest() { + s.tearDownTestHelper(true) + log.Destroy() +} + +// func (s *fileTestSuite) TestCleanupShares() { +// marker := azfile.Marker{} +// for { +// shareList, _ := s.serviceUrl.ListSharesSegment(ctx, marker, azfile.ListSharesOptions{Prefix: "fuseutc"}) + +// for _, share := range shareList.ShareItems { +// fmt.Println(share.Name) +// s.serviceUrl.NewShareURL(share.Name).Delete(ctx, azfile.DeleteSnapshotsOptionInclude) +// } +// new_marker := shareList.NextMarker +// marker = new_marker + +// if !marker.NotDone() { +// break +// } +// } +// } + +func (s *fileTestSuite) TestInvalidRangeSize() { + defer s.cleanupTest() + // max range size is 4MiB + configuration := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n block-size-mb: 5\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + _, err := newTestAzStorage(configuration) + s.assert.NotNil(err) +} + +func (s *fileTestSuite) TestDefault() { + defer s.cleanupTest() + s.assert.Equal(storageTestConfigurationParameters.FileAccount, s.az.stConfig.authConfig.AccountName) + s.assert.Equal(EAccountType.FILE(), s.az.stConfig.authConfig.AccountType) + s.assert.False(s.az.stConfig.authConfig.UseHTTP) + s.assert.Equal(storageTestConfigurationParameters.FileKey, s.az.stConfig.authConfig.AccountKey) + s.assert.Empty(s.az.stConfig.authConfig.SASKey) + s.assert.Empty(s.az.stConfig.authConfig.ApplicationID) + s.assert.Empty(s.az.stConfig.authConfig.ResourceID) + s.assert.Empty(s.az.stConfig.authConfig.ActiveDirectoryEndpoint) + s.assert.Empty(s.az.stConfig.authConfig.ClientSecret) + s.assert.Empty(s.az.stConfig.authConfig.TenantID) + s.assert.Empty(s.az.stConfig.authConfig.ClientID) + s.assert.EqualValues("https://"+s.az.stConfig.authConfig.AccountName+".file.core.windows.net/", s.az.stConfig.authConfig.Endpoint) + s.assert.Equal(EAuthType.KEY(), s.az.stConfig.authConfig.AuthMode) + s.assert.Equal(s.container, s.az.stConfig.container) + s.assert.Empty(s.az.stConfig.prefixPath) + s.assert.EqualValues(0, s.az.stConfig.blockSize) + s.assert.EqualValues(32, s.az.stConfig.maxConcurrency) + s.assert.EqualValues(AccessTiers["none"], s.az.stConfig.defaultTier) + s.assert.EqualValues(0, s.az.stConfig.cancelListForSeconds) + s.assert.EqualValues(5, s.az.stConfig.maxRetries) + s.assert.EqualValues(900, s.az.stConfig.maxTimeout) + s.assert.EqualValues(4, s.az.stConfig.backoffTime) + s.assert.EqualValues(60, s.az.stConfig.maxRetryDelay) + s.assert.Empty(s.az.stConfig.proxyAddress) +} + +func (s *fileTestSuite) TestNoEndpoint() { + defer s.cleanupTest() + // Setup + s.tearDownTestHelper(false) // Don't delete the generated container. + config := fmt.Sprintf("azstorage:\n account-name: %s\n type: file\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + s.setupTestHelper(config, s.container, true) + + err := s.az.storage.TestPipeline() + s.assert.Nil(err) +} + +func (s *fileTestSuite) TestListShares() { + defer s.cleanupTest() + // Setup + num := 10 + prefix := generateContainerName() + for i := 0; i < num; i++ { + c := s.serviceUrl.NewShareURL(prefix + fmt.Sprint(i)) + c.Create(ctx, nil, 0) + defer c.Delete(ctx, azfile.DeleteSnapshotsOptionNone) + } + + containers, err := s.az.ListContainers() + + s.assert.Nil(err) + s.assert.NotNil(containers) + s.assert.True(len(containers) >= num) + count := 0 + for _, c := range containers { + if strings.HasPrefix(c, prefix) { + count++ + } + } + s.assert.EqualValues(num, count) +} + +// TODO : ListContainersHuge: Maybe this is overkill? + +func (s *fileTestSuite) TestCreateDir() { + defer s.cleanupTest() + // Testing dir and dir/ + var paths = []string{generateDirectoryName(), generateDirectoryName() + "/"} + for _, path := range paths { + log.Debug(path) + s.Run(path, func() { + err := s.az.CreateDir(internal.CreateDirOptions{Name: path}) + + s.assert.Nil(err) + // Directory should be in the account + dir := s.shareUrl.NewDirectoryURL(internal.TruncateDirName(path)) + + props, err := dir.GetProperties(ctx) + s.assert.Nil(err) + s.assert.NotNil(props) + s.assert.NotEmpty(props.NewMetadata()) + s.assert.Contains(props.NewMetadata(), folderKey) + s.assert.EqualValues("true", props.NewMetadata()[folderKey]) + }) + } +} + +func (s *fileTestSuite) TestDeleteDir() { + defer s.cleanupTest() + // Testing dir and dir/ + var paths = []string{generateDirectoryName(), generateDirectoryName() + "/"} + for _, path := range paths { + log.Debug(path) + s.Run(path, func() { + s.az.CreateDir(internal.CreateDirOptions{Name: path}) + + err := s.az.DeleteDir(internal.DeleteDirOptions{Name: path}) + + s.assert.Nil(err) + // Directory should not be in the account + dir := s.shareUrl.NewDirectoryURL(internal.TruncateDirName(path)) + _, err = dir.GetProperties(ctx) + s.assert.NotNil(err) + }) + } +} + +func (s *fileTestSuite) setupHierarchy(base string) (*list.List, *list.List, *list.List) { + // Hierarchy looks as follows, a = base + // a/ + // a/c1/ + // a/c1/gc1 + // a/c2 + // ab/ + // ab/c1 + // ac + err := s.az.CreateDir(internal.CreateDirOptions{Name: base}) + s.assert.Nil(err) + c1 := base + "/c1" + err = s.az.CreateDir(internal.CreateDirOptions{Name: c1}) + s.assert.Nil(err) + gc1 := c1 + "/gc1" + _, err = s.az.CreateFile(internal.CreateFileOptions{Name: gc1}) + s.assert.Nil(err) + c2 := base + "/c2" + _, err = s.az.CreateFile(internal.CreateFileOptions{Name: c2}) + s.assert.Nil(err) + abPath := base + "b" + err = s.az.CreateDir(internal.CreateDirOptions{Name: abPath}) + s.assert.Nil(err) + abc1 := abPath + "/c1" + _, err = s.az.CreateFile(internal.CreateFileOptions{Name: abc1}) + s.assert.Nil(err) + acPath := base + "c" + _, err = s.az.CreateFile(internal.CreateFileOptions{Name: acPath}) + s.assert.Nil(err) + + a, ab, ac := generateNestedDirectory(base) + + // Validate the paths were setup correctly and all paths exist + for p := a.Front(); p != nil; p = p.Next() { + _, err := s.shareUrl.NewDirectoryURL(p.Value.(string)).GetProperties(ctx) + if err != nil { + + fileName, dirPath := getFileAndDirFromPath(p.Value.(string)) + _, err := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + s.assert.Nil(err) + } else { + s.assert.Nil(err) + } + } + for p := ab.Front(); p != nil; p = p.Next() { + _, err := s.shareUrl.NewDirectoryURL(p.Value.(string)).GetProperties(ctx) + if err != nil { + fileName, dirPath := getFileAndDirFromPath(p.Value.(string)) + _, err := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + s.assert.Nil(err) + } else { + s.assert.Nil(err) + } + } + for p := ac.Front(); p != nil; p = p.Next() { + _, err := s.shareUrl.NewDirectoryURL(p.Value.(string)).GetProperties(ctx) + if err != nil { + fileName, dirPath := getFileAndDirFromPath(p.Value.(string)) + _, err := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + s.assert.Nil(err) + } else { + s.assert.Nil(err) + } + } + return a, ab, ac +} + +func (s *fileTestSuite) TestDeleteDirHierarchy() { + defer s.cleanupTest() + // Setup + base := generateDirectoryName() + a, ab, ac := s.setupHierarchy(base) + + err := s.az.DeleteDir(internal.DeleteDirOptions{Name: base}) + s.assert.Nil(err) + + // a paths should be deleted + for p := a.Front(); p != nil; p = p.Next() { + _, direrr := s.shareUrl.NewDirectoryURL(p.Value.(string)).GetProperties(ctx) + + fileName, dirPath := getFileAndDirFromPath(p.Value.(string)) + _, fileerr := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + + s.assert.NotNil(direrr) + s.assert.NotNil(fileerr) + } + + ab.PushBackList(ac) // ab and ac paths should exist + for p := ab.Front(); p != nil; p = p.Next() { + _, err = s.shareUrl.NewDirectoryURL(p.Value.(string)).GetProperties(ctx) + + if err != nil { + fileName, dirPath := getFileAndDirFromPath(p.Value.(string)) + _, err := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + s.assert.Nil(err) + } else { + s.assert.Nil(err) + } + } +} + +func (s *fileTestSuite) TestDeleteSubDirPrefixPath() { + defer s.cleanupTest() + // Setup + base := generateDirectoryName() + a, ab, ac := s.setupHierarchy(base) + + s.az.storage.SetPrefixPath(base) + + err := s.az.DeleteDir(internal.DeleteDirOptions{Name: "c1"}) + s.assert.Nil(err) + + // a paths under c1 should be deleted + for p := a.Front(); p != nil; p = p.Next() { + path := p.Value.(string) + + // 4 cases: nonexistent file & directory, existing file & directory + _, direrr := s.shareUrl.NewDirectoryURL(path).GetProperties(ctx) + + if direrr != nil { + fileName, dirPath := getFileAndDirFromPath(path) + _, fileerr := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + if fileerr == nil { // existing file + if strings.HasPrefix(path, base+"/c1") { + s.assert.NotNil(fileerr) + } else { + s.assert.Nil(fileerr) + } + break + } + + if strings.HasPrefix(path, base+"/c1") { // nonexistent file and dir + s.assert.NotNil(direrr) + s.assert.NotNil(fileerr) + } else { + s.assert.Nil(direrr) + s.assert.Nil(fileerr) + } + } else { // existing dir + if strings.HasPrefix(path, base+"/c1") { + s.assert.NotNil(direrr) + } else { + s.assert.Nil(direrr) + } + } + + } + + ab.PushBackList(ac) // ab and ac paths should exist + for p := ab.Front(); p != nil; p = p.Next() { + _, err = s.shareUrl.NewDirectoryURL(p.Value.(string)).GetProperties(ctx) + + if err != nil { + fileName, dirPath := getFileAndDirFromPath(p.Value.(string)) + _, err := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + s.assert.Nil(err) + } else { + s.assert.Nil(err) + } + } +} + +func (s *fileTestSuite) TestDeleteDirError() { + defer s.cleanupTest() + // Setup + name := generateDirectoryName() + + err := s.az.DeleteDir(internal.DeleteDirOptions{Name: name}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOENT, storeFileErrToErr(err)) + + // Directory should not be in the account + dir := s.shareUrl.NewDirectoryURL(name) + _, err = dir.GetProperties(ctx) + s.assert.NotNil(err) +} + +func (s *fileTestSuite) TestIsDirEmpty() { + defer s.cleanupTest() + // Setup + name := generateDirectoryName() + s.az.CreateDir(internal.CreateDirOptions{Name: name}) + + // Testing dir and dir/ + var paths = []string{name, name + "/"} + for _, path := range paths { + log.Debug(path) + s.Run(path, func() { + empty := s.az.IsDirEmpty(internal.IsDirEmptyOptions{Name: name}) + + s.assert.True(empty) + }) + } +} + +func (s *fileTestSuite) TestIsDirEmptyFalse() { + defer s.cleanupTest() + // Setup + name := generateDirectoryName() + s.az.CreateDir(internal.CreateDirOptions{Name: name}) + + file := name + "/" + generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: file}) + + empty := s.az.IsDirEmpty(internal.IsDirEmptyOptions{Name: name}) + + s.assert.False(empty) +} + +func (s *fileTestSuite) TestIsDirEmptyError() { + defer s.cleanupTest() + // Setup + name := generateDirectoryName() + + empty := s.az.IsDirEmpty(internal.IsDirEmptyOptions{Name: name}) + s.assert.False(empty) // Note: FileShare fails for nonexistent directory. + // FileShare behaves differently from BlockBlob (See comment in BlockBlob.List). + + // Directory should not be in the account + dir := s.shareUrl.NewDirectoryURL(name) + _, err := dir.GetProperties(ctx) + s.assert.NotNil(err) +} + +func (s *fileTestSuite) TestReadDir() { + defer s.cleanupTest() + // This tests the default listBlocked = 0. It should return the expected paths. + // Setup + name := generateDirectoryName() + s.az.CreateDir(internal.CreateDirOptions{Name: name}) + childName := name + "/" + generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: childName}) + + // Testing dir and dir/ + var paths = []string{name, name + "/"} + for _, path := range paths { + log.Debug(path) + s.Run(path, func() { + entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: name}) + s.assert.Nil(err) + s.assert.EqualValues(1, len(entries)) + }) + } +} + +func (s *fileTestSuite) TestReadDirHierarchy() { + defer s.cleanupTest() + // Setup + base := generateDirectoryName() + s.setupHierarchy(base) + + // TODO: test metadata retrieval once SDK is updated (in this method and others below) + + // ReadDir only reads the first level of the hierarchy + entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: base}) + s.assert.Nil(err) + s.assert.EqualValues(2, len(entries)) + // Check the file + s.assert.EqualValues(base+"/c2", entries[0].Path) + s.assert.EqualValues("c2", entries[0].Name) + s.assert.False(entries[0].IsDir()) + s.assert.False(entries[0].IsMetadataRetrieved()) + s.assert.True(entries[0].IsModeDefault()) + // Check the dir + s.assert.EqualValues(base+"/c1", entries[1].Path) + s.assert.EqualValues("c1", entries[1].Name) + s.assert.True(entries[1].IsDir()) + s.assert.False(entries[1].IsMetadataRetrieved()) + s.assert.True(entries[1].IsModeDefault()) +} + +func (s *fileTestSuite) TestReadDirRoot() { + defer s.cleanupTest() + // Setup + base := generateDirectoryName() + s.setupHierarchy(base) + + // Testing dir and dir/ + var paths = []string{"", "/"} + for _, path := range paths { + log.Debug(path) + s.Run(path, func() { + // ReadDir only reads the first level of the hierarchy + entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: ""}) + s.assert.Nil(err) + s.assert.EqualValues(3, len(entries)) + // Check the base dir + s.assert.EqualValues(base, entries[1].Path) + s.assert.EqualValues(base, entries[1].Name) + s.assert.True(entries[1].IsDir()) + s.assert.False(entries[1].IsMetadataRetrieved()) + s.assert.True(entries[1].IsModeDefault()) + // Check the baseb dir + s.assert.EqualValues(base+"b", entries[2].Path) + s.assert.EqualValues(base+"b", entries[2].Name) + s.assert.True(entries[2].IsDir()) + s.assert.False(entries[2].IsMetadataRetrieved()) + s.assert.True(entries[2].IsModeDefault()) + // Check the basec file + s.assert.EqualValues(base+"c", entries[0].Path) + s.assert.EqualValues(base+"c", entries[0].Name) + s.assert.False(entries[0].IsDir()) + s.assert.False(entries[0].IsMetadataRetrieved()) + s.assert.True(entries[0].IsModeDefault()) + }) + } +} + +func (s *fileTestSuite) TestReadDirSubDir() { + defer s.cleanupTest() + // Setup + base := generateDirectoryName() + s.setupHierarchy(base) + + // ReadDir only reads the first level of the hierarchy + entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: base + "/c1"}) + s.assert.Nil(err) + s.assert.EqualValues(1, len(entries)) + // Check the dir + s.assert.EqualValues(base+"/c1"+"/gc1", entries[0].Path) + s.assert.EqualValues("gc1", entries[0].Name) + s.assert.False(entries[0].IsDir()) + s.assert.False(entries[0].IsMetadataRetrieved()) + s.assert.True(entries[0].IsModeDefault()) +} + +func (s *fileTestSuite) TestReadDirSubDirPrefixPath() { + defer s.cleanupTest() + // Setup + base := generateDirectoryName() + s.setupHierarchy(base) + + s.az.storage.SetPrefixPath(base) + + // ReadDir only reads the first level of the hierarchy + entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: "/c1"}) + s.assert.Nil(err) + s.assert.EqualValues(1, len(entries)) + // Check the dir + s.assert.EqualValues("c1"+"/gc1", entries[0].Path) + s.assert.EqualValues("gc1", entries[0].Name) + s.assert.False(entries[0].IsDir()) + s.assert.False(entries[0].IsMetadataRetrieved()) + s.assert.True(entries[0].IsModeDefault()) +} + +func (s *fileTestSuite) TestReadDirError() { + defer s.cleanupTest() + // Setup + name := generateDirectoryName() + + entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: name}) + + s.assert.NotNil(err) // Note: FileShare fails for nonexistent directory. + // FileShare behaves differently from BlockBlob (See comment in BlockBlob.List). + s.assert.Empty(entries) + // Directory should not be in the account + dir := s.shareUrl.NewDirectoryURL(name) + _, err = dir.GetProperties(ctx) + s.assert.NotNil(err) +} + +func (s *fileTestSuite) TestReadDirListBlocked() { + defer s.cleanupTest() + // Setup + s.tearDownTestHelper(false) // Don't delete the generated container. + + listBlockedTime := 10 + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n block-list-on-mount-sec: %d\n fail-unsupported-op: true\n", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container, listBlockedTime) + s.setupTestHelper(config, s.container, true) + + name := generateDirectoryName() + s.az.CreateDir(internal.CreateDirOptions{Name: name}) + childName := name + "/" + generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: childName}) + + entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: name}) + s.assert.Nil(err) + s.assert.EqualValues(0, len(entries)) // Since we block the list, it will return an empty list. +} + +func (s *fileTestSuite) TestRenameDir() { + defer s.cleanupTest() + // Test handling "dir" and "dir/" + var inputs = []struct { + src string + dst string + }{ + {src: generateDirectoryName(), dst: generateDirectoryName()}, + {src: generateDirectoryName() + "/", dst: generateDirectoryName()}, + {src: generateDirectoryName(), dst: generateDirectoryName() + "/"}, + {src: generateDirectoryName() + "/", dst: generateDirectoryName() + "/"}, + } + + for _, input := range inputs { + s.Run(input.src+"->"+input.dst, func() { + // Setup + s.az.CreateDir(internal.CreateDirOptions{Name: input.src}) + + err := s.az.RenameDir(internal.RenameDirOptions{Src: input.src, Dst: input.dst}) + s.assert.Nil(err) + // Src should not be in the account + dir := s.shareUrl.NewDirectoryURL(internal.TruncateDirName(input.src)) + _, err = dir.GetProperties(ctx) + s.assert.NotNil(err) + + // Dst should be in the account + dir = s.shareUrl.NewDirectoryURL(internal.TruncateDirName(input.dst)) + _, err = dir.GetProperties(ctx) + s.assert.Nil(err) + }) + } + +} + +func (s *fileTestSuite) TestRenameDirHierarchy() { + defer s.cleanupTest() + // Setup + baseSrc := generateDirectoryName() + aSrc, abSrc, acSrc := s.setupHierarchy(baseSrc) + baseDst := generateDirectoryName() + aDst, abDst, acDst := generateNestedDirectory(baseDst) + + err := s.az.RenameDir(internal.RenameDirOptions{Src: baseSrc, Dst: baseDst}) + s.assert.Nil(err) + + // Source + // aSrc paths should be deleted + for p := aSrc.Front(); p != nil; p = p.Next() { + _, direrr := s.shareUrl.NewDirectoryURL(p.Value.(string)).GetProperties(ctx) + + fileName, dirPath := getFileAndDirFromPath(p.Value.(string)) + _, fileerr := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + + s.assert.NotNil(direrr) + s.assert.NotNil(fileerr) + } + abSrc.PushBackList(acSrc) // abSrc and acSrc paths should exist + for p := abSrc.Front(); p != nil; p = p.Next() { + _, direrr := s.shareUrl.NewDirectoryURL(p.Value.(string)).GetProperties(ctx) + if direrr != nil { + fileName, dirPath := getFileAndDirFromPath(p.Value.(string)) + _, fileerr := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + + if fileerr != nil { // nonexistent file and dir + s.assert.NotNil(fileerr) + s.assert.NotNil(direrr) + } else { // existing file + s.assert.Nil(fileerr) + s.assert.NotNil(direrr) + } + } else { // existing dir + s.assert.Nil(direrr) + } + } + // Destination + // aDst paths should exist + for p := aDst.Front(); p != nil; p = p.Next() { + _, direrr := s.shareUrl.NewDirectoryURL(p.Value.(string)).GetProperties(ctx) + if direrr != nil { + fileName, dirPath := getFileAndDirFromPath(p.Value.(string)) + _, fileerr := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + + if fileerr != nil { // nonexistent file and dir + s.assert.NotNil(fileerr) + s.assert.NotNil(direrr) + } else { // existing file + s.assert.Nil(fileerr) + s.assert.NotNil(direrr) + } + } else { // existing dir + s.assert.Nil(direrr) + } + } + abDst.PushBackList(acDst) // abDst and acDst paths should not exist + for p := abDst.Front(); p != nil; p = p.Next() { + _, direrr := s.shareUrl.NewDirectoryURL(p.Value.(string)).GetProperties(ctx) + + fileName, dirPath := getFileAndDirFromPath(p.Value.(string)) + _, fileerr := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + + s.assert.NotNil(direrr) + s.assert.NotNil(fileerr) + } +} + +func (s *fileTestSuite) TestRenameDirSubDirPrefixPath() { + defer s.cleanupTest() + // Setup + baseSrc := generateDirectoryName() + aSrc, abSrc, acSrc := s.setupHierarchy(baseSrc) + baseDst := generateDirectoryName() + + s.az.storage.SetPrefixPath(baseSrc) + + err := s.az.RenameDir(internal.RenameDirOptions{Src: "c1", Dst: baseDst}) + s.assert.Nil(err) + + // Source + // aSrc paths under c1 should be deleted + for p := aSrc.Front(); p != nil; p = p.Next() { + path := p.Value.(string) + _, direrr := s.shareUrl.NewDirectoryURL(path).GetProperties(ctx) + + if direrr != nil { + fileName, dirPath := getFileAndDirFromPath(path) + _, fileerr := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + if fileerr == nil { // existing file + if strings.HasPrefix(path, baseDst+"/c1") { + s.assert.NotNil(fileerr) + } else { + s.assert.Nil(fileerr) + } + break + } + // nonexistent file and dir + if strings.HasPrefix(path, baseSrc+"/c1") { + s.assert.NotNil(direrr) + s.assert.NotNil(fileerr) + } else { + s.assert.Nil(direrr) + s.assert.Nil(fileerr) + } + } else { // existing dir + if strings.HasPrefix(path, baseSrc+"/c1") { + s.assert.NotNil(direrr) + } else { + s.assert.Nil(direrr) + } + } + + if strings.HasPrefix(path, baseSrc+"/c1") { // nonexistent dir + s.assert.NotNil(direrr) + } else { // existing dir + s.assert.Nil(direrr) + } + } + + abSrc.PushBackList(acSrc) // abSrc and acSrc paths should exist + for p := abSrc.Front(); p != nil; p = p.Next() { + _, err = s.shareUrl.NewDirectoryURL(p.Value.(string)).GetProperties(ctx) + + if err != nil { + fileName, dirPath := getFileAndDirFromPath(p.Value.(string)) + _, err := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + s.assert.Nil(err) + } else { + s.assert.Nil(err) + } + } + // Destination + // aDst paths should exist -> aDst and aDst/gc1 + _, err = s.shareUrl.NewDirectoryURL(baseSrc + "/" + baseDst).GetProperties(ctx) + s.assert.Nil(err) + fileName, dirPath := getFileAndDirFromPath(baseSrc + "/" + baseDst + "/gc1") + _, err = s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName).GetProperties(ctx) + s.assert.Nil(err) +} + +func (s *fileTestSuite) TestRenameDirTargetExistsError() { + defer s.cleanupTest() + // Setup + src := generateDirectoryName() + dst := generateDirectoryName() + + s.az.CreateDir(internal.CreateDirOptions{Name: dst}) + + err := s.az.RenameDir(internal.RenameDirOptions{Src: src, Dst: dst}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOENT, storeFileErrToErr(err)) + + // Only target directory should be in the account + dir := s.shareUrl.NewDirectoryURL(dst) + _, err = dir.GetProperties(ctx) + s.assert.Nil(err) + + dir = s.shareUrl.NewDirectoryURL(src) + _, err = dir.GetProperties(ctx) + s.assert.NotNil(err) +} + +func (s *fileTestSuite) TestRenameDirError() { + defer s.cleanupTest() + // Setup + src := generateDirectoryName() + dst := generateDirectoryName() + + err := s.az.RenameDir(internal.RenameDirOptions{Src: src, Dst: dst}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOENT, storeFileErrToErr(err)) + + // Neither directory should be in the account + dir := s.shareUrl.NewDirectoryURL(dst) + _, err = dir.GetProperties(ctx) + s.assert.NotNil(err) + + dir = s.shareUrl.NewDirectoryURL(src) + _, err = dir.GetProperties(ctx) + s.assert.NotNil(err) + +} + +func (s *fileTestSuite) TestCreateFile() { + defer s.cleanupTest() + // Setup + name := generateFileName() + + h, err := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + + s.assert.Nil(err) + s.assert.NotNil(h) + s.assert.EqualValues(name, h.Path) + s.assert.EqualValues(0, h.Size) + // File should be in the account + file := s.shareUrl.NewRootDirectoryURL().NewFileURL(name) + props, err := file.GetProperties(ctx) + s.assert.Nil(err) + s.assert.NotNil(props) + s.assert.Empty(props.NewMetadata()) +} + +func (s *fileTestSuite) TestOpenFile() { + defer s.cleanupTest() + // Setup + name := generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: name}) + + h, err := s.az.OpenFile(internal.OpenFileOptions{Name: name}) + s.assert.Nil(err) + s.assert.NotNil(h) + s.assert.EqualValues(name, h.Path) + s.assert.EqualValues(0, h.Size) +} + +func (s *fileTestSuite) TestOpenFileError() { + defer s.cleanupTest() + // Setup + name := generateFileName() + + h, err := s.az.OpenFile(internal.OpenFileOptions{Name: name}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOENT, err) + s.assert.Nil(h) +} + +func (s *fileTestSuite) TestOpenFileSize() { + defer s.cleanupTest() + // Setup + name := generateFileName() + size := 10 + s.az.CreateFile(internal.CreateFileOptions{Name: name}) + s.az.TruncateFile(internal.TruncateFileOptions{Name: name, Size: int64(size)}) + + h, err := s.az.OpenFile(internal.OpenFileOptions{Name: name}) + s.assert.Nil(err) + s.assert.NotNil(h) + s.assert.EqualValues(name, h.Path) + s.assert.EqualValues(size, h.Size) +} + +func (s *fileTestSuite) TestCloseFile() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + + // This method does nothing. + err := s.az.CloseFile(internal.CloseFileOptions{Handle: h}) + s.assert.Nil(err) +} + +func (s *fileTestSuite) TestCloseFileFakeHandle() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h := handlemap.NewHandle(name) + + // This method does nothing. + err := s.az.CloseFile(internal.CloseFileOptions{Handle: h}) + s.assert.Nil(err) +} + +func (s *fileTestSuite) TestDeleteFile() { + defer s.cleanupTest() + // Setup + name := generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: name}) + + err := s.az.DeleteFile(internal.DeleteFileOptions{Name: name}) + s.assert.Nil(err) + + // File should not be in the account + fileName, dirPath := getFileAndDirFromPath(name) + file := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + _, err = file.GetProperties(ctx) + s.assert.NotNil(err) +} + +func (s *fileTestSuite) TestDeleteFileError() { + defer s.cleanupTest() + // Setup + name := generateFileName() + + err := s.az.DeleteFile(internal.DeleteFileOptions{Name: name}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOENT, err) + + // File should not be in the account + fileName, dirPath := getFileAndDirFromPath(name) + file := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + _, err = file.GetProperties(ctx) + s.assert.NotNil(err) +} + +func (s *fileTestSuite) TestRenameFile() { + defer s.cleanupTest() + // Setup + src := generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: src}) + dst := generateFileName() + + err := s.az.RenameFile(internal.RenameFileOptions{Src: src, Dst: dst}) + s.assert.Nil(err) + + // Src should not be in the account + fileName, dirPath := getFileAndDirFromPath(src) + source := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + _, err = source.GetProperties(ctx) + s.assert.NotNil(err) + // Dst should be in the account + fileName, dirPath = getFileAndDirFromPath(dst) + destination := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + _, err = destination.GetProperties(ctx) + s.assert.Nil(err) +} + +func (s *fileTestSuite) TestRenameFileMetadataConservation() { + defer s.cleanupTest() + // Setup + src := generateFileName() + fileName, dirPath := getFileAndDirFromPath(src) + source := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + s.az.CreateFile(internal.CreateFileOptions{Name: src}) + + // Add srcMeta to source + srcMeta := make(azfile.Metadata) + srcMeta["foo"] = "bar" + source.SetMetadata(ctx, srcMeta) + + dst := generateFileName() + err := s.az.RenameFile(internal.RenameFileOptions{Src: src, Dst: dst}) + s.assert.Nil(err) + + // Src should not be in the account + _, err = source.GetProperties(ctx) + s.assert.NotNil(err) + // Dst should be in the account + fileName, dirPath = getFileAndDirFromPath(dst) + destination := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + props, err := destination.GetProperties(ctx) + s.assert.Nil(err) + // Dst should have metadata + destMeta := props.NewMetadata() + s.assert.Contains(destMeta, "foo") + s.assert.EqualValues("bar", destMeta["foo"]) +} + +func (s *fileTestSuite) TestRenameFileTargetExistsError() { + defer s.cleanupTest() + // Setup + src := generateFileName() + dst := generateFileName() + + s.az.CreateFile(internal.CreateFileOptions{Name: dst}) + + err := s.az.RenameFile(internal.RenameFileOptions{Src: src, Dst: dst}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOENT, err) + + // Only destination should be in the account + fileName, dirPath := getFileAndDirFromPath(src) + source := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + _, err = source.GetProperties(ctx) + s.assert.NotNil(err) + fileName, dirPath = getFileAndDirFromPath(dst) + destination := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + _, err = destination.GetProperties(ctx) + s.assert.Nil(err) +} + +func (s *fileTestSuite) TestRenameFileError() { + defer s.cleanupTest() + // Setup + src := generateFileName() + dst := generateFileName() + + err := s.az.RenameFile(internal.RenameFileOptions{Src: src, Dst: dst}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOENT, err) + + // Src and destination should not be in the account + fileName, dirPath := getFileAndDirFromPath(src) + source := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + _, err = source.GetProperties(ctx) + s.assert.NotNil(err) + fileName, dirPath = getFileAndDirFromPath(dst) + destination := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + _, err = destination.GetProperties(ctx) + s.assert.NotNil(err) +} + +func (s *fileTestSuite) TestReadFile() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "test data" + data := []byte(testData) + s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + h, _ = s.az.OpenFile(internal.OpenFileOptions{Name: name}) + + output, err := s.az.ReadFile(internal.ReadFileOptions{Handle: h}) + s.assert.Nil(err) + s.assert.EqualValues(testData, output) +} + +func (s *fileTestSuite) TestReadFileEmpty() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + + output, err := s.az.ReadFile(internal.ReadFileOptions{Handle: h}) + s.assert.Nil(err) + s.assert.EqualValues("", output) +} + +func (s *fileTestSuite) TestReadFileError() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h := handlemap.NewHandle(name) + + _, err := s.az.ReadFile(internal.ReadFileOptions{Handle: h}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOENT, err) +} + +func (s *fileTestSuite) TestReadInBuffer() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "test data" + data := []byte(testData) + s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + h, _ = s.az.OpenFile(internal.OpenFileOptions{Name: name}) + + output := make([]byte, 9) + len, err := s.az.ReadInBuffer(internal.ReadInBufferOptions{Handle: h, Offset: 0, Data: output}) + s.assert.Nil(err) + s.assert.EqualValues(9, len) + s.assert.EqualValues(testData[:9], output) +} + +func (s *fileTestSuite) TestReadInBufferLargeBuffer() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "test data" + data := []byte(testData) + s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + h, _ = s.az.OpenFile(internal.OpenFileOptions{Name: name}) + + output := make([]byte, 1000) // Testing that passing in a super large buffer will still work + len, err := s.az.ReadInBuffer(internal.ReadInBufferOptions{Handle: h, Offset: 0, Data: output}) + s.assert.Nil(err) + s.assert.EqualValues(h.Size, len) + s.assert.EqualValues(testData, output[:h.Size]) +} + +func (s *fileTestSuite) TestReadInBufferEmpty() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + + output := make([]byte, 10) + len, err := s.az.ReadInBuffer(internal.ReadInBufferOptions{Handle: h, Offset: 0, Data: output}) + s.assert.Nil(err) + s.assert.EqualValues(0, len) +} + +func (s *fileTestSuite) TestReadInBufferBadRangeNonzeroOffset() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h := handlemap.NewHandle(name) + h.Size = 10 + + _, err := s.az.ReadInBuffer(internal.ReadInBufferOptions{Handle: h, Offset: 20, Data: make([]byte, 2)}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ERANGE, err) +} + +func (s *fileTestSuite) TestReadInBufferError() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h := handlemap.NewHandle(name) + h.Size = 10 + + _, err := s.az.ReadInBuffer(internal.ReadInBufferOptions{Handle: h, Offset: 0, Data: make([]byte, 2)}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOENT, err) +} + +func (s *fileTestSuite) TestWriteFile() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + + testData := "test data" + data := []byte(testData) + count, err := s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + s.assert.Nil(err) + s.assert.EqualValues(len(data), count) + + // File should have updated data + fileName, dirPath := getFileAndDirFromPath(name) + file := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + resp, err := file.Download(ctx, 0, int64(len(data)), false) + s.assert.Nil(err) + output, _ := ioutil.ReadAll(resp.Body(azfile.RetryReaderOptions{})) + s.assert.EqualValues(testData, output) +} + +func (s *fileTestSuite) TestTruncateFileSmaller() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "test data" + data := []byte(testData) + truncatedLength := 5 + s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + + err := s.az.TruncateFile(internal.TruncateFileOptions{Name: name, Size: int64(truncatedLength)}) + s.assert.Nil(err) + + // File should have updated data + fileName, dirPath := getFileAndDirFromPath(name) + file := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + resp, err := file.Download(ctx, 0, int64(truncatedLength), false) + s.assert.Nil(err) + s.assert.EqualValues(truncatedLength, resp.ContentLength()) + output, _ := ioutil.ReadAll(resp.Body(azfile.RetryReaderOptions{})) + s.assert.EqualValues(testData[:truncatedLength], output) +} + +func (s *fileTestSuite) TestTruncateFileEqual() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "test data" + data := []byte(testData) + truncatedLength := 9 + s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + + err := s.az.TruncateFile(internal.TruncateFileOptions{Name: name, Size: int64(truncatedLength)}) + s.assert.Nil(err) + + // File should have updated data + fileName, dirPath := getFileAndDirFromPath(name) + file := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + resp, err := file.Download(ctx, 0, int64(truncatedLength), false) + s.assert.Nil(err) + s.assert.EqualValues(truncatedLength, resp.ContentLength()) + output, _ := ioutil.ReadAll(resp.Body(azfile.RetryReaderOptions{})) + s.assert.EqualValues(testData, output) +} + +func (s *fileTestSuite) TestTruncateFileBigger() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "test data" + data := []byte(testData) + truncatedLength := 15 + s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + + err := s.az.TruncateFile(internal.TruncateFileOptions{Name: name, Size: int64(truncatedLength)}) + s.assert.Nil(err) + + // File should have updated data + fileName, dirPath := getFileAndDirFromPath(name) + file := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + resp, err := file.Download(ctx, 0, int64(truncatedLength), false) + s.assert.Nil(err) + s.assert.EqualValues(truncatedLength, resp.ContentLength()) + output, _ := ioutil.ReadAll(resp.Body(azfile.RetryReaderOptions{})) + s.assert.EqualValues(testData, output[:len(data)]) +} + +func (s *fileTestSuite) TestTruncateFileError() { + defer s.cleanupTest() + // Setup + name := generateFileName() + + err := s.az.TruncateFile(internal.TruncateFileOptions{Name: name}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOENT, storeFileErrToErr(err)) +} + +func (s *fileTestSuite) TestOverwrite() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "test-replace-data" + data := []byte(testData) + dataLen := len(data) + _, err := s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + s.assert.Nil(err) + f, _ := ioutil.TempFile("", name+".tmp") + defer os.Remove(f.Name()) + newTestData := []byte("newdata") + _, err = s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 5, Data: newTestData}) + s.assert.Nil(err) + + currentData := []byte("test-newdata-data") + output := make([]byte, len(currentData)) + + err = s.az.CopyToFile(internal.CopyToFileOptions{Name: name, File: f}) + s.assert.Nil(err) + + f, _ = os.Open(f.Name()) + len, err := f.Read(output) + s.assert.Nil(err) + s.assert.EqualValues(dataLen, len) + s.assert.EqualValues(currentData, output) + f.Close() +} + +func (s *fileTestSuite) TestOverwriteAndAppend() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "test-data" + data := []byte(testData) + + _, err := s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + s.assert.Nil(err) + f, _ := ioutil.TempFile("", name+".tmp") + defer os.Remove(f.Name()) + newTestData := []byte("newdata") + _, err = s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 5, Data: newTestData}) + s.assert.Nil(err) + + currentData := []byte("test-newdata") + dataLen := len(currentData) + output := make([]byte, dataLen) + + err = s.az.CopyToFile(internal.CopyToFileOptions{Name: name, File: f}) + s.assert.Nil(err) + + f, _ = os.Open(f.Name()) + len, err := f.Read(output) + s.assert.Nil(err) + s.assert.EqualValues(dataLen, len) + s.assert.EqualValues(currentData, output) + f.Close() +} + +func (s *fileTestSuite) TestAppend() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "test-data" + data := []byte(testData) + + _, err := s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + s.assert.Nil(err) + f, _ := ioutil.TempFile("", name+".tmp") + defer os.Remove(f.Name()) + newTestData := []byte("-newdata") + _, err = s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 9, Data: newTestData}) + s.assert.Nil(err) + + currentData := []byte("test-data-newdata") + dataLen := len(currentData) + output := make([]byte, dataLen) + + err = s.az.CopyToFile(internal.CopyToFileOptions{Name: name, File: f}) + s.assert.Nil(err) + + f, _ = os.Open(f.Name()) + len, err := f.Read(output) + s.assert.Nil(err) + s.assert.EqualValues(dataLen, len) + s.assert.EqualValues(currentData, output) + f.Close() +} + +func (s *fileTestSuite) TestAppendOffsetLargerThanFile() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "test-data" + data := []byte(testData) + + _, err := s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + s.assert.Nil(err) + f, _ := ioutil.TempFile("", name+".tmp") + defer os.Remove(f.Name()) + newTestData := []byte("newdata") + _, err = s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 12, Data: newTestData}) + s.assert.Nil(err) + + currentData := []byte("test-data\x00\x00\x00newdata") + dataLen := len(currentData) + output := make([]byte, dataLen) + + err = s.az.CopyToFile(internal.CopyToFileOptions{Name: name, File: f}) + s.assert.Nil(err) + + f, _ = os.Open(f.Name()) + len, err := f.Read(output) + s.assert.Nil(err) + s.assert.EqualValues(dataLen, len) + s.assert.EqualValues(currentData, output) + f.Close() +} + +func (s *fileTestSuite) TestCopyToFileError() { + defer s.cleanupTest() + // Setup + name := generateFileName() + f, _ := ioutil.TempFile("", name+".tmp") + defer os.Remove(f.Name()) + + err := s.az.CopyToFile(internal.CopyToFileOptions{Name: name, File: f}) + s.assert.NotNil(err) +} + +// Upload existing, nonempty local file to existing Azure file +func (s *fileTestSuite) TestCopyFromFile() { + defer s.cleanupTest() + // Setup + name := generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "test data" + data := []byte(testData) + homeDir, _ := os.UserHomeDir() + f, _ := ioutil.TempFile(homeDir, name+".tmp") + defer os.Remove(f.Name()) + f.Write(data) + + err := s.az.CopyFromFile(internal.CopyFromFileOptions{Name: name, File: f}) + s.assert.Nil(err) + + // File should have updated data + fileName, dirPath := getFileAndDirFromPath(name) + file := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + resp, err := file.Download(ctx, 0, int64(len(data)), false) + s.assert.Nil(err) + output, _ := ioutil.ReadAll(resp.Body(azfile.RetryReaderOptions{})) + s.assert.EqualValues(testData, output) +} + +// Upload existing, empty local file to existing Azure file +func (s *fileTestSuite) TestCopyFromFileEmpty() { + defer s.cleanupTest() + // Setup + name := generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: name}) + homeDir, _ := os.UserHomeDir() + f, _ := ioutil.TempFile(homeDir, name+".tmp") + defer os.Remove(f.Name()) + + err := s.az.CopyFromFile(internal.CopyFromFileOptions{Name: name, File: f}) + s.assert.Nil(err) + + // File should have no data + fileName, dirPath := getFileAndDirFromPath(name) + file := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + resp, err := file.Download(ctx, 0, 0, false) + s.assert.Nil(err) + output, _ := ioutil.ReadAll(resp.Body(azfile.RetryReaderOptions{})) + s.assert.EqualValues("", output) +} + +func (s *fileTestSuite) TestCreateLink() { + defer s.cleanupTest() + // Setup + target := generateFileName() + _, err := s.az.CreateFile(internal.CreateFileOptions{Name: target}) + s.assert.Nil(err) + + source := generateFileName() + err = s.az.CreateLink(internal.CreateLinkOptions{Name: source, Target: target}) + s.assert.Nil(err) + + // Link should be in the account + fileName, dirPath := getFileAndDirFromPath(source) + link := s.shareUrl.NewDirectoryURL(dirPath).NewFileURL(fileName) + props, err := link.GetProperties(ctx) + s.assert.Nil(err) + s.assert.NotNil(props) + s.assert.NotEmpty(props.NewMetadata()) + s.assert.Contains(props.NewMetadata(), symlinkKey) + s.assert.EqualValues("true", props.NewMetadata()[symlinkKey]) + resp, err := link.Download(ctx, 0, props.ContentLength(), false) + s.assert.Nil(err) + data, _ := ioutil.ReadAll(resp.Body(azfile.RetryReaderOptions{})) + s.assert.EqualValues(target, data) +} + +func (s *fileTestSuite) TestReadLink() { + defer s.cleanupTest() + // Setup + target := generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: target}) + name := generateFileName() + s.az.CreateLink(internal.CreateLinkOptions{Name: name, Target: target}) + + read, err := s.az.ReadLink(internal.ReadLinkOptions{Name: name}) + s.assert.Nil(err) + s.assert.EqualValues(target, read) +} + +func (s *fileTestSuite) TestReadLinkError() { + defer s.cleanupTest() + // Setup + name := generateFileName() + + _, err := s.az.ReadLink(internal.ReadLinkOptions{Name: name}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOENT, err) +} + +func (s *fileTestSuite) TestGetAttrDir() { + defer s.cleanupTest() + // Setup + name := generateDirectoryName() + s.az.CreateDir(internal.CreateDirOptions{Name: name}) + + props, err := s.az.GetAttr(internal.GetAttrOptions{Name: name}) + s.assert.Nil(err) + s.assert.NotNil(props) + s.assert.True(props.IsDir()) + s.assert.NotEmpty(props.Metadata) + s.assert.Contains(props.Metadata, folderKey) + s.assert.EqualValues("true", props.Metadata[folderKey]) +} + +func (s *fileTestSuite) TestGetAttrFile() { + defer s.cleanupTest() + // Setup + name := generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: name}) + + props, err := s.az.GetAttr(internal.GetAttrOptions{Name: name}) + s.assert.Nil(err) + s.assert.NotNil(props) + s.assert.False(props.IsDir()) + s.assert.False(props.IsSymlink()) +} + +func (s *fileTestSuite) TestGetAttrLink() { + defer s.cleanupTest() + // Setup + target := generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: target}) + name := generateFileName() + s.az.CreateLink(internal.CreateLinkOptions{Name: name, Target: target}) + + props, err := s.az.GetAttr(internal.GetAttrOptions{Name: name}) + s.assert.Nil(err) + s.assert.NotNil(props) + s.assert.True(props.IsSymlink()) + s.assert.NotEmpty(props.Metadata) + s.assert.Contains(props.Metadata, symlinkKey) + s.assert.EqualValues("true", props.Metadata[symlinkKey]) +} + +func (s *fileTestSuite) TestGetAttrFileSize() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "test data" + data := []byte(testData) + s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + + props, err := s.az.GetAttr(internal.GetAttrOptions{Name: name}) + s.assert.Nil(err) + s.assert.NotNil(props) + s.assert.False(props.IsDir()) + s.assert.False(props.IsSymlink()) + s.assert.EqualValues(len(testData), props.Size) +} + +func (s *fileTestSuite) TestGetAttrFileTime() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "test data" + data := []byte(testData) + s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + + before, err := s.az.GetAttr(internal.GetAttrOptions{Name: name}) + s.assert.Nil(err) + s.assert.NotNil(before.Mtime) + + time.Sleep(time.Second * 3) // Wait 3 seconds and then modify the file again + + s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + + after, err := s.az.GetAttr(internal.GetAttrOptions{Name: name}) + s.assert.Nil(err) + s.assert.NotNil(after.Mtime) + + s.assert.True(after.Mtime.After(before.Mtime)) +} + +func (s *fileTestSuite) TestGetAttrError() { + defer s.cleanupTest() + // Setup + name := generateFileName() + + _, err := s.az.GetAttr(internal.GetAttrOptions{Name: name}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOENT, err) +} + +// If support for chown or chmod are ever added to files, add tests for error cases and modify the following tests. +func (s *fileTestSuite) TestChmod() { + defer s.cleanupTest() + // Setup + name := generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: name}) + + err := s.az.Chmod(internal.ChmodOptions{Name: name, Mode: 0666}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOTSUP, err) +} + +func (s *fileTestSuite) TestChmodIgnore() { + defer s.cleanupTest() + // Setup + s.tearDownTestHelper(false) // Don't delete the generated container. + + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: false\n", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + s.setupTestHelper(config, s.container, true) + name := generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: name}) + + err := s.az.Chmod(internal.ChmodOptions{Name: name, Mode: 0666}) + s.assert.Nil(err) +} + +func (s *fileTestSuite) TestChown() { + defer s.cleanupTest() + // Setup + name := generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: name}) + + err := s.az.Chown(internal.ChownOptions{Name: name, Owner: 6, Group: 5}) + s.assert.NotNil(err) + s.assert.EqualValues(syscall.ENOTSUP, err) +} + +func (s *fileTestSuite) TestChownIgnore() { + defer s.cleanupTest() + // Setup + s.tearDownTestHelper(false) // Don't delete the generated container. + + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: false\n", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + s.setupTestHelper(config, s.container, true) + name := generateFileName() + s.az.CreateFile(internal.CreateFileOptions{Name: name}) + + err := s.az.Chown(internal.ChownOptions{Name: name, Owner: 6, Group: 5}) + s.assert.Nil(err) +} + +func (s *fileTestSuite) TestBlockSize() { + defer s.cleanupTest() + // Setup + name := generateFileName() + + fs := FileShare{} + + // For filesize 0 expected rangesize is 4MB + filerng, err := fs.calculateRangeSize(name, 0) + s.assert.Nil(err) + s.assert.EqualValues(filerng, azfile.FileMaxUploadRangeBytes) + + // For filesize 100MB expected rangesize is 4MB + filerng, err = fs.calculateRangeSize(name, (100 * 1024 * 1024)) + s.assert.Nil(err) + s.assert.EqualValues(filerng, azfile.FileMaxUploadRangeBytes) + + // For filesize 500MB expected rangesize is 4MB + filerng, err = fs.calculateRangeSize(name, (500 * 1024 * 1024)) + s.assert.Nil(err) + s.assert.EqualValues(filerng, azfile.FileMaxUploadRangeBytes) + + // For filesize 1GB expected rangesize is 4MB + filerng, err = fs.calculateRangeSize(name, (1 * 1024 * 1024 * 1024)) + s.assert.Nil(err) + s.assert.EqualValues(filerng, azfile.FileMaxUploadRangeBytes) + + // For filesize 500GB expected rangesize is 4MB + filerng, err = fs.calculateRangeSize(name, (500 * 1024 * 1024 * 1024)) + s.assert.Nil(err) + s.assert.EqualValues(filerng, azfile.FileMaxUploadRangeBytes) + + // For filesize 1TB expected rangesize is 4MB + filerng, err = fs.calculateRangeSize(name, (1 * 1024 * 1024 * 1024 * 1024)) + s.assert.Nil(err) + s.assert.EqualValues(filerng, azfile.FileMaxUploadRangeBytes) + + // For filesize 4TB expected rangesize is 4MB + filerng, err = fs.calculateRangeSize(name, (4 * 1024 * 1024 * 1024 * 1024)) + s.assert.Nil(err) + s.assert.EqualValues(filerng, azfile.FileMaxUploadRangeBytes) + + // Boundary condition which is exactly max size supported by SDK + filerng, err = fs.calculateRangeSize(name, FileMaxSizeInBytes) + s.assert.Nil(err) + s.assert.EqualValues(filerng, azfile.FileMaxUploadRangeBytes) // 4194304000 + + // For Filesize created using dd for 1TB size + filerng, err = fs.calculateRangeSize(name, (1 * 1024 * 1024 * 1024 * 1024)) + s.assert.Nil(err) + s.assert.EqualValues(filerng, azfile.FileMaxUploadRangeBytes) + + // Boundary condition 5 bytes less then max expected file size + filerng, err = fs.calculateRangeSize(name, (FileMaxSizeInBytes - 5)) + s.assert.Nil(err) + s.assert.EqualValues(filerng, azfile.FileMaxUploadRangeBytes) + + // Boundary condition 1 bytes more then max expected file size + filerng, err = fs.calculateRangeSize(name, (FileMaxSizeInBytes + 1)) + s.assert.NotNil(err) + s.assert.EqualValues(filerng, 0) + + // Boundary condition 5 bytes more then max expected file size + filerng, err = fs.calculateRangeSize(name, (FileMaxSizeInBytes + 5)) + s.assert.NotNil(err) + s.assert.EqualValues(filerng, 0) + + // Boundary condition one byte more then max range size + filerng, err = fs.calculateRangeSize(name, ((azfile.FileMaxUploadRangeBytes + 1) * FileShareMaxRanges)) + s.assert.NotNil(err) + s.assert.EqualValues(filerng, 0) + + // For filesize 5, error is expected as max 4TB only supported + filerng, err = fs.calculateRangeSize(name, (5 * 1024 * 1024 * 1024 * 1024)) + s.assert.NotNil(err) + s.assert.EqualValues(filerng, 0) +} + +func (s *fileTestSuite) TestGetFileBlockOffsetsRangedFile() { + defer s.cleanupTest() + // Setup + name := generateFileName() + h, _ := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + testData := "testdatates1dat1tes2dat2tes3dat3tes4dat4" + data := []byte(testData) + + s.az.WriteFile(internal.WriteFileOptions{Handle: h, Offset: 0, Data: data}) + + // GetFileBlockOffsets + offsetList, err := s.az.GetFileBlockOffsets(internal.GetFileBlockOffsetsOptions{Name: name}) + s.assert.Nil(err) + s.assert.Len(offsetList.BlockList, 1) + s.assert.Zero(offsetList.Flags) +} + +func (s *fileTestSuite) TestMD5SetOnUpload() { + defer s.cleanupTest() + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + configs := []string{"", vdConfig} + for _, c := range configs { + // This is a little janky but required since testify suite does not support running setup or clean up for subtests. + s.tearDownTestHelper(false) + s.setupTestHelper(c, s.container, true) + testName := "" + if c != "" { + testName = "virtual-directory" + } + s.Run(testName, func() { + // Setup + s.tearDownTestHelper(false) // Don't delete the generated container. + + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + s.setupTestHelper(config, s.container, true) + + name := generateFileName() + f, err := os.Create(name) + s.assert.Nil(err) + s.assert.NotNil(f) + + data := make([]byte, azblob.BlockBlobMaxUploadBlobBytes+1) + _, _ = rand.Read(data) + + n, err := f.Write(data) + s.assert.Nil(err) + s.assert.EqualValues(n, azblob.BlockBlobMaxUploadBlobBytes+1) + _, _ = f.Seek(0, 0) + + err = s.az.storage.WriteFromFile(name, nil, f) + s.assert.Nil(err) + + prop, err := s.az.storage.GetAttr(name) + s.assert.Nil(err) + s.assert.NotEmpty(prop.MD5) + + _, _ = f.Seek(0, 0) + localMD5, err := getMD5(f) + s.assert.Nil(err) + s.assert.EqualValues(localMD5, prop.MD5) + + _ = s.az.storage.DeleteFile(name) + _ = f.Close() + _ = os.Remove(name) + }) + } +} + +func (s *fileTestSuite) TestMD5NotSetOnUpload() { + defer s.cleanupTest() + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + configs := []string{"", vdConfig} + for _, c := range configs { + // This is a little janky but required since testify suite does not support running setup or clean up for subtests. + s.tearDownTestHelper(false) + s.setupTestHelper(c, s.container, true) + testName := "" + if c != "" { + testName = "virtual-directory" + } + s.Run(testName, func() { + // Setup + s.tearDownTestHelper(false) // Don't delete the generated container. + + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n update-md5: false\n", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + s.setupTestHelper(config, s.container, true) + + name := generateFileName() + f, err := os.Create(name) + s.assert.Nil(err) + s.assert.NotNil(f) + + data := make([]byte, azblob.BlockBlobMaxUploadBlobBytes+1) + _, _ = rand.Read(data) + + n, err := f.Write(data) + s.assert.Nil(err) + s.assert.EqualValues(n, azblob.BlockBlobMaxUploadBlobBytes+1) + _, _ = f.Seek(0, 0) + + err = s.az.storage.WriteFromFile(name, nil, f) + s.assert.Nil(err) + + prop, err := s.az.storage.GetAttr(name) + s.assert.Nil(err) + s.assert.Empty(prop.MD5) + + _ = s.az.storage.DeleteFile(name) + _ = f.Close() + _ = os.Remove(name) + }) + } +} + +func (s *fileTestSuite) TestInvalidateMD5PostUpload() { + defer s.cleanupTest() + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + configs := []string{"", vdConfig} + for _, c := range configs { + // This is a little janky but required since testify suite does not support running setup or clean up for subtests. + s.tearDownTestHelper(false) + s.setupTestHelper(c, s.container, true) + testName := "" + if c != "" { + testName = "virtual-directory" + } + s.Run(testName, func() { + // Setup + s.tearDownTestHelper(false) // Don't delete the generated container. + + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n validate-md5: true\n", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + s.setupTestHelper(config, s.container, true) + + name := generateFileName() + f, err := os.Create(name) + s.assert.Nil(err) + s.assert.NotNil(f) + + data := make([]byte, 100) + _, _ = rand.Read(data) + + n, err := f.Write(data) + s.assert.Nil(err) + s.assert.EqualValues(n, 100) + _, _ = f.Seek(0, 0) + + err = s.az.storage.WriteFromFile(name, nil, f) + s.assert.Nil(err) + + blobURL := s.shareUrl.NewRootDirectoryURL().NewFileURL(name) + _, _ = blobURL.SetHTTPHeaders(context.Background(), azfile.FileHTTPHeaders{ContentMD5: []byte("blobfuse")}) + + prop, err := s.az.storage.GetAttr(name) + s.assert.Nil(err) + s.assert.NotEmpty(prop.MD5) + + _, _ = f.Seek(0, 0) + localMD5, err := getMD5(f) + s.assert.Nil(err) + s.assert.NotEqualValues(localMD5, prop.MD5) + + _ = s.az.storage.DeleteFile(name) + _ = f.Close() + _ = os.Remove(name) + }) + } +} + +func (s *fileTestSuite) TestValidateManualMD5OnRead() { + defer s.cleanupTest() + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + configs := []string{"", vdConfig} + for _, c := range configs { + // This is a little janky but required since testify suite does not support running setup or clean up for subtests. + s.tearDownTestHelper(false) + s.setupTestHelper(c, s.container, true) + testName := "" + if c != "" { + testName = "virtual-directory" + } + s.Run(testName, func() { + // Setup + s.tearDownTestHelper(false) // Don't delete the generated container. + + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n validate-md5: true\n", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + s.setupTestHelper(config, s.container, true) + + name := generateFileName() + f, err := os.Create(name) + s.assert.Nil(err) + s.assert.NotNil(f) + + data := make([]byte, azblob.BlockBlobMaxUploadBlobBytes+1) + _, _ = rand.Read(data) + + n, err := f.Write(data) + s.assert.Nil(err) + s.assert.EqualValues(n, azblob.BlockBlobMaxUploadBlobBytes+1) + _, _ = f.Seek(0, 0) + + err = s.az.storage.WriteFromFile(name, nil, f) + s.assert.Nil(err) + _ = f.Close() + _ = os.Remove(name) + + prop, err := s.az.storage.GetAttr(name) + s.assert.Nil(err) + s.assert.NotEmpty(prop.MD5) + + f, err = os.Create(name) + s.assert.Nil(err) + s.assert.NotNil(f) + + err = s.az.storage.ReadToFile(name, 0, azblob.BlockBlobMaxUploadBlobBytes+1, f) + s.assert.Nil(err) + + _ = s.az.storage.DeleteFile(name) + _ = os.Remove(name) + }) + } +} + +func (s *fileTestSuite) TestInvalidMD5OnRead() { + defer s.cleanupTest() + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + configs := []string{"", vdConfig} + for _, c := range configs { + // This is a little janky but required since testify suite does not support running setup or clean up for subtests. + s.tearDownTestHelper(false) + s.setupTestHelper(c, s.container, true) + testName := "" + if c != "" { + testName = "virtual-directory" + } + s.Run(testName, func() { + // Setup + s.tearDownTestHelper(false) // Don't delete the generated container. + + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n validate-md5: true\n", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + s.setupTestHelper(config, s.container, true) + + name := generateFileName() + f, err := os.Create(name) + s.assert.Nil(err) + s.assert.NotNil(f) + + data := make([]byte, 100) + _, _ = rand.Read(data) + + n, err := f.Write(data) + s.assert.Nil(err) + s.assert.EqualValues(n, 100) + _, _ = f.Seek(0, 0) + + err = s.az.storage.WriteFromFile(name, nil, f) + s.assert.Nil(err) + _ = f.Close() + _ = os.Remove(name) + + blobURL := s.shareUrl.NewRootDirectoryURL().NewFileURL(name) + _, _ = blobURL.SetHTTPHeaders(context.Background(), azfile.FileHTTPHeaders{ContentMD5: []byte("blobfuse")}) + + prop, err := s.az.storage.GetAttr(name) + s.assert.Nil(err) + s.assert.NotEmpty(prop.MD5) + + f, err = os.Create(name) + s.assert.Nil(err) + s.assert.NotNil(f) + + err = s.az.storage.ReadToFile(name, 0, 100, f) + s.assert.NotNil(err) + s.assert.Contains(err.Error(), "md5 sum mismatch on download") + + _ = s.az.storage.DeleteFile(name) + _ = os.Remove(name) + }) + } +} + +func (s *fileTestSuite) TestInvalidMD5OnReadNoVaildate() { + defer s.cleanupTest() + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + configs := []string{"", vdConfig} + for _, c := range configs { + // This is a little janky but required since testify suite does not support running setup or clean up for subtests. + s.tearDownTestHelper(false) + s.setupTestHelper(c, s.container, true) + testName := "" + if c != "" { + testName = "virtual-directory" + } + s.Run(testName, func() { + // Setup + s.tearDownTestHelper(false) // Don't delete the generated container. + + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.file.core.windows.net/\n type: file\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n validate-md5: false\n", + storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileAccount, storageTestConfigurationParameters.FileKey, s.container) + s.setupTestHelper(config, s.container, true) + + name := generateFileName() + f, err := os.Create(name) + s.assert.Nil(err) + s.assert.NotNil(f) + + data := make([]byte, 100) + _, _ = rand.Read(data) + + n, err := f.Write(data) + s.assert.Nil(err) + s.assert.EqualValues(n, 100) + _, _ = f.Seek(0, 0) + + err = s.az.storage.WriteFromFile(name, nil, f) + s.assert.Nil(err) + _ = f.Close() + _ = os.Remove(name) + + blobURL := s.shareUrl.NewRootDirectoryURL().NewFileURL(name) + _, _ = blobURL.SetHTTPHeaders(context.Background(), azfile.FileHTTPHeaders{ContentMD5: []byte("blobfuse")}) + + prop, err := s.az.storage.GetAttr(name) + s.assert.Nil(err) + s.assert.NotEmpty(prop.MD5) + + f, err = os.Create(name) + s.assert.Nil(err) + s.assert.NotNil(f) + + err = s.az.storage.ReadToFile(name, 0, 100, f) + s.assert.Nil(err) + + _ = s.az.storage.DeleteFile(name) + _ = os.Remove(name) + }) + } +} + +func TestFileShare(t *testing.T) { + suite.Run(t, new(fileTestSuite)) +} diff --git a/component/azstorage/utils.go b/component/azstorage/utils.go index 08a404465..2708c849b 100644 --- a/component/azstorage/utils.go +++ b/component/azstorage/utils.go @@ -49,15 +49,14 @@ import ( "strings" "time" - "github.com/Azure/azure-storage-fuse/v2/common" - "github.com/Azure/azure-storage-fuse/v2/common/log" - "github.com/Azure/azure-storage-fuse/v2/internal" - + "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-azcopy/v10/azbfs" "github.com/Azure/azure-storage-azcopy/v10/ste" - - "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-storage-file-go/azfile" + "github.com/Azure/azure-storage-fuse/v2/common" + "github.com/Azure/azure-storage-fuse/v2/common/log" + "github.com/Azure/azure-storage-fuse/v2/internal" ) // ----------- Helper to create pipeline options --------------- @@ -141,6 +140,35 @@ func getAzBfsPipelineOptions(conf AzStorageConfig) (azbfs.PipelineOptions, ste.X retryOptions } +// getAzFilePipelineOptions : Create pipeline options based on the config +func getAzFilePipelineOptions(conf AzStorageConfig) (azfile.PipelineOptions, azfile.RetryOptions) { + retryOptions := azfile.RetryOptions{ + Policy: azfile.RetryPolicyExponential, // Use exponential backoff as opposed to linear + MaxTries: conf.maxRetries, // Try at most 3 times to perform the operation (set to 1 to disable retries) + TryTimeout: time.Second * time.Duration(conf.maxTimeout), // Maximum time allowed for any single try + RetryDelay: time.Second * time.Duration(conf.backoffTime), // Backoff amount for each retry (exponential or linear) + MaxRetryDelay: time.Second * time.Duration(conf.maxRetryDelay), // Max delay between retries + } + telemetryOptions := azfile.TelemetryOptions{ + Value: UserAgent() + " (" + common.GetCurrentDistro() + ")", + } + + sysLogDisabled := log.GetType() == "silent" // If logging is enabled, allow the SDK to log retries to syslog. + requestLogOptions := azfile.RequestLogOptions{ + // TODO: We can potentially consider making LogWarningIfTryOverThreshold a user settable option. For now lets use the default + SyslogDisabled: sysLogDisabled, + } + logOptions := getLogOptions(conf.sdkTrace) + // Create custom HTTPClient to pass to the factory in order to set our proxy + return azfile.PipelineOptions{ + Log: logOptions, + RequestLog: requestLogOptions, + Telemetry: telemetryOptions, + }, + // Set RetryOptions to control how HTTP request are retried when retryable failures occur + retryOptions +} + // Create an HTTP Client with configured proxy // TODO: More configurations for other http client parameters? func newBlobfuse2HttpClient(conf AzStorageConfig) *http.Client { @@ -317,6 +345,25 @@ func storeDatalakeErrToErr(err error) uint16 { return ErrNoErr } +// Convert file storage error to common errors +func storeFileErrToErr(err error) uint16 { + if serr, ok := err.(azfile.StorageError); ok { + switch serr.ServiceCode() { + case azfile.ServiceCodeResourceAlreadyExists: + return ErrFileAlreadyExists + case azfile.ServiceCodeResourceNotFound: + return ErrFileNotFound + case azfile.ServiceCodeInvalidRange: + return InvalidRange + case azfile.ServiceCodeInsufficientAccountPermissions: + return InvalidPermission + default: + return ErrUnknown + } + } + return ErrNoErr +} + // ----------- Metadata handling --------------- // // Converts datalake properties to a metadata map diff --git a/component/file_cache/file_cache.go b/component/file_cache/file_cache.go index 3205d00e4..eb68afcbd 100644 --- a/component/file_cache/file_cache.go +++ b/component/file_cache/file_cache.go @@ -856,7 +856,7 @@ func (fc *FileCache) OpenFile(options internal.OpenFileOptions) (*handlemap.Hand } // Open the file in write mode. - f, err = os.OpenFile(localPath, os.O_CREATE|os.O_RDWR, options.Mode) + f, err = os.OpenFile(localPath, os.O_CREATE|os.O_RDWR, options.Mode) // need these permissions for file share to work if err != nil { log.Err("FileCache::OpenFile : error creating new file %s [%s]", options.Name, err.Error()) return nil, err diff --git a/go.mod b/go.mod index 7b9242715..a382614dc 100755 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.4-0.20220425205405-09e6f201e1e4 github.com/Azure/azure-storage-azcopy/v10 v10.18.1 github.com/Azure/azure-storage-blob-go v0.15.0 + github.com/Azure/azure-storage-file-go v0.6.1-0.20220815164042-f37a99d62e3f github.com/Azure/go-autorest/autorest v0.11.29 github.com/Azure/go-autorest/autorest/adal v0.9.23 github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda @@ -27,38 +28,38 @@ require ( ) require ( - cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go v0.110.2 // indirect + cloud.google.com/go/compute v1.19.2 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.12.0 // indirect + cloud.google.com/go/iam v1.0.1 // indirect cloud.google.com/go/storage v1.30.1 // indirect - github.com/Azure/azure-storage-file-go v0.6.1-0.20201111053559-3c1754dc00a5 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/danieljoos/wincred v1.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-ini/ini v1.66.4 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect + github.com/google/s2a-go v0.1.3 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.7.1 // indirect + github.com/googleapis/gax-go/v2 v2.8.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/hillu/go-ntdll v0.0.0-20220217145204-be7b5318100d // indirect + github.com/hillu/go-ntdll v0.0.0-20230408164318-f8894bfa00af // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-ieproxy v0.0.3 // indirect + github.com/mattn/go-ieproxy v0.0.10 // indirect github.com/minio/minio-go v6.0.14+incompatible // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pkg/xattr v0.4.6 // indirect + github.com/pkg/xattr v0.4.9 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/afero v1.9.5 // indirect @@ -68,18 +69,18 @@ require ( github.com/wastore/keychain v0.0.0-20180920053336-f2c902a3d807 // indirect github.com/wastore/keyctl v0.3.1 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.8.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/oauth2 v0.6.0 // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/crypto v0.9.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/api v0.114.0 // indirect + google.golang.org/api v0.122.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 // indirect - google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.29.1 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/grpc v1.55.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect ) replace github.com/spf13/cobra => github.com/gapra-msft/cobra v1.4.1-0.20220411185530-5b83e8ba06dd diff --git a/go.sum b/go.sum index 6c58e8092..4c2aa8082 100644 --- a/go.sum +++ b/go.sum @@ -17,22 +17,22 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= -cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= -cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.2 h1:GbJtPo8OKVHbVep8jvM57KidbYHxeE68LOVqouNLrDY= +cloud.google.com/go/compute v1.19.2/go.mod h1:5f5a+iC1IriXYauaQ0EyQmEAEq9CGRnV5xJSQSlTV08= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= -cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v1.0.1 h1:lyeCAU6jpnVNrE9zGQkTl3WgNgK/X+uWwaw0kynZJMU= +cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -54,8 +54,8 @@ github.com/Azure/azure-storage-azcopy/v10 v10.18.1 h1:AT3PcMkLeLDXs+G6bPD9ZQ+T4Y github.com/Azure/azure-storage-azcopy/v10 v10.18.1/go.mod h1:pEhwI7Rr1V90GC/tCzPjU/Hi7QI08s0GaW6yIAcTP5w= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= -github.com/Azure/azure-storage-file-go v0.6.1-0.20201111053559-3c1754dc00a5 h1:aHEvBM4oXIWSTOVdL55nCYXO0Cl7ie3Ui5xMQhLVez8= -github.com/Azure/azure-storage-file-go v0.6.1-0.20201111053559-3c1754dc00a5/go.mod h1:++L7GP2pRyUNuastZ7m02vYV69JHmqlWXfCaGoL0v4s= +github.com/Azure/azure-storage-file-go v0.6.1-0.20220815164042-f37a99d62e3f h1:2UzXCkwhkGuCW2tkl2WOzjGrCWZj8Fn0AlM6D7kMN5k= +github.com/Azure/azure-storage-file-go v0.6.1-0.20220815164042-f37a99d62e3f/go.mod h1:++L7GP2pRyUNuastZ7m02vYV69JHmqlWXfCaGoL0v4s= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= @@ -79,7 +79,9 @@ github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda h1:NOo6+gM9NNP github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda/go.mod h1:2CaSFTh2ph9ymS6goiOKIBdfhwWUVsX4nQ5QjIYFHHs= github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY= github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -87,8 +89,13 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -99,6 +106,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= @@ -106,11 +114,12 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gapra-msft/cobra v1.4.1-0.20220411185530-5b83e8ba06dd h1:U3d5Jlb0ANsyxk2lnlhYh7/Ov4bZpIBUxJTsVuJM9G0= github.com/gapra-msft/cobra v1.4.1-0.20220411185530-5b83e8ba06dd/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.66.4 h1:dKjMqkcbkzfddhIhyglTPgMoJnkvmG+bSLrU9cTHc5M= -github.com/go-ini/ini v1.66.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= @@ -144,8 +153,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -177,6 +187,8 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.3 h1:FAgZmpLl/SXurPEZyCMPBIiiYeTbqfjlbdnCNTAkbGE= +github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -185,15 +197,16 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9 github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= -github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc= +github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hillu/go-ntdll v0.0.0-20220217145204-be7b5318100d h1:WR9P3d5kJAzyQrNROnys5/wElv7ELkFCJ9t6gObxQbs= -github.com/hillu/go-ntdll v0.0.0-20220217145204-be7b5318100d/go.mod h1:cHjYsnAnSckPDx8/H01Y+owD1hf2adLA6VRiw4guEbA= +github.com/hillu/go-ntdll v0.0.0-20230408164318-f8894bfa00af h1:6qDlty5ZH5RftbuLnx78wNTpPcWy3gUV1NbfBZR7e5g= +github.com/hillu/go-ntdll v0.0.0-20230408164318-f8894bfa00af/go.mod h1:cHjYsnAnSckPDx8/H01Y+owD1hf2adLA6VRiw4guEbA= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -214,8 +227,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= -github.com/mattn/go-ieproxy v0.0.3 h1:YkaHmK1CzE5C4O7A3hv3TCbfNDPSCf0RKZFX+VhBeYk= -github.com/mattn/go-ieproxy v0.0.3/go.mod h1:6ZpRmhBaYuBX1U2za+9rC9iCGLsSp2tftelZne7CPko= +github.com/mattn/go-ieproxy v0.0.10 h1:P+2QihaKCLgbs/32dhFLbxXlqsy8tIG1LUXHIoPaQPo= +github.com/mattn/go-ieproxy v0.0.10/go.mod h1:/NsJd+kxZBmjMc5hrJCKMbP57B84rvq9BiDRbtO9AS0= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -232,13 +245,14 @@ github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pkg/xattr v0.4.6 h1:0vqthLIMxQKA9VscyMcxjvAUGvyfzlk009vwLE8OZJg= -github.com/pkg/xattr v0.4.6/go.mod h1:sBD3RAqlr8Q+RC3FutZcikpT8nyDrIEEBw2J744gVWs= +github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= +github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8tUE= github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -288,6 +302,7 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -299,10 +314,11 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -376,11 +392,10 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -390,8 +405,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -404,8 +419,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -435,7 +450,6 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -447,7 +461,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220110181412-a018aaa089fe/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -465,6 +479,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -545,8 +560,8 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= -google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.122.0 h1:zDobeejm3E7pEG1mNHvdxvjs5XJoCMzyNH+CmwL94Es= +google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZOyms= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -578,6 +593,7 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -591,8 +607,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 h1:khxVcsk/FhnzxMKOyD+TDGwjbEOpcPuIpmafPGFmhMA= -google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -606,11 +622,14 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -623,8 +642,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= -google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -634,6 +653,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/setup/baseConfig.yaml b/setup/baseConfig.yaml index 684ab60b0..c6a6b6e68 100644 --- a/setup/baseConfig.yaml +++ b/setup/baseConfig.yaml @@ -110,7 +110,7 @@ loopbackfs: # Azure storage configuration azstorage: # Required - type: block|adls + type: block|adls|file account-name: container: endpoint: diff --git a/test/accoutcleanup/accountcleanup_test.go b/test/accountcleanup_container/accountcleanup_container_test.go similarity index 100% rename from test/accoutcleanup/accountcleanup_test.go rename to test/accountcleanup_container/accountcleanup_container_test.go diff --git a/test/accountcleanup_share/accountcleanup_share_test.go b/test/accountcleanup_share/accountcleanup_share_test.go new file mode 100644 index 000000000..4bde7aaf9 --- /dev/null +++ b/test/accountcleanup_share/accountcleanup_share_test.go @@ -0,0 +1,106 @@ +//go:build !unittest +// +build !unittest + +/* + _____ _____ _____ ____ ______ _____ ------ + | | | | | | | | | | | | | + | | | | | | | | | | | | | + | --- | | | | |-----| |---- | | |-----| |----- ------ + | | | | | | | | | | | | | + | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ + + + Licensed under the MIT License . + + Copyright © 2020-2023 Microsoft Corporation. All rights reserved. + Author : + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package account_cleanup + +import ( + "context" + "errors" + "log" + "net/url" + "os" + "regexp" + "testing" + + "github.com/Azure/azure-storage-file-go/azfile" +) + +func getGenericFileCredential() (*azfile.SharedKeyCredential, error) { + accountNameEnvVar := "STORAGE_ACCOUNT_NAME" + accountKeyEnvVar := "STORAGE_ACCOUNT_KEY" + accountName, accountKey := os.Getenv(accountNameEnvVar), os.Getenv(accountKeyEnvVar) + + if accountName == "" || accountKey == "" { + return nil, errors.New(accountNameEnvVar + " and/or " + accountKeyEnvVar + " environment variables not specified.") + } + return azfile.NewSharedKeyCredential(accountName, accountKey) +} + +func getGenericFSU() (azfile.ServiceURL, error) { + credential, err := getGenericFileCredential() + if err != nil { + return azfile.ServiceURL{}, err + } + + pipeline := azfile.NewPipeline(credential, azfile.PipelineOptions{}) + filePrimaryURL, _ := url.Parse("https://" + credential.AccountName() + ".file.core.windows.net/") + return azfile.NewServiceURL(*filePrimaryURL, pipeline), nil +} + +func TestDeleteAllTempShares(t *testing.T) { + ctx := context.Background() + fsu, err := getGenericFSU() + if err != nil { + log.Fatal(err) + } + + marker := azfile.Marker{} + pattern := "fuseut*" + + for marker.NotDone() { + resp, err := fsu.ListSharesSegment(ctx, marker, azfile.ListSharesOptions{}) + + if err != nil { + log.Fatal(err) + } + + for _, v := range resp.ShareItems { + matched, err := regexp.MatchString(pattern, v.Name) + if matched && err == nil { + shareURL := fsu.NewShareURL(v.Name) + shareURL.Delete(ctx, azfile.DeleteSnapshotsOptionNone) + t.Log("Deleting share :", v.Name) + } else { + t.Log("Skipping share :", v.Name) + } + } + marker = resp.NextMarker + } +} + +func TestMain(m *testing.M) { + m.Run() +} diff --git a/test/e2e_tests/data_validation_test.go b/test/e2e_tests/data_validation_test.go index 35b4bdde5..bdd08d303 100644 --- a/test/e2e_tests/data_validation_test.go +++ b/test/e2e_tests/data_validation_test.go @@ -54,6 +54,7 @@ import ( var dataValidationMntPathPtr string var dataValidationTempPathPtr string var dataValidationAdlsPtr string +var dataValidationFileSharePtr string var quickTest string var streamDirectTest string var distro string @@ -66,6 +67,7 @@ type dataValidationTestSuite struct { testLocalPath string testCachePath string adlsTest bool + fileShareTest bool } func regDataValidationTestFlag(p *string, name string, value string, usage string) { @@ -81,6 +83,7 @@ func getDataValidationTestFlag(name string) string { func initDataValidationFlags() { dataValidationMntPathPtr = getDataValidationTestFlag("mnt-path") dataValidationAdlsPtr = getDataValidationTestFlag("adls") + dataValidationFileSharePtr = getDataValidationTestFlag("fileshare") dataValidationTempPathPtr = getDataValidationTestFlag("tmp-path") quickTest = getDataValidationTestFlag("quick-test") streamDirectTest = getDataValidationTestFlag("stream-direct-test") @@ -395,6 +398,9 @@ func TestDataValidationTestSuite(t *testing.T) { if dataValidationAdlsPtr == "true" || dataValidationAdlsPtr == "True" { fmt.Println("ADLS Testing...") dataValidationTest.adlsTest = true + } else if dataValidationFileSharePtr == "true" || dataValidationFileSharePtr == "True" { + fmt.Println("FileShare Testing...") + dataValidationTest.fileShareTest = true } else { fmt.Println("BLOCK Blob Testing...") } @@ -428,6 +434,7 @@ func TestDataValidationTestSuite(t *testing.T) { func init() { regDataValidationTestFlag(&dataValidationMntPathPtr, "mnt-path", "", "Mount Path of Container") regDataValidationTestFlag(&dataValidationAdlsPtr, "adls", "", "Account is ADLS or not") + regDataValidationTestFlag(&dataValidationFileSharePtr, "fileshare", "", "Account is FileShare or not") regDataValidationTestFlag(&dataValidationTempPathPtr, "tmp-path", "", "Cache dir path") regDataValidationTestFlag(&quickTest, "quick-test", "true", "Run quick tests") regDataValidationTestFlag(&streamDirectTest, "stream-direct-test", "false", "Run stream direct tests") diff --git a/test/e2e_tests/dir_test.go b/test/e2e_tests/dir_test.go index fc48af7b6..83dcfd79b 100644 --- a/test/e2e_tests/dir_test.go +++ b/test/e2e_tests/dir_test.go @@ -53,15 +53,17 @@ import ( type dirTestSuite struct { suite.Suite - testPath string - adlsTest bool - minBuff []byte - medBuff []byte - hugeBuff []byte + testPath string + adlsTest bool + fileShareTest bool + minBuff []byte + medBuff []byte + hugeBuff []byte } var pathPtr string var adlsPtr string +var fileSharePtr string var clonePtr string var streamDirectPtr string @@ -78,6 +80,7 @@ func getDirTestFlag(name string) string { func initDirFlags() { pathPtr = getDirTestFlag("mnt-path") adlsPtr = getDirTestFlag("adls") + fileSharePtr = getDirTestFlag("fileshare") clonePtr = getDirTestFlag("clone") streamDirectPtr = getDirTestFlag("stream-direct-test") } @@ -122,23 +125,39 @@ func (suite *dirTestSuite) TestDirCreateDuplicate() { } // # Create Directory with special characters in name +// For fileshare, skip tests that include Greek/Arabic letters and slashes in the file/dir name as these are not supported according to documentation +// https://docs.microsoft.com/en-us/rest/api/storageservices/create-directory +// https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-shares--directories--files--and-metadata func (suite *dirTestSuite) TestDirCreateSplChar() { + if suite.fileShareTest { + fmt.Println("Skipping this test case for file share") + return + } dirName := suite.testPath + "/" + "@#$^&*()_+=-{}[]|?><.,~" err := os.Mkdir(dirName, 0777) suite.Equal(nil, err) // cleanup suite.dirTestCleanup([]string{dirName}) + } // # Create Directory with slash in name +// For fileshare, skip tests that include Greek/Arabic letters and slashes in the file/dir name as these are not supported according to documentation +// https://docs.microsoft.com/en-us/rest/api/storageservices/create-directory +// https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-shares--directories--files--and-metadata func (suite *dirTestSuite) TestDirCreateSlashChar() { + if suite.fileShareTest { + fmt.Println("Skipping this test case for file share") + return + } dirName := suite.testPath + "/" + "PRQ\\STUV" err := os.Mkdir(dirName, 0777) suite.Equal(nil, err) // cleanup suite.dirTestCleanup([]string{dirName}) + } // # Rename a directory @@ -526,6 +545,12 @@ func TestDirTestSuite(t *testing.T) { if adlsPtr == "true" || adlsPtr == "True" { fmt.Println("ADLS Testing...") dirTest.adlsTest = true + } else if fileSharePtr == "true" || fileSharePtr == "True" { + fmt.Println("FileShare Testing...") + dirTest.fileShareTest = true + // For fileshare, skip tests that include Greek/Arabic letters and slashes in the file/dir name as these are not supported according to documentation + // https://docs.microsoft.com/en-us/rest/api/storageservices/create-directory + // https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-shares--directories--files--and-metadata } else { fmt.Println("BLOCK Blob Testing...") } @@ -554,5 +579,6 @@ func TestDirTestSuite(t *testing.T) { func init() { regDirTestFlag(&pathPtr, "mnt-path", "", "Mount Path of Container") regDirTestFlag(&adlsPtr, "adls", "", "Account is ADLS or not") + regDirTestFlag(&fileSharePtr, "fileshare", "", "Account is FileShare or not") regFileTestFlag(&fileTestGitClonePtr, "clone", "", "Git clone test is enable or not") } diff --git a/test/e2e_tests/file_test.go b/test/e2e_tests/file_test.go index 1e527323d..d9408817b 100644 --- a/test/e2e_tests/file_test.go +++ b/test/e2e_tests/file_test.go @@ -52,17 +52,19 @@ import ( var fileTestPathPtr string var fileTestAdlsPtr string +var fileTestFileSharePtr string var fileTestGitClonePtr string var fileTestStreamDirectPtr string var fileTestDistroName string type fileTestSuite struct { suite.Suite - testPath string - adlsTest bool - minBuff []byte - medBuff []byte - hugeBuff []byte + testPath string + adlsTest bool + fileShareTest bool + minBuff []byte + medBuff []byte + hugeBuff []byte } func regFileTestFlag(p *string, name string, value string, usage string) { @@ -78,6 +80,7 @@ func getFileTestFlag(name string) string { func initFileFlags() { fileTestPathPtr = getFileTestFlag("mnt-path") fileTestAdlsPtr = getFileTestFlag("adls") + fileTestFileSharePtr = getDataValidationTestFlag("fileshare") fileTestGitClonePtr = getFileTestFlag("clone") fileTestStreamDirectPtr = getFileTestFlag("stream-direct-test") fileTestDistroName = getFileTestFlag("distro-name") @@ -118,7 +121,14 @@ func (suite *fileTestSuite) TestFileCreateUtf8Char() { suite.fileTestCleanup([]string{fileName}) } +// For fileshare, skip tests that include Greek/Arabic letters and slashes in the file/dir name as these are not supported according to documentation +// https://docs.microsoft.com/en-us/rest/api/storageservices/create-directory +// https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-shares--directories--files--and-metadata func (suite *fileTestSuite) TestFileCreatSpclChar() { + if suite.fileShareTest { + fmt.Println("Skipping this test case for file share") + return + } speclChar := "abcd%23ABCD%34123-._~!$&'()*+,;=!@ΣΑΠΦΩ$भारत.txt" fileName := suite.testPath + "/" + speclChar @@ -143,6 +153,7 @@ func (suite *fileTestSuite) TestFileCreatSpclChar() { suite.Equal(true, found) suite.fileTestCleanup([]string{fileName}) + } func (suite *fileTestSuite) TestFileCreatEncodeChar() { @@ -172,7 +183,14 @@ func (suite *fileTestSuite) TestFileCreatEncodeChar() { suite.fileTestCleanup([]string{fileName}) } +// For fileshare, skip tests that include Greek/Arabic letters and slashes in the file/dir name as these are not supported according to documentation +// https://docs.microsoft.com/en-us/rest/api/storageservices/create-directory +// https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-shares--directories--files--and-metadata func (suite *fileTestSuite) TestFileCreateMultiSpclCharWithinSpclDir() { + if suite.fileShareTest { + fmt.Println("Skipping this test case for file share") + return + } speclChar := "abcd%23ABCD%34123-._~!$&'()*+,;=!@ΣΑΠΦΩ$भारत.txt" speclDirName := suite.testPath + "/" + "abc%23%24%25efg-._~!$&'()*+,;=!@ΣΑΠΦΩ$भारत" secFile := speclDirName + "/" + "abcd123~!@#$%^&*()_+=-{}][\":;'?><,.|\\abcd123~!@#$%^&*()_+=-{}][\":;'?><,.|.txt" @@ -206,6 +224,7 @@ func (suite *fileTestSuite) TestFileCreateMultiSpclCharWithinSpclDir() { suite.Equal(true, found) suite.fileTestCleanup([]string{fileName, secFile, speclDirName + "/" + "abcd123~!@#$%^&*()_+=-{}][\":;'?><,.|", speclDirName}) + } func (suite *fileTestSuite) TestFileCreateLongName() { @@ -217,7 +236,14 @@ func (suite *fileTestSuite) TestFileCreateLongName() { suite.fileTestCleanup([]string{fileName}) } +// For fileshare, skip tests that include Greek/Arabic letters and slashes in the file/dir name as these are not supported according to documentation +// https://docs.microsoft.com/en-us/rest/api/storageservices/create-directory +// https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-shares--directories--files--and-metadata func (suite *fileTestSuite) TestFileCreateSlashName() { + if suite.fileShareTest { + fmt.Println("Skipping this test case for file share") + return + } fileName := suite.testPath + "/abcd\\efg.txt" srcFile, err := os.OpenFile(fileName, os.O_CREATE, 0777) @@ -225,6 +251,7 @@ func (suite *fileTestSuite) TestFileCreateSlashName() { srcFile.Close() suite.fileTestCleanup([]string{fileName}) + } func (suite *fileTestSuite) TestFileCreateLabel() { @@ -591,6 +618,13 @@ func TestFileTestSuite(t *testing.T) { if fileTestAdlsPtr == "true" || fileTestAdlsPtr == "True" { fmt.Println("ADLS Testing...") fileTest.adlsTest = true + } else if fileTestFileSharePtr == "true" || fileTestFileSharePtr == "True" { + fmt.Println("FileShare Testing...") + fileTest.fileShareTest = true + // For fileshare, skip tests that include Greek/Arabic letters and slashes in the file/dir name as these are not supported according to documentation + // https://docs.microsoft.com/en-us/rest/api/storageservices/create-directory + // https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-shares--directories--files--and-metadata + } else { fmt.Println("BLOCK Blob Testing...") } @@ -618,5 +652,6 @@ func TestFileTestSuite(t *testing.T) { func init() { regFileTestFlag(&fileTestPathPtr, "mnt-path", "", "Mount Path of Container") regFileTestFlag(&fileTestAdlsPtr, "adls", "", "Account is ADLS or not") + regFileTestFlag(&fileTestFileSharePtr, "fileshare", "", "Account is FileShare or not") regFileTestFlag(&fileTestGitClonePtr, "clone", "", "Git clone test is enable or not") }