From 670edc02ef768c1256cc8a77b53f1daae9a11ffb Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Fri, 17 Jan 2025 19:56:10 +0300 Subject: [PATCH] Revert "Export views (#12824)" This reverts commit 8c2ef27eded51a36fd5c3c9abe4f89221d04e9c0. --- ydb/core/tx/schemeshard/schemeshard__init.cpp | 1 - .../tx/schemeshard/schemeshard_export.cpp | 7 +- .../schemeshard_export__create.cpp | 146 +------- .../schemeshard_export_flow_proposals.cpp | 3 - .../schemeshard_export_scheme_uploader.cpp | 317 ------------------ .../schemeshard_export_scheme_uploader.h | 16 - ...eshard_export_scheme_uploader_fallback.cpp | 39 --- ydb/core/tx/schemeshard/schemeshard_impl.cpp | 10 - ydb/core/tx/schemeshard/schemeshard_impl.h | 2 - .../tx/schemeshard/schemeshard_info_types.h | 5 +- ydb/core/tx/schemeshard/schemeshard_path.cpp | 14 - ydb/core/tx/schemeshard/schemeshard_path.h | 1 - ydb/core/tx/schemeshard/schemeshard_private.h | 20 -- ydb/core/tx/schemeshard/schemeshard_schema.h | 4 +- ydb/core/tx/schemeshard/ya.make | 3 - ydb/library/backup/backup.cpp | 76 ++++- .../lib/ydb_cli/dump/util/view_utils.cpp | 86 +---- ydb/public/lib/ydb_cli/dump/util/view_utils.h | 19 +- .../flat_schemeshard.schema | 8 +- 19 files changed, 103 insertions(+), 674 deletions(-) delete mode 100644 ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.cpp delete mode 100644 ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.h delete mode 100644 ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader_fallback.cpp diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 63df4a3729e1..93f41ff501d9 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4326,7 +4326,6 @@ struct TSchemeShard::TTxInit : public TTransactionBase { item.SourcePathId.OwnerId = rowset.GetValueOrDefault(selfId); item.SourcePathId.LocalPathId = rowset.GetValue(); - item.SourcePathType = rowset.GetValue(); item.State = static_cast(rowset.GetValue()); item.WaitTxId = rowset.GetValueOrDefault(InvalidTxId); diff --git a/ydb/core/tx/schemeshard/schemeshard_export.cpp b/ydb/core/tx/schemeshard/schemeshard_export.cpp index cba13774cc96..ea6f881ee39a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export.cpp @@ -170,8 +170,7 @@ void TSchemeShard::PersistCreateExport(NIceDb::TNiceDb& db, const TExportInfo::T NIceDb::TUpdate(item.SourcePathName), NIceDb::TUpdate(item.SourcePathId.OwnerId), NIceDb::TUpdate(item.SourcePathId.LocalPathId), - NIceDb::TUpdate(static_cast(item.State)), - NIceDb::TUpdate(item.SourcePathType) + NIceDb::TUpdate(static_cast(item.State)) ); } } @@ -232,10 +231,6 @@ void TSchemeShard::Handle(TEvExport::TEvListExportsRequest::TPtr& ev, const TAct Execute(CreateTxListExports(ev), ctx); } -void TSchemeShard::Handle(TEvPrivate::TEvExportSchemeUploadResult::TPtr& ev, const TActorContext& ctx) { - Execute(CreateTxProgressExport(ev), ctx); -} - void TSchemeShard::ResumeExports(const TVector& exportIds, const TActorContext& ctx) { for (const ui64 id : exportIds) { Execute(CreateTxProgressExport(id), ctx); diff --git a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp index 4705dfe6b638..7f63a60a7970 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp @@ -1,9 +1,8 @@ #include "schemeshard_xxport__tx_base.h" #include "schemeshard_xxport__helpers.h" -#include "schemeshard_export.h" #include "schemeshard_export_flow_proposals.h" #include "schemeshard_export_helpers.h" -#include "schemeshard_export_scheme_uploader.h" +#include "schemeshard_export.h" #include "schemeshard_audit_log.h" #include "schemeshard_impl.h" @@ -204,7 +203,7 @@ struct TSchemeShard::TExport::TTxCreate: public TSchemeShard::TXxport::TTxBase { .IsResolved() .NotDeleted() .NotUnderDeleting() - .IsSupportedInExports() + .IsTable() .FailOnRestrictedCreateInTempZone(); if (!checks) { @@ -213,7 +212,7 @@ struct TSchemeShard::TExport::TTxCreate: public TSchemeShard::TXxport::TTxBase { } } - exportInfo->Items.emplace_back(item.source_path(), path.Base()->PathId, path->PathType); + exportInfo->Items.emplace_back(item.source_path(), path.Base()->PathId); exportInfo->PendingItems.push_back(itemIdx); } @@ -231,7 +230,6 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase ui64 Id; TEvTxAllocatorClient::TEvAllocateResult::TPtr AllocateResult = nullptr; TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr ModifyResult = nullptr; - TEvPrivate::TEvExportSchemeUploadResult::TPtr SchemeUploadResult = nullptr; TTxId CompletedTxId = InvalidTxId; explicit TTxProgress(TSelf* self, ui64 id) @@ -252,12 +250,6 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase { } - explicit TTxProgress(TSelf* self, TEvPrivate::TEvExportSchemeUploadResult::TPtr& ev) - : TXxport::TTxBase(self) - , SchemeUploadResult(ev) - { - } - explicit TTxProgress(TSelf* self, TTxId completedTxId) : TXxport::TTxBase(self) , CompletedTxId(completedTxId) @@ -275,8 +267,6 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase OnAllocateResult(txc, ctx); } else if (ModifyResult) { OnModifyResult(txc, ctx); - } else if (SchemeUploadResult) { - OnSchemeUploadResult(txc, ctx); } else if (CompletedTxId) { OnNotifyResult(txc, ctx); } else { @@ -300,22 +290,16 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase Send(Self->SelfId(), MkDirPropose(Self, txId, exportInfo)); } - bool CopyTables(TExportInfo::TPtr exportInfo, TTxId txId) { + void CopyTables(TExportInfo::TPtr exportInfo, TTxId txId) { LOG_I("TExport::TTxProgress: CopyTables propose" << ": info# " << exportInfo->ToString() << ", txId# " << txId); Y_ABORT_UNLESS(exportInfo->WaitTxId == InvalidTxId); - if (AnyOf(exportInfo->Items, [](const TExportInfo::TItem& item) { - return item.SourcePathType == NKikimrSchemeOp::EPathTypeTable; - })) { - Send(Self->SelfId(), CopyTablesPropose(Self, txId, exportInfo)); - return true; - } - return false; + Send(Self->SelfId(), CopyTablesPropose(Self, txId, exportInfo)); } - void TransferData(TExportInfo::TPtr exportInfo, ui32 itemIdx, TTxId txId, const TActorContext& ctx) { + void TransferData(TExportInfo::TPtr exportInfo, ui32 itemIdx, TTxId txId) { Y_ABORT_UNLESS(itemIdx < exportInfo->Items.size()); auto& item = exportInfo->Items.at(itemIdx); @@ -327,17 +311,7 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase << ", txId# " << txId); Y_ABORT_UNLESS(item.WaitTxId == InvalidTxId); - if (item.SourcePathType == NKikimrSchemeOp::EPathTypeTable) { - Send(Self->SelfId(), BackupPropose(Self, txId, exportInfo, itemIdx)); - } else if (item.SourcePathType == NKikimrSchemeOp::EPathTypeView) { - Ydb::Export::ExportToS3Settings exportSettings; - Y_ABORT_UNLESS(exportSettings.ParseFromString(exportInfo->Settings)); - const auto databaseRoot = TStringBuilder() << '/' << JoinSeq('/', Self->RootPathElements); - - item.SchemeUploader = ctx.Register(CreateSchemeUploader( - Self->SelfId(), exportInfo->Id, itemIdx, item.SourcePathId, exportSettings, databaseRoot - )); - } + Send(Self->SelfId(), BackupPropose(Self, txId, exportInfo, itemIdx)); } bool CancelTransferring(TExportInfo::TPtr exportInfo, ui32 itemIdx) { @@ -646,7 +620,7 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase } } - void OnAllocateResult(TTransactionContext& txc, const TActorContext& ctx) { + void OnAllocateResult(TTransactionContext&, const TActorContext&) { Y_ABORT_UNLESS(AllocateResult); const auto txId = TTxId(AllocateResult->Get()->TxIds.front()); @@ -677,27 +651,13 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase break; case EState::CopyTables: - if (!CopyTables(exportInfo, txId)) { - // none of the items is a table - NIceDb::TNiceDb db(txc.DB); - - for (ui32 itemIdx : xrange(exportInfo->Items.size())) { - exportInfo->Items[itemIdx].State = EState::Transferring; - Self->PersistExportItemState(db, exportInfo, itemIdx); - - AllocateTxId(exportInfo, itemIdx); - } - - exportInfo->State = EState::Transferring; - Self->PersistExportState(db, exportInfo); - return; - } + CopyTables(exportInfo, txId); break; case EState::Transferring: if (exportInfo->PendingItems) { itemIdx = popPendingItemIdx(exportInfo->PendingItems); - TransferData(exportInfo, itemIdx, txId, ctx); + TransferData(exportInfo, itemIdx, txId); } else { return; } @@ -910,71 +870,6 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase SubscribeTx(txId); } - void OnSchemeUploadResult(TTransactionContext& txc, const TActorContext& ctx) { - Y_ABORT_UNLESS(SchemeUploadResult); - - LOG_D("TExport::TTxProgress: OnSchemeUploadResult" - << ": success# " << SchemeUploadResult->Get()->Success - << ", error# " << SchemeUploadResult->Get()->Error - ); - - const auto exportId = SchemeUploadResult->Get()->ExportId; - auto exportInfo = Self->Exports.Value(exportId, nullptr); - if (!exportInfo) { - LOG_E("TExport::TTxProgress: OnSchemeUploadResult received unknown export id" - << ": id# " << exportId - ); - return; - } - - ui32 itemIdx = SchemeUploadResult->Get()->ItemIdx; - if (itemIdx >= exportInfo->Items.size()) { - LOG_E("TExport::TTxProgress: OnSchemeUploadResult item index out of range" - << ": id# " << exportId - << ", item index# " << itemIdx - << ", number of items# " << exportInfo->Items.size() - ); - return; - } - - NIceDb::TNiceDb db(txc.DB); - - auto& item = exportInfo->Items[itemIdx]; - item.SchemeUploader = TActorId(); - - if (!SchemeUploadResult->Get()->Success) { - item.State = EState::Cancelled; - item.Issue = SchemeUploadResult->Get()->Error; - Self->PersistExportItemState(db, exportInfo, itemIdx); - - if (!exportInfo->IsInProgress()) { - return; - } - - Cancel(exportInfo, itemIdx, "unsuccessful scheme upload"); - - Self->PersistExportState(db, exportInfo); - return SendNotificationsIfFinished(exportInfo); - } - - if (exportInfo->State == EState::Transferring) { - item.State = EState::Done; - Self->PersistExportItemState(db, exportInfo, itemIdx); - - if (AllOf(exportInfo->Items, &TExportInfo::TItem::IsDone)) { - exportInfo->State = EState::Done; - exportInfo->EndTime = TAppData::TimeProvider->Now(); - - Self->PersistExportState(db, exportInfo); - SendNotificationsIfFinished(exportInfo); - - if (exportInfo->IsFinished()) { - AuditLogExportEnd(*exportInfo.Get(), Self); - } - } - } - } - void OnNotifyResult(TTransactionContext& txc, const TActorContext&) { Y_ABORT_UNLESS(CompletedTxId); LOG_D("TExport::TTxProgress: OnNotifyResult" @@ -1053,18 +948,15 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase item.State = EState::Done; item.WaitTxId = InvalidTxId; - bool itemHasIssues = false; - if (item.SourcePathType == NKikimrSchemeOp::EPathTypeTable) { - if (const auto issue = GetIssues(ItemPathId(Self, exportInfo, itemIdx), txId)) { - item.Issue = *issue; - Cancel(exportInfo, itemIdx, "issues during backing up"); - itemHasIssues = true; + if (const auto issue = GetIssues(ItemPathId(Self, exportInfo, itemIdx), txId)) { + item.Issue = *issue; + Cancel(exportInfo, itemIdx, "issues during backing up"); + } else { + if (AllOf(exportInfo->Items, &TExportInfo::TItem::IsDone)) { + exportInfo->State = EState::Done; + exportInfo->EndTime = TAppData::TimeProvider->Now(); } } - if (!itemHasIssues && AllOf(exportInfo->Items, &TExportInfo::TItem::IsDone)) { - exportInfo->State = EState::Done; - exportInfo->EndTime = TAppData::TimeProvider->Now(); - } Self->PersistExportItemState(db, exportInfo, itemIdx); break; @@ -1124,10 +1016,6 @@ ITransaction* TSchemeShard::CreateTxProgressExport(TEvSchemeShard::TEvModifySche return new TExport::TTxProgress(this, ev); } -ITransaction* TSchemeShard::CreateTxProgressExport(TEvPrivate::TEvExportSchemeUploadResult::TPtr& ev) { - return new TExport::TTxProgress(this, ev); -} - ITransaction* TSchemeShard::CreateTxProgressExport(TTxId completedTxId) { return new TExport::TTxProgress(this, completedTxId); } diff --git a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp index b927c91bf147..0a0e2f24cef2 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp @@ -59,9 +59,6 @@ THolder CopyTablesPropose( for (ui32 itemIdx : xrange(exportInfo->Items.size())) { const auto& item = exportInfo->Items.at(itemIdx); - if (item.SourcePathType != NKikimrSchemeOp::EPathTypeTable) { - continue; - } auto& desc = *copyTables.Add(); desc.SetSrcPath(item.SourcePathName); diff --git a/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.cpp b/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.cpp deleted file mode 100644 index 90bd2e48e3f1..000000000000 --- a/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.cpp +++ /dev/null @@ -1,317 +0,0 @@ -#include "schemeshard.h" -#include "schemeshard_export_scheme_uploader.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace NKikimr::NSchemeShard { - -class TSchemeUploader: public TActorBootstrapped { - - using TS3ExternalStorageConfig = NWrappers::NExternalStorage::TS3ExternalStorageConfig; - using TEvExternalStorage = NWrappers::TEvExternalStorage; - using TPutObjectResult = Aws::Utils::Outcome; - - static TString BuildViewScheme(const TString& path, const NKikimrSchemeOp::TViewDescription& viewDescription, const TString& backupRoot, TString& error) { - NYql::TIssues issues; - auto scheme = NYdb::NDump::BuildCreateViewQuery(viewDescription.GetName(), path, viewDescription.GetQueryText(), backupRoot, issues); - if (!scheme) { - error = issues.ToString(); - } - return scheme; - } - - bool BuildSchemeToUpload(const NKikimrScheme::TEvDescribeSchemeResult& describeResult, TString& error) { - const auto pathType = describeResult.GetPathDescription().GetSelf().GetPathType(); - switch (pathType) { - case NKikimrSchemeOp::EPathTypeView: { - Scheme = BuildViewScheme(describeResult.GetPath(), describeResult.GetPathDescription().GetViewDescription(), DatabaseRoot, error); - return !Scheme.empty(); - } - default: - error = TStringBuilder() << "unsupported path type: " << pathType; - return false; - } - } - - void HandleSchemeDescription(TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) { - const auto& describeResult = ev->Get()->GetRecord(); - - LOG_D("HandleSchemeDescription TEvSchemeShard::TEvDescribeSchemeResult" - << ", self: " << this->SelfId() - << ", status: " << describeResult.GetStatus() - ); - - auto reportError = [&](const TString& error) { - Send(SchemeShard, new TEvPrivate::TEvExportSchemeUploadResult( - ExportId, ItemIdx, false, error - )); - }; - - if (describeResult.GetStatus() != TEvSchemeShard::EStatus::StatusSuccess) { - reportError(describeResult.GetReason()); - return; - } - - TString error; - if (!BuildSchemeToUpload(describeResult, error)) { - reportError(error); - return; - } - - if (auto permissions = NDataShard::GenYdbPermissions(describeResult.GetPathDescription())) { - google::protobuf::TextFormat::PrintToString(permissions.GetRef(), &Permissions); - } else { - reportError("cannot infer permissions"); - return; - } - - Restart(); - } - - void Restart() { - if (Attempt) { - this->Send(std::exchange(StorageOperator, TActorId()), new TEvents::TEvPoisonPill()); - } - - StorageOperator = this->RegisterWithSameMailbox( - NWrappers::CreateS3Wrapper(ExternalStorageConfig->ConstructStorageOperator()) - ); - - if (!SchemeUploaded) { - UploadScheme(); - } else if (!PermissionsUploaded) { - UploadPermissions(); - } - } - - void UploadScheme() { - Y_ABORT_UNLESS(!SchemeUploaded); - - if (!Scheme) { - return Finish(false, "cannot infer scheme"); - } - auto request = Aws::S3::Model::PutObjectRequest() - .WithKey(Sprintf("%s/create_view.sql", DestinationPrefix.c_str())); - - this->Send(StorageOperator, new TEvExternalStorage::TEvPutObjectRequest(request, TString(Scheme))); - this->Become(&TThis::StateUploadScheme); - } - - void HandleSchemePutResponse(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - LOG_D("HandleSchemePutResponse TEvExternalStorage::TEvPutObjectResponse" - << ", self: " << this->SelfId() - << ", result: " << result - ); - - if (!CheckResult(result, TStringBuf("PutObject (scheme)"))) { - return; - } - SchemeUploaded = true; - UploadPermissions(); - } - - void UploadPermissions() { - Y_ABORT_UNLESS(!PermissionsUploaded); - - if (!Permissions) { - return Finish(false, "cannot infer permissions"); - } - auto request = Aws::S3::Model::PutObjectRequest() - .WithKey(Sprintf("%s/permissions.pb", DestinationPrefix.c_str())); - - this->Send(StorageOperator, new TEvExternalStorage::TEvPutObjectRequest(request, TString(Permissions))); - this->Become(&TThis::StateUploadPermissions); - } - - void HandlePermissionsPutResponse(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { - const auto& result = ev->Get()->Result; - - LOG_D("HandlePermissionsPutResponse TEvExternalStorage::TEvPutObjectResponse" - << ", self: " << this->SelfId() - << ", result: " << result - ); - - if (!CheckResult(result, TStringBuf("PutObject (permissions)"))) { - return; - } - PermissionsUploaded = true; - Finish(); - } - - bool CheckResult(const TPutObjectResult& result, const TStringBuf marker) { - if (result.IsSuccess()) { - return true; - } - - LOG_E("Error at '" << marker << "'" - << ", self: " << this->SelfId() - << ", error: " << result - ); - - RetryOrFinish(result.GetError()); - return false; - } - - static bool ShouldRetry(const Aws::S3::S3Error& error) { - if (error.ShouldRetry()) { - return true; - } - return error.GetExceptionName() == "TooManyRequests"; - } - - void Retry() { - Delay = Min(Delay * ++Attempt, MaxDelay); - const TDuration random = TDuration::FromValue(TAppData::RandomProvider->GenRand64() % Delay.MicroSeconds()); - this->Schedule(Delay + random, new TEvents::TEvWakeup()); - } - - void RetryOrFinish(const Aws::S3::S3Error& error) { - if (Attempt < Retries && ShouldRetry(error)) { - Retry(); - } else { - Finish(false, TStringBuilder() << "S3 error: " << error.GetMessage()); - } - } - - void Finish(bool success = true, const TString& error = TString()) { - LOG_I("Finish" - << ", self: " << this->SelfId() - << ", success: " << success - << ", error: " << error - ); - - Send(SchemeShard, new TEvPrivate::TEvExportSchemeUploadResult(ExportId, ItemIdx, success, error)); - PassAway(); - } - - void PassAway() override { - this->Send(StorageOperator, new TEvents::TEvPoisonPill()); - IActor::PassAway(); - } - -public: - - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::EXPORT_S3_UPLOADER_ACTOR; - } - - TSchemeUploader( - TActorId schemeShard, - ui64 exportId, - ui32 itemIdx, - TPathId sourcePathId, - const Ydb::Export::ExportToS3Settings& settings, - const TString& databaseRoot - ) - : SchemeShard(schemeShard) - , ExportId(exportId) - , ItemIdx(itemIdx) - , SourcePathId(sourcePathId) - , ExternalStorageConfig(new TS3ExternalStorageConfig(settings)) - , Retries(settings.number_of_retries()) - , DatabaseRoot(databaseRoot) - { - if (itemIdx < ui32(settings.items_size())) { - DestinationPrefix = settings.items(itemIdx).destination_prefix(); - } - } - - void Bootstrap() { - if (!DestinationPrefix) { - Finish(false, TStringBuilder() << "cannot determine destination prefix, item index: " << ItemIdx << " out of range"); - return; - } - if (!Scheme || !Permissions) { - Send(SchemeShard, new TEvSchemeShard::TEvDescribeScheme(SourcePathId)); - this->Become(&TThis::StateDescribe); - return; - } - if (!SchemeUploaded) { - UploadScheme(); - return; - } - if (!PermissionsUploaded) { - UploadPermissions(); - return; - } - Finish(); - } - - STATEFN(StateBase) { - switch (ev->GetTypeRewrite()) { - sFunc(TEvents::TEvWakeup, Bootstrap); - sFunc(TEvents::TEvPoisonPill, PassAway); - } - } - - STATEFN(StateDescribe) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvSchemeShard::TEvDescribeSchemeResult, HandleSchemeDescription); - default: - return StateBase(ev); - } - } - - STATEFN(StateUploadScheme) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvExternalStorage::TEvPutObjectResponse, HandleSchemePutResponse); - default: - return StateBase(ev); - } - } - - STATEFN(StateUploadPermissions) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvExternalStorage::TEvPutObjectResponse, HandlePermissionsPutResponse); - default: - return StateBase(ev); - } - } - -private: - - TActorId SchemeShard; - - ui64 ExportId; - ui32 ItemIdx; - TPathId SourcePathId; - - NWrappers::IExternalStorageConfig::TPtr ExternalStorageConfig; - TString DestinationPrefix; - - ui32 Attempt = 0; - const ui32 Retries; - - TString DatabaseRoot; - - TActorId StorageOperator; - - TDuration Delay = TDuration::Minutes(1); - static constexpr TDuration MaxDelay = TDuration::Minutes(10); - - TString Scheme; - bool SchemeUploaded = false; - - TString Permissions; - bool PermissionsUploaded = false; - -}; // TSchemeUploader - -IActor* CreateSchemeUploader(TActorId schemeShard, ui64 exportId, ui32 itemIdx, TPathId sourcePathId, - const Ydb::Export::ExportToS3Settings& settings, const TString& databaseRoot -) { - return new TSchemeUploader(schemeShard, exportId, itemIdx, sourcePathId, settings, databaseRoot); -} - -} // NSchemeShard::NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.h b/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.h deleted file mode 100644 index 2262ab4cf674..000000000000 --- a/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include -#include - -namespace Ydb::Export { - class ExportToS3Settings; -} - -namespace NKikimr::NSchemeShard { - -NActors::IActor* CreateSchemeUploader(NActors::TActorId schemeShard, ui64 exportId, ui32 itemIdx, TPathId sourcePathId, - const Ydb::Export::ExportToS3Settings& settings, const TString& databaseRoot -); - -} diff --git a/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader_fallback.cpp b/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader_fallback.cpp deleted file mode 100644 index 3afbc69d61c5..000000000000 --- a/ydb/core/tx/schemeshard/schemeshard_export_scheme_uploader_fallback.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include "schemeshard_export_scheme_uploader.h" - -#include -#include - -using namespace NActors; - -namespace NKikimr::NSchemeShard { - -class TSchemeUploaderFallback: public TActorBootstrapped { -public: - explicit TSchemeUploaderFallback(TActorId schemeShard, ui64 exportId, ui32 itemIdx) - : SchemeShard(schemeShard) - , ExportId(exportId) - , ItemIdx(itemIdx) - { - } - - void Bootstrap() { - Send(SchemeShard, new TEvPrivate::TEvExportSchemeUploadResult(ExportId, ItemIdx, false, - "Exports to S3 are disabled" - )); - PassAway(); - } - -private: - TActorId SchemeShard; - ui64 ExportId; - ui32 ItemIdx; -}; - - -IActor* CreateSchemeUploader(TActorId schemeShard, ui64 exportId, ui32 itemIdx, TPathId sourcePathId, - const Ydb::Export::ExportToS3Settings& settings, const TString& databaseRoot -) { - return new TSchemeUploaderFallback(schemeShard, exportId, itemIdx); -} - -} // NSchemeShard::NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 6f4fc8471f13..75485145b536 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -4556,15 +4556,6 @@ void TSchemeShard::Die(const TActorContext &ctx) { if (TabletMigrator) { ctx.Send(TabletMigrator, new TEvents::TEvPoisonPill()); } - for (const auto& [id, exportInfo] : Exports) { - if (!exportInfo->IsDone()) { - for (const auto& item : exportInfo->Items) { - if (item.SchemeUploader != TActorId()) { - ctx.Send(item.SchemeUploader, new TEvents::TEvPoisonPill()); - } - } - } - } IndexBuildPipes.Shutdown(ctx); CdcStreamScanPipes.Shutdown(ctx); @@ -4853,7 +4844,6 @@ void TSchemeShard::StateWork(STFUNC_SIG) { HFuncTraced(TEvExport::TEvListExportsRequest, Handle); // } // NExport HFuncTraced(NBackground::TEvListRequest, Handle); - HFuncTraced(TEvPrivate::TEvExportSchemeUploadResult, Handle); // namespace NImport { HFuncTraced(TEvImport::TEvCreateImportRequest, Handle); diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 39e6ec1318d8..68aef64b5e07 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -1233,7 +1233,6 @@ class TSchemeShard NTabletFlatExecutor::ITransaction* CreateTxProgressExport(TEvTxAllocatorClient::TEvAllocateResult::TPtr& ev); NTabletFlatExecutor::ITransaction* CreateTxProgressExport(TEvSchemeShard::TEvModifySchemeTransactionResult::TPtr& ev); NTabletFlatExecutor::ITransaction* CreateTxProgressExport(TTxId completedTxId); - NTabletFlatExecutor::ITransaction* CreateTxProgressExport(TEvPrivate::TEvExportSchemeUploadResult::TPtr& ev); void Handle(TEvExport::TEvCreateExportRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvExport::TEvGetExportRequest::TPtr& ev, const TActorContext& ctx); @@ -1241,7 +1240,6 @@ class TSchemeShard void Handle(TEvExport::TEvForgetExportRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvExport::TEvListExportsRequest::TPtr& ev, const TActorContext& ctx); void Handle(TAutoPtr>& ev, const TActorContext& ctx); - void Handle(TEvPrivate::TEvExportSchemeUploadResult::TPtr& ev, const TActorContext& ctx); void ResumeExports(const TVector& exportIds, const TActorContext& ctx); // } // NExport diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 9a026c8f3e8b..81a4b4012fdb 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2664,20 +2664,17 @@ struct TExportInfo: public TSimpleRefCount { TString SourcePathName; TPathId SourcePathId; - NKikimrSchemeOp::EPathType SourcePathType; EState State = EState::Waiting; ESubState SubState = ESubState::AllocateTxId; TTxId WaitTxId = InvalidTxId; - TActorId SchemeUploader; TString Issue; TItem() = default; - explicit TItem(const TString& sourcePathName, const TPathId sourcePathId, NKikimrSchemeOp::EPathType sourcePathType) + explicit TItem(const TString& sourcePathName, const TPathId sourcePathId) : SourcePathName(sourcePathName) , SourcePathId(sourcePathId) - , SourcePathType(sourcePathType) { } diff --git a/ydb/core/tx/schemeshard/schemeshard_path.cpp b/ydb/core/tx/schemeshard/schemeshard_path.cpp index 0202ec3d6463..cfed8c7e2215 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path.cpp @@ -922,20 +922,6 @@ const TPath::TChecker& TPath::TChecker::IsBackupCollection(EStatus status) const << " (" << BasicPathInfo(Path.Base()) << ")"); } -const TPath::TChecker& TPath::TChecker::IsSupportedInExports(EStatus status) const { - if (Failed) { - return *this; - } - - if (Path.Base()->IsTable() || Path.Base()->IsView()) { - return *this; - } - - return Fail(status, TStringBuilder() << "path type is not supported in exports" - << " (" << BasicPathInfo(Path.Base()) << ")" - ); -} - const TPath::TChecker& TPath::TChecker::PathShardsLimit(ui64 delta, EStatus status) const { if (Failed) { return *this; diff --git a/ydb/core/tx/schemeshard/schemeshard_path.h b/ydb/core/tx/schemeshard/schemeshard_path.h index 4e017c51b05a..81e3ee13d09b 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.h +++ b/ydb/core/tx/schemeshard/schemeshard_path.h @@ -104,7 +104,6 @@ class TPath { const TChecker& FailOnRestrictedCreateInTempZone(bool allowCreateInTemporaryDir = false, EStatus status = EStatus::StatusPreconditionFailed) const; const TChecker& IsResourcePool(EStatus status = EStatus::StatusNameConflict) const; const TChecker& IsBackupCollection(EStatus status = EStatus::StatusNameConflict) const; - const TChecker& IsSupportedInExports(EStatus status = EStatus::StatusNameConflict) const; }; public: diff --git a/ydb/core/tx/schemeshard/schemeshard_private.h b/ydb/core/tx/schemeshard/schemeshard_private.h index a85a3c4f6748..2ba95e78b0c8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_private.h +++ b/ydb/core/tx/schemeshard/schemeshard_private.h @@ -15,7 +15,6 @@ namespace TEvPrivate { EvRunConditionalErase, EvIndexBuildBilling, EvImportSchemeReady, - EvExportSchemeUploadResult, EvServerlessStorageBilling, EvCleanDroppedPaths, EvCleanDroppedSubDomains, @@ -96,25 +95,6 @@ namespace TEvPrivate { {} }; - struct TEvExportSchemeUploadResult: public TEventLocal { - ui64 ExportId; - ui32 ItemIdx; - bool Success; - TString Error; - - TEvExportSchemeUploadResult(ui64 id, ui32 itemIdx, bool success, const TString& error) - : ExportId(id) - , ItemIdx(itemIdx) - , Success(success) - , Error(error) - {} - - void SetError(const TString& error) { - Success = false; - Error = error; - } - }; - struct TEvServerlessStorageBilling: public TEventLocal { TEvServerlessStorageBilling() {} diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index bedd67656a9b..aba6993077ef 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -1209,7 +1209,6 @@ struct Schema : NIceDb::Schema { struct BackupTxId : Column<6, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; struct Issue : Column<7, NScheme::NTypeIds::Utf8> {}; struct SourceOwnerPathId : Column<8, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; }; - struct SourcePathType : Column<9, NScheme::NTypeIds::Uint32> { using Type = NKikimrSchemeOp::EPathType; static constexpr Type Default = NKikimrSchemeOp::EPathTypeTable; }; using TKey = TableKey; using TColumns = TableColumns< @@ -1220,8 +1219,7 @@ struct Schema : NIceDb::Schema { State, BackupTxId, Issue, - SourceOwnerPathId, - SourcePathType + SourceOwnerPathId >; }; diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make index 810c5783186f..1f681881caa8 100644 --- a/ydb/core/tx/schemeshard/ya.make +++ b/ydb/core/tx/schemeshard/ya.make @@ -306,7 +306,6 @@ PEERDIR( ydb/library/login ydb/library/login/protos ydb/library/protobuf_printer - ydb/public/lib/ydb_cli/dump/util yql/essentials/minikql yql/essentials/providers/common/proto ydb/services/bg_tasks @@ -318,12 +317,10 @@ YQL_LAST_ABI_VERSION() IF (OS_WINDOWS) SRCS( - schemeshard_export_scheme_uploader_fallback.cpp schemeshard_import_scheme_getter_fallback.cpp ) ELSE() SRCS( - schemeshard_export_scheme_uploader.cpp schemeshard_import_scheme_getter.cpp ) ENDIF() diff --git a/ydb/library/backup/backup.cpp b/ydb/library/backup/backup.cpp index 0f9f58d32f8a..2e054cddf909 100644 --- a/ydb/library/backup/backup.cpp +++ b/ydb/library/backup/backup.cpp @@ -546,6 +546,67 @@ NView::TViewDescription DescribeView(NView::TViewClient& client, const TString& return status.GetViewDescription(); } +struct TViewQuerySplit { + TString ContextRecreation; + TString Select; +}; + +TViewQuerySplit SplitViewQuery(TStringInput query) { + // to do: make the implementation more versatile + TViewQuerySplit split; + + TString line; + while (query.ReadLine(line)) { + (line.StartsWith("--") || line.StartsWith("PRAGMA ") + ? split.ContextRecreation + : split.Select + ) += line; + } + + return split; +} + +void ValidateViewQuery(const TString& query, const TString& dbPath, NYql::TIssues& accumulatedIssues) { + NYql::TIssues subIssues; + if (!NDump::ValidateViewQuery(query, subIssues)) { + NYql::TIssue restorabilityIssue( + TStringBuilder() << "Restorability of the view: " << dbPath.Quote() + << " storing the following query:\n" + << query + << "\ncannot be guaranteed. For more information, please consult the 'ydb tools dump' documentation." + ); + restorabilityIssue.Severity = NYql::TSeverityIds::S_WARNING; + for (const auto& subIssue : subIssues) { + restorabilityIssue.AddSubIssue(MakeIntrusive(subIssue)); + } + accumulatedIssues.AddIssue(std::move(restorabilityIssue)); + } +} + +TString BuildCreateViewQuery(TStringBuf name, const NView::TViewDescription& description, TStringBuf backupRoot) { + auto [contextRecreation, select] = SplitViewQuery(description.GetQueryText()); + + const TString query = std::format( + "-- backup root: \"{}\"\n" + "{}\n" + "CREATE VIEW IF NOT EXISTS `{}` WITH (security_invoker = TRUE) AS\n" + " {};\n", + backupRoot.data(), + contextRecreation.data(), + name.data(), + select.data() + ); + + TString formattedQuery; + TString errors; + Y_ENSURE(NSQLFormat::SqlFormatSimple( + query, + formattedQuery, + errors + ), errors); + return formattedQuery; +} + } /*! @@ -569,20 +630,17 @@ void BackupView(TDriver driver, const TString& dbBackupRoot, const TString& dbPa NView::TViewClient client(driver); auto viewDescription = DescribeView(client, dbPath); + ValidateViewQuery(viewDescription.GetQueryText(), dbPath, issues); + const auto fsPath = fsBackupDir.Child(NDump::NFiles::CreateView().FileName); LOG_D("Write view creation query to " << fsPath.GetPath().Quote()); - const auto creationQuery = NDump::BuildCreateViewQuery( + TFileOutput output(fsPath); + output << BuildCreateViewQuery( TFsPath(dbPathRelativeToBackupRoot).GetName(), - dbPath, - viewDescription.GetQueryText(), - dbBackupRoot, - issues + viewDescription, + dbBackupRoot ); - Y_ENSURE(creationQuery, issues.ToString()); - - TFileOutput output(fsPath); - output << creationQuery; BackupPermissions(driver, dbBackupRoot, dbPathRelativeToBackupRoot, fsBackupDir); } diff --git a/ydb/public/lib/ydb_cli/dump/util/view_utils.cpp b/ydb/public/lib/ydb_cli/dump/util/view_utils.cpp index 4c61632a87b2..ac5d51f62e76 100644 --- a/ydb/public/lib/ydb_cli/dump/util/view_utils.cpp +++ b/ydb/public/lib/ydb_cli/dump/util/view_utils.cpp @@ -11,8 +11,6 @@ #include #include -#include - using namespace NSQLv1Generated; namespace { @@ -105,6 +103,15 @@ void VisitAllFields(const NProtoBuf::Message& msg, TTokenCollector& callback) { } } +bool Format(const TString& query, TString& formattedQuery, NYql::TIssues& issues) { + google::protobuf::Arena arena; + NSQLTranslation::TTranslationSettings settings; + settings.Arena = &arena; + + auto formatter = NSQLFormat::MakeSqlFormatter(settings); + return formatter->Format(query, formattedQuery, issues); +} + struct TTableRefValidator { // returns true if the message is not a table ref and we need to dive deeper to find it @@ -177,6 +184,10 @@ bool SqlToProtoAst(const TString& query, TRule_sql_query& queryProto, NYql::TIss return true; } +} + +namespace NYdb::NDump { + bool ValidateViewQuery(const TString& query, NYql::TIssues& issues) { TRule_sql_query queryProto; if (!SqlToProtoAst(query, queryProto, issues)) { @@ -185,10 +196,6 @@ bool ValidateViewQuery(const TString& query, NYql::TIssues& issues) { return ValidateTableRefs(queryProto, issues); } -} - -namespace NYdb::NDump { - TString RewriteAbsolutePath(TStringBuf path, TStringBuf backupRoot, TStringBuf restoreRoot) { if (backupRoot == restoreRoot) { return TString(path); @@ -224,71 +231,4 @@ bool RewriteTableRefs(TString& query, TStringBuf backupRoot, TStringBuf restoreR return true; } -TViewQuerySplit SplitViewQuery(TStringInput query) { - // to do: make the implementation more versatile - TViewQuerySplit split; - - TString line; - while (query.ReadLine(line)) { - (line.StartsWith("--") || line.StartsWith("PRAGMA ") - ? split.ContextRecreation - : split.Select - ) += line; - } - - return split; -} - -void ValidateViewQuery(const TString& query, const TString& dbPath, NYql::TIssues& issues) { - NYql::TIssues subIssues; - if (!::ValidateViewQuery(query, subIssues)) { - NYql::TIssue restorabilityIssue( - TStringBuilder() << "Restorability of the view: " << dbPath.Quote() - << " storing the following query:\n" - << query - << "\ncannot be guaranteed. For more information, please refer to the 'ydb tools dump' documentation." - ); - restorabilityIssue.Severity = NYql::TSeverityIds::S_WARNING; - for (const auto& subIssue : subIssues) { - restorabilityIssue.AddSubIssue(MakeIntrusive(subIssue)); - } - issues.AddIssue(std::move(restorabilityIssue)); - } -} - -bool Format(const TString& query, TString& formattedQuery, NYql::TIssues& issues) { - google::protobuf::Arena arena; - NSQLTranslation::TTranslationSettings settings; - settings.Arena = &arena; - - auto formatter = NSQLFormat::MakeSqlFormatter(settings); - return formatter->Format(query, formattedQuery, issues); -} - -TString BuildCreateViewQuery( - const TString& name, const TString& dbPath, const TString& viewQuery, const TString& backupRoot, - NYql::TIssues& issues -) { - auto [contextRecreation, select] = SplitViewQuery(viewQuery); - - const TString creationQuery = std::format( - "-- backup root: \"{}\"\n" - "{}\n" - "CREATE VIEW IF NOT EXISTS `{}` WITH (security_invoker = TRUE) AS\n" - " {};\n", - backupRoot.data(), - contextRecreation.data(), - name.data(), - select.data() - ); - - ValidateViewQuery(creationQuery, dbPath, issues); - - TString formattedQuery; - if (!Format(creationQuery, formattedQuery, issues)) { - return ""; - } - return formattedQuery; -} - } diff --git a/ydb/public/lib/ydb_cli/dump/util/view_utils.h b/ydb/public/lib/ydb_cli/dump/util/view_utils.h index 2b87a5022e5b..6e65ca7c78ab 100644 --- a/ydb/public/lib/ydb_cli/dump/util/view_utils.h +++ b/ydb/public/lib/ydb_cli/dump/util/view_utils.h @@ -2,25 +2,10 @@ namespace NYdb::NDump { +bool ValidateViewQuery(const TString& query, NYql::TIssues& issues); + TString RewriteAbsolutePath(TStringBuf path, TStringBuf backupRoot, TStringBuf restoreRoot); bool RewriteTableRefs(TString& scheme, TStringBuf backupRoot, TStringBuf restoreRoot, NYql::TIssues& issues); -struct TViewQuerySplit { - TString ContextRecreation; - TString Select; -}; - -TViewQuerySplit SplitViewQuery(TStringInput query); - -// returns void, because the validation is non-blocking -void ValidateViewQuery(const TString& query, const TString& dbPath, NYql::TIssues& issues); - -bool Format(const TString& query, TString& formattedQuery, NYql::TIssues& issues); - -TString BuildCreateViewQuery( - const TString& name, const TString& dbPath, const TString& viewQuery, const TString& backupRoot, - NYql::TIssues& issues -); - } diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index a0a079627918..a5ff50b60f12 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -3205,11 +3205,6 @@ "ColumnId": 8, "ColumnName": "SourceOwnerPathId", "ColumnType": "Uint64" - }, - { - "ColumnId": 9, - "ColumnName": "SourcePathType", - "ColumnType": "Uint32" } ], "ColumnsDropped": [], @@ -3223,8 +3218,7 @@ 5, 6, 7, - 8, - 9 + 8 ], "RoomID": 0, "Codec": 0,