From 60b430c0b4a5421c9dfc94e32488efaff621585c Mon Sep 17 00:00:00 2001 From: Sridhar R Manikarnike Date: Thu, 12 Dec 2024 17:13:51 +0000 Subject: [PATCH] [Enhancement] (nereids)implement showStagesCommand in nereids --- .../org/apache/doris/nereids/DorisParser.g4 | 2 +- .../nereids/parser/LogicalPlanBuilder.java | 7 + .../doris/nereids/trees/plans/PlanType.java | 1 + .../plans/commands/ShowStagesCommand.java | 136 ++++++++++++++++++ .../trees/plans/visitor/CommandVisitor.java | 5 + .../test_show_stages_command.groovy | 31 ++++ 6 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowStagesCommand.java create mode 100644 regression-test/suites/nereids_p0/test_show_stages_command.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 274a354b63bc034..851c39b298e0ebf 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -258,6 +258,7 @@ supportedShowStatement ON tableName=multipartIdentifier #showCreateMaterializedView | SHOW (WARNINGS | ERRORS) limitClause? #showWarningErrors | SHOW BACKENDS #showBackends + | SHOW STAGES #showStages | SHOW REPLICA DISTRIBUTION FROM baseTableRef #showReplicaDistribution | SHOW FULL? TRIGGERS ((FROM | IN) database=multipartIdentifier)? wildWhere? #showTriggers | SHOW TABLET DIAGNOSIS tabletId=INTEGER_VALUE #showDiagnoseTablet @@ -310,7 +311,6 @@ lockTable unsupportedShowStatement : SHOW ROW POLICY (FOR (userIdentify | (ROLE role=identifier)))? #showRowPolicy | SHOW STORAGE POLICY (USING (FOR policy=identifierOrText)?)? #showStoragePolicy - | SHOW STAGES #showStages | SHOW STORAGE (VAULT | VAULTS) #showStorageVault | SHOW OPEN TABLES ((FROM | IN) database=multipartIdentifier)? wildWhere? #showOpenTables | SHOW TABLE STATUS ((FROM | IN) database=multipartIdentifier)? wildWhere? #showTableStatus diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 56de50d620eb6d8..469f3caac754140 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -272,6 +272,7 @@ import org.apache.doris.nereids.DorisParser.ShowRolesContext; import org.apache.doris.nereids.DorisParser.ShowSmallFilesContext; import org.apache.doris.nereids.DorisParser.ShowSqlBlockRuleContext; +import org.apache.doris.nereids.DorisParser.ShowStagesContext; import org.apache.doris.nereids.DorisParser.ShowStorageEnginesContext; import org.apache.doris.nereids.DorisParser.ShowTableCreationContext; import org.apache.doris.nereids.DorisParser.ShowTableIdContext; @@ -580,6 +581,7 @@ import org.apache.doris.nereids.trees.plans.commands.ShowRolesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowSmallFilesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowSqlBlockRuleCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowStagesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTableCreationCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand; @@ -4834,6 +4836,11 @@ public LogicalPlan visitShowDynamicPartition(ShowDynamicPartitionContext ctx) { return new ShowDynamicPartitionCommand(dbName); } + @Override + public LogicalPlan visitShowStages(ShowStagesContext ctx) { + return new ShowStagesCommand(); + } + @Override public LogicalPlan visitRecoverDatabase(RecoverDatabaseContext ctx) { String dbName = ctx.name.getText(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 204d4b07111bec8..3aa43468402777f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -233,6 +233,7 @@ public enum PlanType { SHOW_REPOSITORIES_COMMAND, SHOW_ROLE_COMMAND, SHOW_SMALL_FILES_COMMAND, + SHOW_STAGES_COMMAND, SHOW_STORAGE_ENGINES_COMMAND, SHOW_TABLE_ID_COMMAND, SHOW_TRASH_COMMAND, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowStagesCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowStagesCommand.java new file mode 100644 index 000000000000000..9e0903762f6d182 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowStagesCommand.java @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.cloud.datasource.CloudInternalCatalog; +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.qe.StmtExecutor; + +import com.google.gson.GsonBuilder; +import org.apache.commons.lang3.StringUtils; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Command for SHOW STAGES. + */ +public class ShowStagesCommand extends ShowCommand { + + private static final String NAME_COL = "StageName"; + private static final String ID_COL = "StageId"; + private static final String ENDPOINT_COL = "Endpoint"; + private static final String REGION_COL = "Region"; + private static final String BUCKET_COL = "Bucket"; + private static final String PREFIX_COL = "Prefix"; + private static final String AK_COL = "AK"; + private static final String SK_COL = "SK"; + private static final String PROVIDER_COL = "Provider"; + private static final String DEFAULT_PROP_COL = "DefaultProperties"; + private static final String COMMENT = "Comment"; + private static final String CREATE_TIME = "CreateTime"; + private static final String ACCESS_TYPE = "AccessType"; + private static final String ROLE_NAME = "RoleName"; + private static final String ARN = "Arn"; + + private static final ShowResultSetMetaData META_DATA = + ShowResultSetMetaData.builder() + .addColumn(new Column(NAME_COL, ScalarType.createVarchar(64))) + .addColumn(new Column(ID_COL, ScalarType.createVarchar(64))) + .addColumn(new Column(ENDPOINT_COL, ScalarType.createVarchar(128))) + .addColumn(new Column(REGION_COL, ScalarType.createVarchar(64))) + .addColumn(new Column(BUCKET_COL, ScalarType.createVarchar(64))) + .addColumn(new Column(PREFIX_COL, ScalarType.createVarchar(128))) + .addColumn(new Column(AK_COL, ScalarType.createVarchar(128))) + .addColumn(new Column(SK_COL, ScalarType.createVarchar(128))) + .addColumn(new Column(PROVIDER_COL, ScalarType.createVarchar(10))) + .addColumn(new Column(DEFAULT_PROP_COL, ScalarType.createVarchar(512))) + .addColumn(new Column(COMMENT, ScalarType.createVarchar(512))) + .addColumn(new Column(CREATE_TIME, ScalarType.createVarchar(64))) + .addColumn(new Column(ACCESS_TYPE, ScalarType.createVarchar(64))) + .addColumn(new Column(ROLE_NAME, ScalarType.createVarchar(64))) + .addColumn(new Column(ARN, ScalarType.createVarchar(64))) + .build(); + + public ShowStagesCommand() { + super(PlanType.SHOW_STAGES_COMMAND); + } + + @Override + public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { + try { + List stages = ((CloudInternalCatalog) Env.getCurrentInternalCatalog()) + .getStage(Cloud.StagePB.StageType.EXTERNAL, null, null, null); + if (stages == null) { + throw new AnalysisException("get stage err"); + } + List> results = new ArrayList<>(); + for (Cloud.StagePB stage : stages) { + // todo(copy into): check priv + // if (!Env.getCurrentEnv().getAuth() + // .checkCloudPriv(ConnectContext.get().getCurrentUserIdentity(), stage.getName(), + // PrivPredicate.USAGE, ResourceTypeEnum.STAGE)) { + // continue; + // } + List result = new ArrayList<>(); + result.add(stage.getName()); + result.add(stage.getStageId()); + result.add(stage.getObjInfo().getEndpoint()); + result.add(stage.getObjInfo().getRegion()); + result.add(stage.getObjInfo().getBucket()); + result.add(stage.getObjInfo().getPrefix()); + result.add(StringUtils.isEmpty(stage.getObjInfo().getAk()) ? "" : "**********"); + result.add(StringUtils.isEmpty(stage.getObjInfo().getSk()) ? "" : "**********"); + result.add(stage.getObjInfo().getProvider().name()); + Map propertiesMap = new HashMap<>(); + propertiesMap.putAll(stage.getPropertiesMap()); + result.add(new GsonBuilder().disableHtmlEscaping().create().toJson(propertiesMap)); + result.add(stage.getComment()); + result.add(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(stage.getCreateTime()))); + result.add(stage.hasAccessType() ? stage.getAccessType().name() + : (StringUtils.isEmpty(stage.getObjInfo().getSk()) ? "" : "AKSK")); + result.add(stage.getRoleName()); + result.add(stage.getArn()); + results.add(result); + } + return new ShowResultSet(META_DATA, results); + } catch (DdlException e) { + throw new AnalysisException(e.getMessage()); + } + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitShowStagesCommand(this, context); + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index e2ac1c863e2f5a7..c5b839bc8d494de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -117,6 +117,7 @@ import org.apache.doris.nereids.trees.plans.commands.ShowRolesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowSmallFilesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowSqlBlockRuleCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowStagesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowStorageEnginesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTableCreationCommand; import org.apache.doris.nereids.trees.plans.commands.ShowTableIdCommand; @@ -328,6 +329,10 @@ default R visitDropCatalogRecycleBinCommand(DropCatalogRecycleBinCommand dropCat return visitCommand(dropCatalogRecycleBinCommand, context); } + default R visitShowStagesCommand(ShowStagesCommand showStagesCommand, C context) { + return visitCommand(showStagesCommand, context); + } + default R visitUnsupportedCommand(UnsupportedCommand unsupportedCommand, C context) { return visitCommand(unsupportedCommand, context); } diff --git a/regression-test/suites/nereids_p0/test_show_stages_command.groovy b/regression-test/suites/nereids_p0/test_show_stages_command.groovy new file mode 100644 index 000000000000000..276b59a36262c5f --- /dev/null +++ b/regression-test/suites/nereids_p0/test_show_stages_command.groovy @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// distributed under the License is distributed on an +// "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_show_stages_command", "nereids_p0") { + try { + if (!cloudMode) + return; + + // Check if stages can be listed + checkNereidsExecute("SHOW STAGES") + } catch (Exception e) { + logger.error("Test failed: ", e) + throw e + } +} +