From ef614004e2d308899bc22691dfbabcc2e500f620 Mon Sep 17 00:00:00 2001 From: Mehdi Salarkia Date: Wed, 12 Aug 2020 18:32:51 -0700 Subject: [PATCH] PHOENIX-6074 Let PQS Act As An Admin Tool Rest Endpoint --- phoenix-queryserver-it/pom.xml | 12 +-- phoenix-queryserver/pom.xml | 4 + .../queryserver/QueryServerOptions.java | 4 + .../queryserver/QueryServerProperties.java | 3 + .../server/ServerCustomizersFactory.java | 8 ++ .../server/admin/AdminCommand.java | 25 +++++++ .../server/admin/AdminCommandRequest.java | 67 +++++++++++++++++ .../server/admin/AdminServerHandler.java | 73 +++++++++++++++++++ .../customizers/AdminServerCustomizer.java | 39 ++++++++++ .../server/customizers/ContextCustomizer.java | 63 ++++++++++++++++ .../HostedClientJarsServerCustomizer.java | 22 +----- .../server/admin/AdminCommandRequestTest.java | 43 +++++++++++ pom.xml | 16 ++-- 13 files changed, 347 insertions(+), 32 deletions(-) create mode 100644 phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/admin/AdminCommand.java create mode 100644 phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/admin/AdminCommandRequest.java create mode 100644 phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/admin/AdminServerHandler.java create mode 100644 phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/AdminServerCustomizer.java create mode 100644 phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/ContextCustomizer.java create mode 100644 phoenix-queryserver/src/test/java/org/apache/phoenix/queryserver/server/admin/AdminCommandRequestTest.java diff --git a/phoenix-queryserver-it/pom.xml b/phoenix-queryserver-it/pom.xml index 45684e9..bc76c34 100644 --- a/phoenix-queryserver-it/pom.xml +++ b/phoenix-queryserver-it/pom.xml @@ -105,12 +105,12 @@ phoenix-client test - - org.apache.phoenix - phoenix-core - test - tests - + + + + + + org.apache.phoenix phoenix-queryserver-orchestrator diff --git a/phoenix-queryserver/pom.xml b/phoenix-queryserver/pom.xml index 441d01b..9617964 100644 --- a/phoenix-queryserver/pom.xml +++ b/phoenix-queryserver/pom.xml @@ -220,6 +220,10 @@ + + org.apache.phoenix + phoenix-core + org.apache.phoenix phoenix-client diff --git a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerOptions.java b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerOptions.java index 4c7db87..c0a8e92 100644 --- a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerOptions.java +++ b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerOptions.java @@ -65,6 +65,10 @@ public class QueryServerOptions { public static final String DEFAULT_CLIENT_JARS_REPO = ""; public static final String DEFAULT_CLIENT_JARS_CONTEXT = "/maven"; + // Admin rest defaults + public static final boolean DEFAULT_ADMIN_REST_ENABLED = false; + public static final String DEFAULT_ADMIN_REST_CONTEXT = "/admin"; + // Common defaults public static final String DEFAULT_EXTRA_JDBC_ARGUMENTS = ""; diff --git a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerProperties.java b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerProperties.java index 35a6dec..17e9095 100644 --- a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerProperties.java +++ b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/QueryServerProperties.java @@ -89,4 +89,7 @@ public class QueryServerProperties { public static final String CLIENT_JARS_ENABLED_ATTRIB = "phoenix.queryserver.client.jars.enabled"; public static final String CLIENT_JARS_REPO_ATTRIB = "phoenix.queryserver.client.jars.repo"; public static final String CLIENT_JARS_CONTEXT_ATTRIB = "phoenix.queryserver.client.jars.context"; + + public static final String ADMIN_REST_ENABLED = "phoenix.queryserver.admin.rest.enabled"; + public static final String ADMIN_REST_CONTEXT_ATTRIB = "phoenix.queryserver.admin.rest.context"; } diff --git a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/ServerCustomizersFactory.java b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/ServerCustomizersFactory.java index 346d3e4..5926dad 100644 --- a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/ServerCustomizersFactory.java +++ b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/ServerCustomizersFactory.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.phoenix.queryserver.QueryServerOptions; import org.apache.phoenix.queryserver.QueryServerProperties; +import org.apache.phoenix.queryserver.server.customizers.AdminServerCustomizer; import org.apache.phoenix.queryserver.server.customizers.HostedClientJarsServerCustomizer; import org.eclipse.jetty.server.Server; import org.slf4j.Logger; @@ -71,6 +72,13 @@ public List> createServerCustomizers(Configuration conf LOG.warn("Empty value provided for {}, ignoring", QueryServerProperties.CLIENT_JARS_REPO_ATTRIB); } } +// if (conf.getBoolean(QueryServerProperties.ADMIN_REST_ENABLED, QueryServerOptions.DEFAULT_ADMIN_REST_ENABLED)) { + String contextPath = conf.get(QueryServerProperties.ADMIN_REST_CONTEXT_ATTRIB, + QueryServerOptions.DEFAULT_ADMIN_REST_CONTEXT); + LOG.info("Creating admin HTTP endpoint {}", contextPath); + AdminServerCustomizer customizer = new AdminServerCustomizer(contextPath); + customizers.add(customizer); +// } return Collections.unmodifiableList(customizers); } } diff --git a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/admin/AdminCommand.java b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/admin/AdminCommand.java new file mode 100644 index 0000000..2ad1108 --- /dev/null +++ b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/admin/AdminCommand.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.queryserver.server.admin; + +/** + * + */ +public enum AdminCommand { + IndexTool, IndexScrutiny +} diff --git a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/admin/AdminCommandRequest.java b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/admin/AdminCommandRequest.java new file mode 100644 index 0000000..bc798e4 --- /dev/null +++ b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/admin/AdminCommandRequest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.queryserver.server.admin; + +import java.util.Map; + +import org.apache.htrace.fasterxml.jackson.annotation.JsonProperty; + +/** + * + */ +public class AdminCommandRequest { + @JsonProperty(required = true) + private AdminCommand command; + private Map parameters; + + public AdminCommandRequest() { + + } + + public AdminCommandRequest(AdminCommand command, Map parameters) { + this.command = command; + this.parameters = parameters; + } + + public AdminCommand getCommand() { + return command; + } + + public void setCommand(AdminCommand command) { + this.command = command; + } + + public Map getParameters() { + return parameters; + } + + public void setParameters(Map parameters) { + this.parameters = parameters; + } + + public String[] getParametersAsInputArgs() { + String[] args = parameters != null ? new String[parameters.size()] : new String[0]; + if (parameters != null) { + int index = 0; + for (String key : parameters.keySet()) { + args[index++] = "-" + key + " " + parameters.get(key); + } + } + return args; + } +} diff --git a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/admin/AdminServerHandler.java b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/admin/AdminServerHandler.java new file mode 100644 index 0000000..2576bc5 --- /dev/null +++ b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/admin/AdminServerHandler.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.queryserver.server.admin; + + +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper; +import org.apache.phoenix.mapreduce.index.IndexScrutinyTool; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.AbstractHandler; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import org.apache.phoenix.mapreduce.index.IndexTool; +/** + * + */ +public class AdminServerHandler extends AbstractHandler { + @Override + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { + if ("POST".equalsIgnoreCase(baseRequest.getMethod())) { + response.setContentType("application/json"); + try { + AdminCommandRequest adminCommand = parseRequest(baseRequest.getInputStream()); + response.setStatus(HttpServletResponse.SC_OK); + Tool tool = null; + switch (adminCommand.getCommand()) { + case IndexTool: + tool = new IndexTool(); + break; + case IndexScrutiny: + tool = new IndexScrutinyTool(); + break; + default: + throw new IllegalArgumentException("Unsupported tool: " + adminCommand.getCommand()); + } + int runStatus = ToolRunner.run(tool, adminCommand.getParametersAsInputArgs()); + } catch (Exception e) { + + } + response.getWriter().println("

Hello OneHandler

"); + response.flushBuffer(); + } + + } + + private AdminCommandRequest parseRequest(InputStream inputStream) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(inputStream, AdminCommandRequest.class); + } + +} diff --git a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/AdminServerCustomizer.java b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/AdminServerCustomizer.java new file mode 100644 index 0000000..37b9d4c --- /dev/null +++ b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/AdminServerCustomizer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.queryserver.server.customizers; +import org.apache.phoenix.queryserver.server.admin.AdminServerHandler; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.handler.ContextHandler; + +/** + * + */ +public class AdminServerCustomizer extends ContextCustomizer { + + public AdminServerCustomizer(String contextPath) { + super(contextPath); + } + + @Override + protected Handler createHandler(String contextPath) { + ContextHandler ctx = new ContextHandler(contextPath); + ctx.setAllowNullPathInfo(true); + ctx.setHandler(new AdminServerHandler()); + return ctx; + } +} diff --git a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/ContextCustomizer.java b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/ContextCustomizer.java new file mode 100644 index 0000000..8fe591a --- /dev/null +++ b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/ContextCustomizer.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.queryserver.server.customizers; + +import org.apache.calcite.avatica.server.ServerCustomizer; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.ContextHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Arrays; + +/** + * A context based customizer + */ +public abstract class ContextCustomizer implements ServerCustomizer { + private static final Logger LOG = LoggerFactory.getLogger(ContextCustomizer.class); + + private final String contextPath; + + /** + * @param contextPath The HTTP path which the repository will be hosted at + */ + protected ContextCustomizer(String contextPath) { + this.contextPath = contextPath; + } + + @Override + public void customize(Server server) { + Handler[] handlers = server.getHandlers(); + if (handlers.length != 1) { + LOG.warn("Observed handlers on server {}", Arrays.toString(handlers)); + throw new IllegalStateException("Expected to find one handler"); + } + HandlerList list = (HandlerList) handlers[0]; + Handler[] realHandlers = list.getChildHandlers(); + Handler[] newHandlers = new Handler[realHandlers.length + 1]; + newHandlers[0] = createHandler(contextPath); + System.arraycopy(realHandlers, 0, newHandlers, 1, realHandlers.length); + server.setHandler(new HandlerList(newHandlers)); + } + + protected abstract Handler createHandler(String contextPath); +} diff --git a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/HostedClientJarsServerCustomizer.java b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/HostedClientJarsServerCustomizer.java index 8112196..48e603e 100644 --- a/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/HostedClientJarsServerCustomizer.java +++ b/phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/customizers/HostedClientJarsServerCustomizer.java @@ -32,42 +32,28 @@ /** * Hosts a Maven repository from local filesystem over HTTP from within PQS. */ -public class HostedClientJarsServerCustomizer implements ServerCustomizer { +public class HostedClientJarsServerCustomizer extends ContextCustomizer { private static final Logger LOG = LoggerFactory.getLogger(HostedClientJarsServerCustomizer.class); private final File repoRoot; - private final String contextPath; /** * @param localMavenRepoRoot The path to the Phoenix-built maven repository on the local filesystem * @param contextPath The HTTP path which the repository will be hosted at */ public HostedClientJarsServerCustomizer(File localMavenRepoRoot, String contextPath) { + super(contextPath); this.repoRoot = localMavenRepoRoot; - this.contextPath = contextPath; } @Override - public void customize(Server server) { - Handler[] handlers = server.getHandlers(); - if (handlers.length != 1) { - LOG.warn("Observed handlers on server {}", Arrays.toString(handlers)); - throw new IllegalStateException("Expected to find one handler"); - } - HandlerList list = (HandlerList) handlers[0]; - + protected Handler createHandler(String contextPath) { ContextHandler ctx = new ContextHandler(contextPath); ResourceHandler resource = new ResourceHandler(); resource.setDirAllowed(true); resource.setDirectoriesListed(false); resource.setResourceBase(repoRoot.getAbsolutePath()); ctx.setHandler(resource); - - Handler[] realHandlers = list.getChildHandlers(); - - Handler[] newHandlers = new Handler[realHandlers.length + 1]; - newHandlers[0] = ctx; - System.arraycopy(realHandlers, 0, newHandlers, 1, realHandlers.length); - server.setHandler(new HandlerList(newHandlers)); + return ctx; } } diff --git a/phoenix-queryserver/src/test/java/org/apache/phoenix/queryserver/server/admin/AdminCommandRequestTest.java b/phoenix-queryserver/src/test/java/org/apache/phoenix/queryserver/server/admin/AdminCommandRequestTest.java new file mode 100644 index 0000000..8bc6ec2 --- /dev/null +++ b/phoenix-queryserver/src/test/java/org/apache/phoenix/queryserver/server/admin/AdminCommandRequestTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.queryserver.server.admin; + +import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper; + +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * + */ +public class AdminCommandRequestTest { + + @Test + public void testDeserialization() throws Exception { + String json = "{\"command\":\"IndexTool\",\"parameters\":{\"param1\":\"value1\"}}"; + AdminCommandRequest adminRequest = new ObjectMapper().readValue(json, AdminCommandRequest.class); + assertEquals(adminRequest.getCommand(), AdminCommand.IndexTool); + Map parameters = new HashMap<>(); + parameters.put("param1", "value1"); + assertEquals(adminRequest.getParameters(), parameters); + } +} diff --git a/pom.xml b/pom.xml index c57292f..ed3e3c0 100644 --- a/pom.xml +++ b/pom.xml @@ -567,18 +567,18 @@ + + + + + + + org.apache.phoenix phoenix-core ${phoenix.version} - tests - test - - - org.apache.phoenix - phoenix-core - ${phoenix.version} - test + commons-io