forked from apache/inlong
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[INLONG-10173][Sort] SortStandalone support request unified configura…
…tion
- Loading branch information
Showing
22 changed files
with
897 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
151 changes: 151 additions & 0 deletions
151
...main/java/org/apache/inlong/sort/standalone/config/holder/v2/SortClusterConfigHolder.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.inlong.sort.standalone.config.holder.v2; | ||
|
|
||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.apache.commons.lang3.ClassUtils; | ||
| import org.apache.commons.lang3.StringUtils; | ||
| import org.apache.flume.Context; | ||
| import org.apache.inlong.common.pojo.sort.SortConfig; | ||
| import org.apache.inlong.common.pojo.sort.SortTaskConfig; | ||
| import org.apache.inlong.sort.standalone.config.holder.CommonPropertiesHolder; | ||
| import org.apache.inlong.sort.standalone.config.holder.SortClusterConfigType; | ||
| import org.apache.inlong.sort.standalone.config.loader.v2.ClassResourceSortClusterConfigLoader; | ||
| import org.apache.inlong.sort.standalone.config.loader.v2.ManagerSortClusterConfigLoader; | ||
| import org.apache.inlong.sort.standalone.config.loader.v2.SortConfigLoader; | ||
|
|
||
| import java.util.Date; | ||
| import java.util.Timer; | ||
| import java.util.TimerTask; | ||
|
|
||
| import static org.apache.inlong.sort.standalone.utils.Constants.RELOAD_INTERVAL; | ||
|
|
||
| @Slf4j | ||
| public class SortClusterConfigHolder { | ||
|
|
||
| private static SortClusterConfigHolder instance; | ||
|
|
||
| private long reloadInterval; | ||
| private Timer reloadTimer; | ||
| private SortConfigLoader loader; | ||
| private SortConfig config; | ||
|
|
||
| private SortClusterConfigHolder() { | ||
|
|
||
| } | ||
|
|
||
| private static SortClusterConfigHolder get() { | ||
| if (instance != null) { | ||
| return instance; | ||
| } | ||
|
|
||
| synchronized (SortClusterConfigHolder.class) { | ||
| instance = new SortClusterConfigHolder(); | ||
| instance.reloadInterval = CommonPropertiesHolder.getLong(RELOAD_INTERVAL, 60000L); | ||
| String loaderType = CommonPropertiesHolder | ||
| .getString(SortClusterConfigType.KEY_TYPE, SortClusterConfigType.MANAGER.name()); | ||
|
|
||
| if (SortClusterConfigType.FILE.name().equalsIgnoreCase(loaderType)) { | ||
| instance.loader = new ClassResourceSortClusterConfigLoader(); | ||
| } else if (SortClusterConfigType.MANAGER.name().equalsIgnoreCase(loaderType)) { | ||
| instance.loader = new ManagerSortClusterConfigLoader(); | ||
| } else { | ||
| // user-defined | ||
| try { | ||
| Class<?> loaderClass = ClassUtils.getClass(loaderType); | ||
| Object loaderObject = loaderClass.getDeclaredConstructor().newInstance(); | ||
| if (loaderObject instanceof SortConfigLoader) { | ||
| instance.loader = (SortConfigLoader) loaderObject; | ||
| } | ||
| } catch (Throwable t) { | ||
| log.error("fail to init loader,loaderType:{},error:{}", loaderType, t.getMessage()); | ||
| } | ||
| } | ||
| if (instance.loader == null) { | ||
| instance.loader = new ClassResourceSortClusterConfigLoader(); | ||
| } | ||
| try { | ||
| instance.loader.configure(new Context(CommonPropertiesHolder.get())); | ||
| instance.reload(); | ||
| instance.setReloadTimer(); | ||
| } catch (Exception e) { | ||
| log.error("failed to reload instance", e); | ||
| } | ||
| } | ||
| return instance; | ||
|
|
||
| } | ||
|
|
||
| /** | ||
| * setReloadTimer | ||
| */ | ||
| private void setReloadTimer() { | ||
| reloadTimer = new Timer(true); | ||
| TimerTask task = new TimerTask() { | ||
|
|
||
| /** | ||
| * run | ||
| */ | ||
| public void run() { | ||
| reload(); | ||
| } | ||
| }; | ||
| reloadTimer.schedule(task, new Date(System.currentTimeMillis() + reloadInterval), reloadInterval); | ||
| } | ||
|
|
||
| /** | ||
| * reload | ||
| */ | ||
| private void reload() { | ||
| try { | ||
| SortConfig newConfig = this.loader.load(); | ||
| if (newConfig != null) { | ||
| this.config = newConfig; | ||
| } | ||
| } catch (Throwable e) { | ||
| log.error(e.getMessage(), e); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * getClusterConfig | ||
| * | ||
| * @return | ||
| */ | ||
| public static SortConfig getSortConfig() { | ||
| return get().config; | ||
| } | ||
|
|
||
| /** | ||
| * getTaskConfig | ||
| * | ||
| * @param sortTaskName | ||
| * @return | ||
| */ | ||
| public static SortTaskConfig getTaskConfig(String sortTaskName) { | ||
| SortConfig config = get().config; | ||
| if (config != null && config.getTasks() != null) { | ||
| for (SortTaskConfig task : config.getTasks()) { | ||
| if (StringUtils.equals(sortTaskName, task.getSortTaskName())) { | ||
| return task; | ||
| } | ||
| } | ||
| } | ||
| return null; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
73 changes: 73 additions & 0 deletions
73
.../apache/inlong/sort/standalone/config/loader/v2/ClassResourceSortClusterConfigLoader.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.inlong.sort.standalone.config.loader.v2; | ||
|
|
||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import org.apache.commons.io.IOUtils; | ||
| import org.apache.flume.Context; | ||
| import org.apache.inlong.common.pojo.sort.SortConfig; | ||
| import org.apache.inlong.sort.standalone.config.holder.SortClusterConfigType; | ||
| import org.apache.inlong.sort.standalone.utils.InlongLoggerFactory; | ||
| import org.slf4j.Logger; | ||
|
|
||
| import java.nio.charset.Charset; | ||
|
|
||
| /** | ||
| * | ||
| * ClassResourceCommonPropertiesLoader | ||
| */ | ||
| public class ClassResourceSortClusterConfigLoader implements SortConfigLoader { | ||
|
|
||
| public static final Logger LOG = InlongLoggerFactory.getLogger(ClassResourceSortClusterConfigLoader.class); | ||
|
|
||
| private Context context; | ||
| private ObjectMapper objectMapper = new ObjectMapper(); | ||
|
|
||
| /** | ||
| * load | ||
| * | ||
| * @return | ||
| */ | ||
| @Override | ||
| public SortConfig load() { | ||
| String fileName = SortClusterConfigType.DEFAULT_FILE; | ||
| try { | ||
| if (context != null) { | ||
| fileName = context.getString(SortClusterConfigType.KEY_FILE, SortClusterConfigType.DEFAULT_FILE); | ||
| } | ||
| String confString = IOUtils.toString(getClass().getClassLoader().getResource(fileName), | ||
| Charset.defaultCharset()); | ||
| int index = confString.indexOf('{'); | ||
| confString = confString.substring(index); | ||
| return objectMapper.readValue(confString, SortConfig.class); | ||
| } catch (Exception e) { | ||
| LOG.error("fail to load properties, file ={}, and e= {}", fileName, e); | ||
| } | ||
| return SortConfig.builder().build(); | ||
| } | ||
|
|
||
| /** | ||
| * configure | ||
| * | ||
| * @param context | ||
| */ | ||
| @Override | ||
| public void configure(Context context) { | ||
| this.context = context; | ||
| } | ||
| } |
Oops, something went wrong.