From 1ba5d259e7a9f98838a6c447da5a3b4b74313a4e Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Fri, 17 Jan 2025 19:29:57 -0800 Subject: [PATCH] Allow overriding of module metadata files in integration tests (#120427) --- .../local/AbstractLocalClusterFactory.java | 42 +++++++++++++------ .../local/AbstractLocalSpecBuilder.java | 14 +++++-- .../test/cluster/local/LocalClusterSpec.java | 6 +-- .../test/cluster/local/LocalSpecBuilder.java | 6 +++ .../test/cluster/util/ArchivePatcher.java | 11 ++++- 5 files changed, 59 insertions(+), 20 deletions(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java index 8aa6ea4bc9e26..4a62d5e0145e3 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java @@ -78,6 +78,8 @@ public abstract class AbstractLocalClusterFactory { DefaultPluginInstallSpec installSpec = plugin.getValue(); - // Path the plugin archive with configured overrides if necessary + // Patch the plugin archive with configured overrides if necessary if (installSpec.entitlementsOverride != null || installSpec.propertiesOverride != null) { Path target; try { @@ -673,13 +675,13 @@ private void installPlugins() { ArchivePatcher patcher = new ArchivePatcher(path, target); if (installSpec.entitlementsOverride != null) { patcher.override( - "entitlement-policy.yaml", + ENTITLEMENT_POLICY_YAML, original -> installSpec.entitlementsOverride.apply(original).asStream() ); } if (installSpec.propertiesOverride != null) { patcher.override( - "plugin-descriptor.properties", + PLUGIN_DESCRIPTOR_PROPERTIES, original -> installSpec.propertiesOverride.apply(original).asStream() ); } @@ -729,11 +731,11 @@ private void installModules() { .map(Path::of) .toList(); - spec.getModules().forEach(module -> installModule(module, modulePaths)); + spec.getModules().forEach((module, spec) -> installModule(module, spec, modulePaths)); } } - private void installModule(String moduleName, List modulePaths) { + private void installModule(String moduleName, DefaultPluginInstallSpec installSpec, List modulePaths) { Path destination = distributionDir.resolve("modules").resolve(moduleName); if (Files.notExists(destination)) { Path modulePath = modulePaths.stream().filter(path -> path.endsWith(moduleName)).findFirst().orElseThrow(() -> { @@ -743,7 +745,7 @@ private void installModule(String moduleName, List modulePaths) { ? "project(xpackModule('" + moduleName.substring(7) + "'))" : "project(':modules:" + moduleName + "')"; - throw new RuntimeException( + return new RuntimeException( "Unable to locate module '" + moduleName + "'. Ensure you've added the following to the build script for project '" @@ -758,20 +760,34 @@ private void installModule(String moduleName, List modulePaths) { }); IOUtils.syncWithCopy(modulePath, destination); + try { + if (installSpec.entitlementsOverride != null) { + Path entitlementsFile = modulePath.resolve(ENTITLEMENT_POLICY_YAML); + String original = Files.exists(entitlementsFile) ? Files.readString(entitlementsFile) : ""; + Path target = destination.resolve(ENTITLEMENT_POLICY_YAML); + installSpec.entitlementsOverride.apply(original).writeTo(target); + } + if (installSpec.propertiesOverride != null) { + Path propertiesFiles = modulePath.resolve(PLUGIN_DESCRIPTOR_PROPERTIES); + String original = Files.exists(propertiesFiles) ? Files.readString(propertiesFiles) : ""; + Path target = destination.resolve(PLUGIN_DESCRIPTOR_PROPERTIES); + installSpec.propertiesOverride.apply(original).writeTo(target); + } + } catch (IOException e) { + throw new UncheckedIOException("Error patching module '" + moduleName + "'", e); + } - // Install any extended plugins + // Install any extended modules Properties pluginProperties = new Properties(); try ( - InputStream in = new BufferedInputStream( - new FileInputStream(modulePath.resolve("plugin-descriptor.properties").toFile()) - ) + InputStream in = new BufferedInputStream(new FileInputStream(modulePath.resolve(PLUGIN_DESCRIPTOR_PROPERTIES).toFile())) ) { pluginProperties.load(in); String extendedProperty = pluginProperties.getProperty("extended.plugins"); if (extendedProperty != null) { - String[] extendedPlugins = extendedProperty.split(","); - for (String plugin : extendedPlugins) { - installModule(plugin, modulePaths); + String[] extendedModules = extendedProperty.split(","); + for (String module : extendedModules) { + installModule(module, new DefaultPluginInstallSpec(), modulePaths); } } } catch (IOException e) { diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java index 1ef4bcbfb6120..9617cb633aa9b 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java @@ -34,7 +34,7 @@ public abstract class AbstractLocalSpecBuilder> im private final Map settings = new HashMap<>(); private final List environmentProviders = new ArrayList<>(); private final Map environment = new HashMap<>(); - private final Set modules = new HashSet<>(); + private final Map modules = new HashMap<>(); private final Map plugins = new HashMap<>(); private final Set features = EnumSet.noneOf(FeatureFlag.class); private final List keystoreProviders = new ArrayList<>(); @@ -123,11 +123,19 @@ DistributionType getDistributionType() { @Override public T module(String moduleName) { - this.modules.add(moduleName); + this.modules.put(moduleName, new DefaultPluginInstallSpec()); return cast(this); } - Set getModules() { + @Override + public T module(String moduleName, Consumer config) { + DefaultPluginInstallSpec spec = new DefaultPluginInstallSpec(); + config.accept(spec); + this.modules.put(moduleName, spec); + return cast(this); + } + + Map getModules() { return inherit(() -> parent.getModules(), modules); } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java index b9e9520e77ebb..ed5c0c5d1bbc0 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java @@ -91,7 +91,7 @@ public static class LocalNodeSpec { private final Map settings; private final List environmentProviders; private final Map environment; - private final Set modules; + private final Map modules; private final Map plugins; private final DistributionType distributionType; private final Set features; @@ -113,7 +113,7 @@ public LocalNodeSpec( Map settings, List environmentProviders, Map environment, - Set modules, + Map modules, Map plugins, DistributionType distributionType, Set features, @@ -175,7 +175,7 @@ public DistributionType getDistributionType() { return distributionType; } - public Set getModules() { + public Map getModules() { return modules; } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java index 2b44126fef4ee..20c92dc2b11d7 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java @@ -69,6 +69,12 @@ interface LocalSpecBuilder> { */ T module(String moduleName); + /** + * Ensure module is installed into the distribution when using the {@link DistributionType#INTEG_TEST} distribution. This is ignored + * when the {@link DistributionType#DEFAULT} is being used. + */ + T module(String moduleName, Consumer config); + /** * Ensure plugin is installed into the distribution. */ diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ArchivePatcher.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ArchivePatcher.java index 269d1dd9f516c..0be5295c53cfb 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ArchivePatcher.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ArchivePatcher.java @@ -50,15 +50,24 @@ public Path patch() { ZipEntry entry = entries.nextElement(); output.putNextEntry(entry); if (overrides.containsKey(entry.getName())) { + Function override = overrides.remove(entry.getName()); try (BufferedReader reader = new BufferedReader(new InputStreamReader(input.getInputStream(entry)))) { String content = reader.lines().collect(Collectors.joining(System.lineSeparator())); - overrides.get(entry.getName()).apply(content).transferTo(output); + override.apply(content).transferTo(output); } } else { input.getInputStream(entry).transferTo(output); } output.closeEntry(); } + + for (Map.Entry> override : overrides.entrySet()) { + ZipEntry entry = new ZipEntry(override.getKey()); + output.putNextEntry(entry); + override.getValue().apply("").transferTo(output); + output.closeEntry(); + } + output.flush(); output.finish(); } catch (IOException e) {