Skip to content

Commit

Permalink
Allow overriding of module metadata files in integration tests (elast…
Browse files Browse the repository at this point in the history 
…ic#120427)
  • Loading branch information
@mark-vieira
mark-vieira committed Jan 18, 2025
1 parent b2e8ed7 commit 75afedf
Show file tree
Hide file tree
Showing 5 changed files with 59 additions and 20 deletions.
42 changes: 29 additions & 13 deletions ...sters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ public abstract class AbstractLocalClusterFactory<S extends LocalClusterSpec, H
private static final String TESTS_CLUSTER_FIPS_JAR_PATH_SYSPROP = "tests.cluster.fips.jars.path";
private static final String TESTS_CLUSTER_DEBUG_ENABLED_SYSPROP = "tests.cluster.debug.enabled";
private static final String ENABLE_DEBUG_JVM_ARGS = "-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=";
private static final String ENTITLEMENT_POLICY_YAML = "entitlement-policy.yaml";
private static final String PLUGIN_DESCRIPTOR_PROPERTIES = "plugin-descriptor.properties";

private final DistributionResolver distributionResolver;

Expand Down Expand Up @@ -660,7 +662,7 @@ private void installPlugins() {
.findFirst()
.map(path -> {
DefaultPluginInstallSpec installSpec = plugin.getValue();
// Path the plugin archive with configured overrides if necessary
// Patch the plugin archive with configured overrides if necessary
if (installSpec.entitlementsOverride != null || installSpec.propertiesOverride != null) {
Path target;
try {
Expand All @@ -671,13 +673,13 @@ private void installPlugins() {
ArchivePatcher patcher = new ArchivePatcher(path, target);
if (installSpec.entitlementsOverride != null) {
patcher.override(
"entitlement-policy.yaml",
ENTITLEMENT_POLICY_YAML,
original -> installSpec.entitlementsOverride.apply(original).asStream()
);
}
if (installSpec.propertiesOverride != null) {
patcher.override(
"plugin-descriptor.properties",
PLUGIN_DESCRIPTOR_PROPERTIES,
original -> installSpec.propertiesOverride.apply(original).asStream()
);
}
Expand Down Expand Up @@ -727,11 +729,11 @@ private void installModules() {
.map(Path::of)
.toList();

spec.getModules().forEach(module -> installModule(module, modulePaths));
spec.getModules().forEach((module, spec) -> installModule(module, spec, modulePaths));
}
}

private void installModule(String moduleName, List<Path> modulePaths) {
private void installModule(String moduleName, DefaultPluginInstallSpec installSpec, List<Path> modulePaths) {
Path destination = distributionDir.resolve("modules").resolve(moduleName);
if (Files.notExists(destination)) {
Path modulePath = modulePaths.stream().filter(path -> path.endsWith(moduleName)).findFirst().orElseThrow(() -> {
Expand All @@ -741,7 +743,7 @@ private void installModule(String moduleName, List<Path> modulePaths) {
? "project(xpackModule('" + moduleName.substring(7) + "'))"
: "project(':modules:" + moduleName + "')";

throw new RuntimeException(
return new RuntimeException(
"Unable to locate module '"
+ moduleName
+ "'. Ensure you've added the following to the build script for project '"
Expand All @@ -756,20 +758,34 @@ private void installModule(String moduleName, List<Path> modulePaths) {
});

IOUtils.syncWithCopy(modulePath, destination);
try {
if (installSpec.entitlementsOverride != null) {
Path entitlementsFile = modulePath.resolve(ENTITLEMENT_POLICY_YAML);
String original = Files.exists(entitlementsFile) ? Files.readString(entitlementsFile) : "";
Path target = destination.resolve(ENTITLEMENT_POLICY_YAML);
installSpec.entitlementsOverride.apply(original).writeTo(target);
}
if (installSpec.propertiesOverride != null) {
Path propertiesFiles = modulePath.resolve(PLUGIN_DESCRIPTOR_PROPERTIES);
String original = Files.exists(propertiesFiles) ? Files.readString(propertiesFiles) : "";
Path target = destination.resolve(PLUGIN_DESCRIPTOR_PROPERTIES);
installSpec.propertiesOverride.apply(original).writeTo(target);
}
} catch (IOException e) {
throw new UncheckedIOException("Error patching module '" + moduleName + "'", e);
}

// Install any extended plugins
// Install any extended modules
Properties pluginProperties = new Properties();
try (
InputStream in = new BufferedInputStream(
new FileInputStream(modulePath.resolve("plugin-descriptor.properties").toFile())
)
InputStream in = new BufferedInputStream(new FileInputStream(modulePath.resolve(PLUGIN_DESCRIPTOR_PROPERTIES).toFile()))
) {
pluginProperties.load(in);
String extendedProperty = pluginProperties.getProperty("extended.plugins");
if (extendedProperty != null) {
String[] extendedPlugins = extendedProperty.split(",");
for (String plugin : extendedPlugins) {
installModule(plugin, modulePaths);
String[] extendedModules = extendedProperty.split(",");
for (String module : extendedModules) {
installModule(module, new DefaultPluginInstallSpec(), modulePaths);
}
}
} catch (IOException e) {
Expand Down
14 changes: 11 additions & 3 deletions ...clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public abstract class AbstractLocalSpecBuilder<T extends LocalSpecBuilder<?>> im
private final Map<String, String> settings = new HashMap<>();
private final List<EnvironmentProvider> environmentProviders = new ArrayList<>();
private final Map<String, String> environment = new HashMap<>();
private final Set<String> modules = new HashSet<>();
private final Map<String, DefaultPluginInstallSpec> modules = new HashMap<>();
private final Map<String, DefaultPluginInstallSpec> plugins = new HashMap<>();
private final Set<FeatureFlag> features = EnumSet.noneOf(FeatureFlag.class);
private final List<SettingsProvider> keystoreProviders = new ArrayList<>();
Expand Down Expand Up @@ -123,11 +123,19 @@ DistributionType getDistributionType() {

@Override
public T module(String moduleName) {
this.modules.add(moduleName);
this.modules.put(moduleName, new DefaultPluginInstallSpec());
return cast(this);
}

Set<String> getModules() {
@Override
public T module(String moduleName, Consumer<? super PluginInstallSpec> config) {
DefaultPluginInstallSpec spec = new DefaultPluginInstallSpec();
config.accept(spec);
this.modules.put(moduleName, spec);
return cast(this);
}

Map<String, DefaultPluginInstallSpec> getModules() {
return inherit(() -> parent.getModules(), modules);
}

Expand Down
6 changes: 3 additions & 3 deletions test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ public static class LocalNodeSpec {
private final Map<String, String> settings;
private final List<EnvironmentProvider> environmentProviders;
private final Map<String, String> environment;
private final Set<String> modules;
private final Map<String, DefaultPluginInstallSpec> modules;
private final Map<String, DefaultPluginInstallSpec> plugins;
private final DistributionType distributionType;
private final Set<FeatureFlag> features;
Expand All @@ -113,7 +113,7 @@ public LocalNodeSpec(
Map<String, String> settings,
List<EnvironmentProvider> environmentProviders,
Map<String, String> environment,
Set<String> modules,
Map<String, DefaultPluginInstallSpec> modules,
Map<String, DefaultPluginInstallSpec> plugins,
DistributionType distributionType,
Set<FeatureFlag> features,
Expand Down Expand Up @@ -175,7 +175,7 @@ public DistributionType getDistributionType() {
return distributionType;
}

public Set<String> getModules() {
public Map<String, DefaultPluginInstallSpec> getModules() {
return modules;
}

Expand Down
6 changes: 6 additions & 0 deletions test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@ interface LocalSpecBuilder<T extends LocalSpecBuilder<?>> {
*/
T module(String moduleName);

/**
* Ensure module is installed into the distribution when using the {@link DistributionType#INTEG_TEST} distribution. This is ignored
* when the {@link DistributionType#DEFAULT} is being used.
*/
T module(String moduleName, Consumer<? super PluginInstallSpec> config);

/**
* Ensure plugin is installed into the distribution.
*/
Expand Down
11 changes: 10 additions & 1 deletion test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ArchivePatcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,24 @@ public Path patch() {
ZipEntry entry = entries.nextElement();
output.putNextEntry(entry);
if (overrides.containsKey(entry.getName())) {
Function<? super String, ? extends InputStream> override = overrides.remove(entry.getName());
try (BufferedReader reader = new BufferedReader(new InputStreamReader(input.getInputStream(entry)))) {
String content = reader.lines().collect(Collectors.joining(System.lineSeparator()));
overrides.get(entry.getName()).apply(content).transferTo(output);
override.apply(content).transferTo(output);
}
} else {
input.getInputStream(entry).transferTo(output);
}
output.closeEntry();
}

for (Map.Entry<String, Function<? super String, ? extends InputStream>> override : overrides.entrySet()) {
ZipEntry entry = new ZipEntry(override.getKey());
output.putNextEntry(entry);
override.getValue().apply("").transferTo(output);
output.closeEntry();
}

output.flush();
output.finish();
} catch (IOException e) {
Expand Down

0 comments on commit 75afedf

Please sign in to comment.