Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.17] Allow overriding of module metadata files in integration tests (#120427) #120429

Merged
merged 1 commit into from
Jan 18, 2025
Merged

[8.17] Allow overriding of module metadata files in integration tests (#120427) #120429

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 29 additions & 13 deletions ...sters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ public abstract class AbstractLocalClusterFactory<S extends LocalClusterSpec, H
private static final String TESTS_CLUSTER_FIPS_JAR_PATH_SYSPROP = "tests.cluster.fips.jars.path";
private static final String TESTS_CLUSTER_DEBUG_ENABLED_SYSPROP = "tests.cluster.debug.enabled";
private static final String ENABLE_DEBUG_JVM_ARGS = "-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=";
private static final String ENTITLEMENT_POLICY_YAML = "entitlement-policy.yaml";
private static final String PLUGIN_DESCRIPTOR_PROPERTIES = "plugin-descriptor.properties";

private final DistributionResolver distributionResolver;

Expand Down Expand Up @@ -660,7 +662,7 @@ private void installPlugins() {
.findFirst()
.map(path -> {
DefaultPluginInstallSpec installSpec = plugin.getValue();
// Path the plugin archive with configured overrides if necessary
// Patch the plugin archive with configured overrides if necessary
if (installSpec.entitlementsOverride != null || installSpec.propertiesOverride != null) {
Path target;
try {
Expand All @@ -671,13 +673,13 @@ private void installPlugins() {
ArchivePatcher patcher = new ArchivePatcher(path, target);
if (installSpec.entitlementsOverride != null) {
patcher.override(
"entitlement-policy.yaml",
ENTITLEMENT_POLICY_YAML,
original -> installSpec.entitlementsOverride.apply(original).asStream()
);
}
if (installSpec.propertiesOverride != null) {
patcher.override(
"plugin-descriptor.properties",
PLUGIN_DESCRIPTOR_PROPERTIES,
original -> installSpec.propertiesOverride.apply(original).asStream()
);
}
Expand Down Expand Up @@ -727,11 +729,11 @@ private void installModules() {
.map(Path::of)
.toList();

spec.getModules().forEach(module -> installModule(module, modulePaths));
spec.getModules().forEach((module, spec) -> installModule(module, spec, modulePaths));
}
}

private void installModule(String moduleName, List<Path> modulePaths) {
private void installModule(String moduleName, DefaultPluginInstallSpec installSpec, List<Path> modulePaths) {
Path destination = distributionDir.resolve("modules").resolve(moduleName);
if (Files.notExists(destination)) {
Path modulePath = modulePaths.stream().filter(path -> path.endsWith(moduleName)).findFirst().orElseThrow(() -> {
Expand All @@ -741,7 +743,7 @@ private void installModule(String moduleName, List<Path> modulePaths) {
? "project(xpackModule('" + moduleName.substring(7) + "'))"
: "project(':modules:" + moduleName + "')";

throw new RuntimeException(
return new RuntimeException(
"Unable to locate module '"
+ moduleName
+ "'. Ensure you've added the following to the build script for project '"
Expand All @@ -756,20 +758,34 @@ private void installModule(String moduleName, List<Path> modulePaths) {
});

IOUtils.syncWithCopy(modulePath, destination);
try {
if (installSpec.entitlementsOverride != null) {
Path entitlementsFile = modulePath.resolve(ENTITLEMENT_POLICY_YAML);
String original = Files.exists(entitlementsFile) ? Files.readString(entitlementsFile) : "";
Path target = destination.resolve(ENTITLEMENT_POLICY_YAML);
installSpec.entitlementsOverride.apply(original).writeTo(target);
}
if (installSpec.propertiesOverride != null) {
Path propertiesFiles = modulePath.resolve(PLUGIN_DESCRIPTOR_PROPERTIES);
String original = Files.exists(propertiesFiles) ? Files.readString(propertiesFiles) : "";
Path target = destination.resolve(PLUGIN_DESCRIPTOR_PROPERTIES);
installSpec.propertiesOverride.apply(original).writeTo(target);
}
} catch (IOException e) {
throw new UncheckedIOException("Error patching module '" + moduleName + "'", e);
}

// Install any extended plugins
// Install any extended modules
Properties pluginProperties = new Properties();
try (
InputStream in = new BufferedInputStream(
new FileInputStream(modulePath.resolve("plugin-descriptor.properties").toFile())
)
InputStream in = new BufferedInputStream(new FileInputStream(modulePath.resolve(PLUGIN_DESCRIPTOR_PROPERTIES).toFile()))
) {
pluginProperties.load(in);
String extendedProperty = pluginProperties.getProperty("extended.plugins");
if (extendedProperty != null) {
String[] extendedPlugins = extendedProperty.split(",");
for (String plugin : extendedPlugins) {
installModule(plugin, modulePaths);
String[] extendedModules = extendedProperty.split(",");
for (String module : extendedModules) {
installModule(module, new DefaultPluginInstallSpec(), modulePaths);
}
}
} catch (IOException e) {
Expand Down
14 changes: 11 additions & 3 deletions ...clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public abstract class AbstractLocalSpecBuilder<T extends LocalSpecBuilder<?>> im
private final Map<String, String> settings = new HashMap<>();
private final List<EnvironmentProvider> environmentProviders = new ArrayList<>();
private final Map<String, String> environment = new HashMap<>();
private final Set<String> modules = new HashSet<>();
private final Map<String, DefaultPluginInstallSpec> modules = new HashMap<>();
private final Map<String, DefaultPluginInstallSpec> plugins = new HashMap<>();
private final Set<FeatureFlag> features = EnumSet.noneOf(FeatureFlag.class);
private final List<SettingsProvider> keystoreProviders = new ArrayList<>();
Expand Down Expand Up @@ -123,11 +123,19 @@ DistributionType getDistributionType() {

@Override
public T module(String moduleName) {
this.modules.add(moduleName);
this.modules.put(moduleName, new DefaultPluginInstallSpec());
return cast(this);
}

Set<String> getModules() {
@Override
public T module(String moduleName, Consumer<? super PluginInstallSpec> config) {
DefaultPluginInstallSpec spec = new DefaultPluginInstallSpec();
config.accept(spec);
this.modules.put(moduleName, spec);
return cast(this);
}

Map<String, DefaultPluginInstallSpec> getModules() {
return inherit(() -> parent.getModules(), modules);
}

Expand Down
6 changes: 3 additions & 3 deletions test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ public static class LocalNodeSpec {
private final Map<String, String> settings;
private final List<EnvironmentProvider> environmentProviders;
private final Map<String, String> environment;
private final Set<String> modules;
private final Map<String, DefaultPluginInstallSpec> modules;
private final Map<String, DefaultPluginInstallSpec> plugins;
private final DistributionType distributionType;
private final Set<FeatureFlag> features;
Expand All @@ -113,7 +113,7 @@ public LocalNodeSpec(
Map<String, String> settings,
List<EnvironmentProvider> environmentProviders,
Map<String, String> environment,
Set<String> modules,
Map<String, DefaultPluginInstallSpec> modules,
Map<String, DefaultPluginInstallSpec> plugins,
DistributionType distributionType,
Set<FeatureFlag> features,
Expand Down Expand Up @@ -175,7 +175,7 @@ public DistributionType getDistributionType() {
return distributionType;
}

public Set<String> getModules() {
public Map<String, DefaultPluginInstallSpec> getModules() {
return modules;
}

Expand Down
6 changes: 6 additions & 0 deletions test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@ interface LocalSpecBuilder<T extends LocalSpecBuilder<?>> {
*/
T module(String moduleName);

/**
* Ensure module is installed into the distribution when using the {@link DistributionType#INTEG_TEST} distribution. This is ignored
* when the {@link DistributionType#DEFAULT} is being used.
*/
T module(String moduleName, Consumer<? super PluginInstallSpec> config);

/**
* Ensure plugin is installed into the distribution.
*/
Expand Down
11 changes: 10 additions & 1 deletion test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ArchivePatcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,24 @@ public Path patch() {
ZipEntry entry = entries.nextElement();
output.putNextEntry(entry);
if (overrides.containsKey(entry.getName())) {
Function<? super String, ? extends InputStream> override = overrides.remove(entry.getName());
try (BufferedReader reader = new BufferedReader(new InputStreamReader(input.getInputStream(entry)))) {
String content = reader.lines().collect(Collectors.joining(System.lineSeparator()));
overrides.get(entry.getName()).apply(content).transferTo(output);
override.apply(content).transferTo(output);
}
} else {
input.getInputStream(entry).transferTo(output);
}
output.closeEntry();
}

for (Map.Entry<String, Function<? super String, ? extends InputStream>> override : overrides.entrySet()) {
ZipEntry entry = new ZipEntry(override.getKey());
output.putNextEntry(entry);
override.getValue().apply("").transferTo(output);
output.closeEntry();
}

output.flush();
output.finish();
} catch (IOException e) {
Expand Down