Release RapidMiner 9.0.0

.gitignore

@@ -16,3 +16,4 @@ config/pmd/*
 .idea/
 *.iml
 **/*.tutorial
+**/*.template

build.gradle

@@ -19,12 +19,20 @@ repositories {
 
 dependencies {
 
+	// belt project for new data core
+	compile 'com.rapidminer:belt:0.2.0'
+
+	// belt adapter for conversion between old and new core
+	compile ('com.rapidminer:belt-adapter:0.1.0'){
+		exclude group: 'com.rapidminer.studio', module: 'rapidminer-studio-core'
+	}
+
     // OS X adapter to add platform specific UI
     compile 'com.rapidminer.studio:rapidminer-studio-osx-adapter:1.0.2'
 
 	// RapidMiner license framework for license management
-	compile 'com.rapidminer.license:rapidminer-license-api:4.0.2'
-	compile('com.rapidminer.license:rapidminer-license-commons:4.0.2'){
+	compile 'com.rapidminer.license:rapidminer-license-api:4.0.5'
+	compile('com.rapidminer.license:rapidminer-license-commons:4.0.5'){
 		exclude group: 'com.fasterxml.jackson.core', module: 'jackson-core'
 		exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
 	}
@@ -88,12 +96,6 @@ dependencies {
 	// XStreams for generic XML serialization (http://xstream.codehaus.org/)
 	compile 'com.thoughtworks.xstream:xstream:1.4.10'
 
-	// XMLRPC for XMLRPC connections to Bugzilla (http://ws.apache.org/xmlrpc/)
-	compile('org.apache.xmlrpc:xmlrpc-client:3.1.3') { exclude group: 'junit', module: 'junit' }
-
-	// HttpClient used by the Bugzilla XML RPC client (http://hc.apache.org/httpcomponents-client)
-	compile 'commons-httpclient:commons-httpclient:3.1'
-
 	// FileUtils useful for various things
 	compile 'commons-io:commons-io:2.6'
 
@@ -113,7 +115,7 @@ dependencies {
 	compile 'org.jasypt:jasypt:1.9.1:lite'
 
 	// antlr for parsing expressions (http://www.antlr.org/)
-	compile 'org.antlr:antlr4-runtime:4.5'
+	compile 'org.antlr:antlr4-runtime:4.5.1'
 
 	// SLF4J API (http://www.slf4j.org) 
 	compile 'org.slf4j:slf4j-api:1.7.13'
@@ -123,7 +125,7 @@ dependencies {
 	compile 'junit:junit:4.12'
 
 	// JGraphx for automatic operator arrangement (https://github.com/jgraph/jgraphx)
-	compile 'com.rapidminer.external:jgraphx:2.1.0.2'
+	compile 'com.rapidminer.external:jgraphx:3.9.4'
 
 	// JMathPlot for 2D and 3D plots like Box plot, Stick plot, etc. (https://code.google.com/p/jmathplot/)
 	compile 'com.rapidminer.external:jmathplot:1.0.0'
@@ -132,7 +134,7 @@ dependencies {
 	compile 'com.github.tdbear:microba:0.4.4.3'
 
 	// Apache Tika for file MIME type detection (https://tika.apache.org/)
-	compile 'org.apache.tika:tika-core:1.17'
+	compile 'org.apache.tika:tika-core:1.18'
 
 	// The xalan version used by the JRE for XML handling contains a bug which results in XSLT not working with an enabled SecurityManager
 	// Using this version manually to override the JRE fixes the problem. Otherwise all our operator documentation would not work at all 
@@ -143,15 +145,18 @@ dependencies {
 	compile 'com.h2database:h2:1.4.194'
 
 	// up until 7.4.1 this was a transitive dependency coming from license-commons (jackson version 2.4.0)
-	compile 'com.fasterxml.jackson.core:jackson-core:2.6.6'
-	compile 'com.fasterxml.jackson.core:jackson-databind:2.6.6'
+	compile 'com.fasterxml.jackson.core:jackson-core:2.9.5'
+	compile 'com.fasterxml.jackson.core:jackson-databind:2.9.5'
 
 	// Advanced Search functionality via Lucene
 	compile 'org.apache.lucene:lucene-queryparser:7.2.0'
 	// Highlight Lucene search result matches
 	compile 'org.apache.lucene:lucene-highlighter:7.2.0'
 	compile 'org.jsoup:jsoup:1.8.3'
 
+	// access to windows registry
+	compile 'net.java.dev.jna:jna-platform:4.5.1'
+
 	// tests require mockito
 	testCompile 'org.mockito:mockito-core:2.13.0'
 
@@ -162,6 +167,7 @@ task wrapper(type: Wrapper) { gradleVersion = '4.0.1' }
 apply from: 'gradle/wsimport.gradle'
 apply from: 'gradle/props.gradle'
 apply from: 'gradle/tutorial.gradle'
+apply from: 'gradle/template.gradle'
 
 // Jacoco for code coverage information
-apply from: 'jacoco.gradle'
+apply from: 'jacoco.gradle'

gradle.properties

@@ -1,2 +1,2 @@
-version=8.2.0
+version=9.0.0
 group=com.rapidminer.studio

gradle/template.gradle

@@ -0,0 +1,21 @@
+apply plugin: 'java'
+
+def templateSrc = file('src/main/resources/com/rapidminer/resources/template/src').absolutePath
+// Generate .template files task
+task buildTemplates << {
+    new File(templateSrc).eachDir {
+        ant.zip(destfile: templateSrc+'/../'+it.name+'.template', basedir: it.path){
+        }
+    }
+}
+buildTemplates.group = 'build'
+buildTemplates.description = 'Assembles template archives.'
+
+//Run buildTemplates before the jar task
+jar.dependsOn(buildTemplates)
+
+//Exclude template source files from build
+sourceSets.main.resources{
+    exclude 'com/rapidminer/resources/template/src/**'
+    exclude 'com/rapidminer/resources/template/src'
+}

src/main/java/com/rapidminer/Process.java

@@ -1218,6 +1218,11 @@ public final IOContainer run(final IOContainer input, int logVerbosity, final Ma
 						throw new LicenseViolationException(op, licenseViolations);
 					}
 
+					// Check if the given operator is blacklisted
+					if (OperatorService.isOperatorBlacklisted(op.getOperatorDescription().getKey())) {
+						throw new UserError(op, "operator_blacklisted");
+					}
+
 					// as a side effect mark all enabled operators as dirty
 					// so it is clear which ones have already been executed
 					op.makeDirty();
@@ -1712,9 +1717,26 @@ public String getImportMessage() {
 
 	// process location (file/repository)
 
-	/** Returns true iff either a file or a repository location is defined. */
+	/**
+	 * Returns if the process has a valid save location.
+	 *
+	 * @return {@code true} iff either a file location is defined or a repository location is defined AND the repository
+	 * is not read-only; {@code false} otherwise
+	 */
 	public boolean hasSaveDestination() {
-		return processLocation != null;
+		if (processLocation == null) {
+			return false;
+		}
+		if (processLocation instanceof RepositoryProcessLocation) {
+			RepositoryProcessLocation repoProcLoc = (RepositoryProcessLocation) processLocation;
+			try {
+				return !repoProcLoc.getRepositoryLocation().getRepository().isReadOnly();
+			} catch (RepositoryException e) {
+				return false;
+			}
+		} else {
+			return true;
+		}
 	}
 
 	/**
@@ -1850,4 +1872,34 @@ public static void checkIfSavable(final Process process) throws Exception {
 		}
 	}
 
+	/**
+	 * Checks if breakpoints are present in this process.
+	 *
+	 * @return {@code true}  if a breakpoint is present. {@code false}  otherwise
+	 * @author Joao Pedro Pinheiro
+	 * @since 8.2.0
+	 */
+	public boolean hasBreakpoints() {
+		for (Operator op: getAllOperators()) {
+			if (op.hasBreakpoint()) {
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	/**
+	 * Removes all breakpoints from the current process
+	 *
+	 * @author Joao Pedro Pinheiro
+	 * @since 8.2.0
+	 */
+	public void removeAllBreakpoints() {
+		for (Operator op: getAllOperators()) {
+			op.setBreakpoint(BreakpointListener.BREAKPOINT_BEFORE, false);
+			op.setBreakpoint(BreakpointListener.BREAKPOINT_AFTER, false);
+		}
+	}
+
 }

src/main/java/com/rapidminer/RapidMiner.java

@@ -26,6 +26,8 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.security.AccessControlException;
+import java.security.AccessController;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -62,6 +64,8 @@
 import com.rapidminer.parameter.ParameterTypePassword;
 import com.rapidminer.parameter.ParameterTypeString;
 import com.rapidminer.repository.RepositoryManager;
+import com.rapidminer.security.PluginSandboxPolicy;
+import com.rapidminer.settings.Telemetry;
 import com.rapidminer.test.asserter.AsserterFactoryRapidMiner;
 import com.rapidminer.test_utils.RapidAssert;
 import com.rapidminer.tools.FileSystemService;
@@ -129,6 +133,10 @@ public enum ExecutionMode {
 		EMBEDDED_AS_APPLET(false, false, false, false),
 		/** RM is running within Java Web Start. */
 		WEBSTART(false, true, true, true),
+		/** RM is running inside the Job Container **/
+		JOB_CONTAINER(true, false, false, false),
+		/** RM is running inside the Scoring Agent **/
+		SCORING_AGENT(true, false, false, false),
 		/** We are executing unit tests. */
 		TEST(true, false, false, true);
 
@@ -395,9 +403,11 @@ public boolean isLoadingManagedExtensions() {
 	public static final String PROPERTY_RAPIDMINER_GENERAL_NUMBER_OF_PROCESSES = "rapidminer.general.number_of_processes";
 
 	/**
-	 * The name of the property indicating whether beta features should be activated.
+	 * The name of the property indicating whether beta features should be activated. Increase the version number if
+	 * the beta features should be set to disabled again for everyone. The settings.xml and Settings.properties
+	 * have to be adjusted as well.
 	 */
-	public static final String PROPERTY_RAPIDMINER_UPDATE_BETA_FEATURES = "rapidminer.update.beta_features";
+	public static final String PROPERTY_RAPIDMINER_UPDATE_BETA_FEATURES = "rapidminer.update.beta_features.v2";
 
 	/**
 	 * The name of the property indicating whether or not additional permissions should be enabled
@@ -443,6 +453,16 @@ public boolean isLoadingManagedExtensions() {
 	public static final String PROPERTY_RAPIDMINER_SOCKS_VERSION = "rapidminer.proxy.socksProxyVersion";
 	public final static String[] RAPIDMINER_SOCKS_VERSIONS = { "Version 4", "Version 5" };
 
+	/**
+	 * Set this parameter to {@code true} to allow https to http redirects
+	 */
+	public static final String RAPIDMINER_FOLLOW_HTTPS_TO_HTTP = "rapidminer.system.network.follow_https_to_http";
+
+	/**
+	 * Set this parameter to {@code true} to allow http to https redirects
+	 */
+	public static final String RAPIDMINER_FOLLOW_HTTP_TO_HTTPS = "rapidminer.system.network.follow_http_to_https";
+
 	public static final String PROCESS_FILE_EXTENSION = "rmp";
 
 	/**
@@ -460,20 +480,24 @@ public boolean isLoadingManagedExtensions() {
 		Set<String> protectedParameters = new HashSet<>();
 
 		// add Protected Preferences to the List
-		protectedParameters.add(RapidMiner.PROPERTY_RAPIDMINER_UPDATE_ADDITIONAL_PERMISSIONS);
+		for (Telemetry telemetry : Telemetry.values()) {
+			protectedParameters.add(telemetry.getKey());
+		}
 
-		PROTECTED_PARAMETERS = Collections.unmodifiableSet(protectedParameters);
+		protectedParameters.add(Plugin.PROPERTY_PLUGINS_WHITELIST);
+		protectedParameters.add(OperatorService.OPERATOR_BLACKLIST_KEY);
+
+		PROTECTED_PARAMETERS = Collections.synchronizedSet(protectedParameters);
 
 		System.setProperty(PROPERTY_RAPIDMINER_VERSION, RapidMiner.getLongVersion());
-		ParameterService.setParameterValue(PROPERTY_RAPIDMINER_VERSION, RapidMiner.getLongVersion());
 
 		parameterTypesDescription = new HashSet<>();
 
 		// set default language to english
 		String[] default_language = new String[1];
 		default_language[0] = "eng";
 
-		// scan language definitons. skip comments
+		// scan language definitions. skip comments
 		Vector<String> languages = new Vector<>();
 		Scanner scanLanguageDefs = new Scanner(
 				RapidMiner.class.getResourceAsStream("/com/rapidminer/resources/i18n/language_definitions.txt"));
@@ -555,8 +579,12 @@ public boolean isLoadingManagedExtensions() {
 
 		registerParameter(new ParameterTypeInt(WebServiceTools.WEB_SERVICE_TIMEOUT, "", 1, Integer.MAX_VALUE, 20000),
 				"system");
+
+		registerParameter(new ParameterTypeBoolean(RAPIDMINER_FOLLOW_HTTP_TO_HTTPS, "", true), "system");
+		registerParameter(new ParameterTypeBoolean(RAPIDMINER_FOLLOW_HTTPS_TO_HTTP, "", true), "system");
+
 		registerParameter(new ParameterTypeBoolean(RapidMiner.PROPERTY_RAPIDMINER_UPDATE_BETA_FEATURES, "", false));
-		registerParameter(new ParameterTypeBoolean(RapidMiner.PROPERTY_RAPIDMINER_UPDATE_ADDITIONAL_PERMISSIONS, "", false));
+		registerProtectedParameter(new ParameterTypeBoolean(RapidMiner.PROPERTY_RAPIDMINER_UPDATE_ADDITIONAL_PERMISSIONS, "", false));
 
 		// initialize the state of IOObjects
 		ioObjectCache = new IOObjectMap();
@@ -681,6 +709,7 @@ public static void init(final Product product, final LicenseLocation licenseLoca
 		// check if this version is started for the first time
 		performInitialSettings();
 		ParameterService.init();
+		ParameterService.setParameterValue(PROPERTY_RAPIDMINER_VERSION, RapidMiner.getLongVersion());
 
 		// initializing networking tools
 		GlobalAuthenticator.init();
@@ -749,8 +778,6 @@ public static void init(final Product product, final LicenseLocation licenseLoca
 		RapidMiner.splashMessage("init_configurables");
 		ConfigurationManager.getInstance().initialize();
 
-
-
 		// initialize xml serialization
 		RapidMiner.splashMessage("xml_serialization");
 		XMLSerialization.init(Plugin.getMajorClassLoader());
@@ -1022,6 +1049,10 @@ private static void launchComplete() {
 	 * The descriptions will be set by {@link #initSettingsDescriptions()} later.
 	 */
 	public static void registerParameter(ParameterType parameterType) {
+		// if it is protected, don't allow an overwrite
+		if (isParameterProtected(parameterType.getKey())) {
+			return;
+		}
 		ParameterService.registerParameter(parameterType);
 		RapidMiner.parameterTypesDescription.add(parameterType);
 	}
@@ -1033,10 +1064,61 @@ public static void registerParameter(ParameterType parameterType) {
 	 * The descriptions will be set by {@link #initSettingsDescriptions()} later.
 	 */
 	public static void registerParameter(ParameterType parameterType, String group) {
+		// if it is protected, don't allow an overwrite
+		if (isParameterProtected(parameterType.getKey())) {
+			return;
+		}
 		ParameterService.registerParameter(parameterType, group);
 		RapidMiner.parameterTypesDescription.add(parameterType);
 	}
 
+	/**
+	 * Registers a {@link ParameterType} as a protected setting. Only signed extensions can do so
+	 * and protected settings can not be overwritten. Protected settings can only be set by signed extensions.
+	 *
+	 * @param type
+	 * 		the type to be registered as protected
+	 * @see #registerProtectedParameter(ParameterType, String)
+	 * @since 9.0
+	 */
+	public static void registerProtectedParameter(ParameterType type) {
+		registerProtectedParameter(type, null);
+	}
+
+	/**
+	 * Registers a {@link ParameterType} as a protected setting. Only signed extensions can do so
+	 * and protected settings can not be overwritten. Protected settings can only be set by signed extensions.
+	 *
+	 * @param type
+	 * 		the type to be registered as protected
+	 * @param group
+	 * 		the group to be assigned to the parameter, can be {@code null}
+	 * @see #isParameterProtected(String)
+	 * @see #registerParameter(ParameterType)
+	 * @see #registerParameter(ParameterType, String)
+	 * @since 9.0
+	 */
+	public static void registerProtectedParameter(ParameterType type, String group) {
+		// if it is already protected, don't allow an overwrite
+		if (isParameterProtected(type.getKey())) {
+			return;
+		}
+		try {
+			// only signed extensions are allowed to register protected settings
+			if (System.getSecurityManager() != null) {
+				AccessController.checkPermission(new RuntimePermission(PluginSandboxPolicy.RAPIDMINER_INTERNAL_PERMISSION));
+			}
+		} catch (AccessControlException e) {
+			return;
+		}
+		if (group == null) {
+			registerParameter(type);
+		} else {
+			registerParameter(type, group);
+		}
+		PROTECTED_PARAMETERS.add(type.getKey());
+	}
+
 	public static ExecutionMode getExecutionMode() {
 		return executionMode;
 	}