SINT: Add tagging of tests with references to (XMPP) specifications

A new annotation is introduced (`SpecificationReference`) that can be used to annotate a SINT test.

These properties are available in the annotation:
- `document`: Iidentifier for a specification document, such as 'RFC 6120' or 'XEP-0485'
- `section`: Identifier for a section (or paragraph), such as '6.2.1'
- `quote`: A quotation of relevant text from the section

The SINT execution framework is modified so that two new configuration options are available:
- `enabledSpecifications`
- `disabledSpecifications`

These operate on the value of the `document` property of the annotation. Their usage is comparable
to that of the pre-existing `enabledTests` and `disabledTest` configuration options.

Execution output now includes the document, section and quote that's on the annotated test, when
the test fails. This allows an end-user to easily correspond a test failure with a particular
specification.

smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/Configuration.java

@@ -111,6 +111,10 @@ public enum DnsResolver {
 
     private final Map<String, Set<String>> disabledTestsMap;
 
+    public final Set<String> enabledSpecifications;
+
+    public final Set<String> disabledSpecifications;
+
     public final String defaultConnectionNickname;
 
     public final Set<String> enabledConnections;
@@ -181,6 +185,8 @@ else if (StringUtils.isNotEmpty(builder.accountOneUsername, builder.accountOnePa
         this.enabledTestsMap = convertTestsToMap(enabledTests);
         this.disabledTests =  CollectionUtil.nullSafeUnmodifiableSet(builder.disabledTests);
         this.disabledTestsMap = convertTestsToMap(disabledTests);
+        this.enabledSpecifications = CollectionUtil.nullSafeUnmodifiableSet(builder.enabledSpecifications);
+        this.disabledSpecifications = CollectionUtil.nullSafeUnmodifiableSet(builder.disabledSpecifications);
         this.defaultConnectionNickname = builder.defaultConnectionNickname;
         this.enabledConnections = builder.enabledConnections;
         this.disabledConnections = builder.disabledConnections;
@@ -252,6 +258,10 @@ public static final class Builder {
 
         private Set<String> disabledTests;
 
+        private Set<String> enabledSpecifications;
+
+        private Set<String> disabledSpecifications;
+
         private String defaultConnectionNickname;
 
         private Set<String> enabledConnections;
@@ -378,6 +388,16 @@ public Builder setDisabledTests(String disabledTestsString) {
             return this;
         }
 
+        public Builder setEnabledSpecifications(String enabledSpecificationsString) {
+            enabledSpecifications = getSpecificationSetFrom(enabledSpecificationsString);
+            return this;
+        }
+
+        public Builder setDisabledSpecifications(String disabledSpecificationsString) {
+            disabledSpecifications = getSpecificationSetFrom(disabledSpecificationsString);
+            return this;
+        }
+
         public Builder setDefaultConnection(String defaultConnectionNickname) {
             this.defaultConnectionNickname = defaultConnectionNickname;
             return this;
@@ -523,6 +543,8 @@ public static Configuration newConfiguration(String[] testPackages)
         builder.setDebugger(properties.getProperty("debugger"));
         builder.setEnabledTests(properties.getProperty("enabledTests"));
         builder.setDisabledTests(properties.getProperty("disabledTests"));
+        builder.setEnabledSpecifications(properties.getProperty("enabledSpecifications"));
+        builder.setDisabledSpecifications(properties.getProperty("disabledSpecifications"));
         builder.setDefaultConnection(properties.getProperty("defaultConnection"));
         builder.setEnabledConnections(properties.getProperty("enabledConnections"));
         builder.setDisabledConnections(properties.getProperty("disabledConnections"));
@@ -587,6 +609,10 @@ private static Set<String> getTestSetFrom(String input) {
         });
     }
 
+    private static Set<String> getSpecificationSetFrom(String input) {
+        return split(input, Configuration::normalizeSpecification);
+    }
+
     private static Map<String, Set<String>> convertTestsToMap(Set<String> tests) {
         Map<String, Set<String>> res = new HashMap<>();
         for (String test : tests) {
@@ -695,4 +721,34 @@ public boolean isMethodDisabled(Method method) {
         return contains(method, disabledTestsMap);
     }
 
+    public boolean isSpecificationEnabled(String specification) {
+        if (enabledSpecifications.isEmpty()) {
+            return true;
+        }
+
+        if (specification == null) {
+            return false;
+        }
+
+        return enabledSpecifications.contains(normalizeSpecification(specification));
+    }
+
+    public boolean isSpecificationDisabled(String specification) {
+        if (disabledSpecifications.isEmpty()) {
+            return false;
+        }
+
+        if (specification == null) {
+            return false;
+        }
+
+        return disabledSpecifications.contains(normalizeSpecification(specification));
+    }
+
+    static String normalizeSpecification(String specification) {
+        if (specification == null || specification.isBlank()) {
+            return null;
+        }
+        return specification.replaceAll("\\s", "").toUpperCase();
+    }
 }

smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/SmackIntegrationTestFramework.java

@@ -44,9 +44,12 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.igniterealtime.smack.inttest.annotations.SpecificationReference;
 import org.jivesoftware.smack.AbstractXMPPConnection;
 import org.jivesoftware.smack.ConnectionConfiguration;
 import org.jivesoftware.smack.ConnectionConfiguration.SecurityMode;
@@ -128,10 +131,19 @@ public static void main(String[] args) throws IOException, KeyManagementExceptio
         final int exitStatus;
         if (failedTests > 0) {
             LOGGER.warning("💀 The following " + failedTests + " tests failed! 💀");
+            final SortedSet<String> bySpecification = new TreeSet<>();
             for (FailedTest failedTest : testRunResult.failedIntegrationTests) {
                 final Throwable cause = failedTest.failureReason;
                 LOGGER.log(Level.SEVERE, failedTest.concreteTest + " failed: " + cause, cause);
+                if (failedTest.concreteTest.method.isAnnotationPresent(SpecificationReference.class)) {
+                    final SpecificationReference spec = failedTest.concreteTest.method.getAnnotation(SpecificationReference.class);
+                    bySpecification.add("- " + spec.document() + " section " + spec.section() + ":\t\"" + spec.quote() + "\" (as tested by '"+failedTest.concreteTest + "')");
+                }
+            }
+            if (!bySpecification.isEmpty()) {
+                LOGGER.log(Level.SEVERE, "The failed tests correspond to the following specifications:" + System.lineSeparator() + String.join(System.lineSeparator(), bySpecification));
             }
+
             exitStatus = 2;
         } else {
             LOGGER.info("All possible Smack Integration Tests completed successfully. \\o/");
@@ -392,6 +404,28 @@ private void runTests(Set<Class<? extends AbstractSmackIntTest>> classes)
                     it.remove();
                     continue;
                 }
+
+                final String specification;
+                if (method.isAnnotationPresent(SpecificationReference.class)) {
+                    final SpecificationReference specificationReferenceAnnotation = method.getAnnotation(SpecificationReference.class);
+                    specification = Configuration.normalizeSpecification(specificationReferenceAnnotation.document());
+                } else {
+                    specification = null;
+                }
+
+                if (!config.isSpecificationEnabled(specification)) {
+                    DisabledTest disabledTest = new DisabledTest(method, "Skipping test method " + methodName + " because it tests a specification ('" + specification + "') that is not enabled");
+                    testRunResult.disabledTests.add(disabledTest);
+                    it.remove();
+                    continue;
+                }
+
+                if (config.isSpecificationDisabled(specification)) {
+                    DisabledTest disabledTest = new DisabledTest(method, "Skipping test method " + methodName + " because it tests a specification ('" + specification + "') that is disabled");
+                    testRunResult.disabledTests.add(disabledTest);
+                    it.remove();
+                    continue;
+                }
             }
 
             if (smackIntegrationTestMethods.isEmpty()) {

smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/annotations/SpecificationReference.java

@@ -0,0 +1,51 @@
+/**
+ *
+ * Copyright 2024 Guus der Kinderen
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.igniterealtime.smack.inttest.annotations;
+
+import java.lang.annotation.*;
+
+/**
+ * Reference to a specific part of a specification.
+ *
+ * @author Guus der Kinderen, [email protected]
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface SpecificationReference
+{
+    /**
+     * Unique identifier for a specification document, such as 'RFC 6120' or 'XEP-0485'
+     *
+     * @return a document identifier
+     */
+    String document();
+
+    /**
+     * Unique identifier for a section (or paragraph) of the document referenced by {@link #document()}, such as '6.2.1'.
+     *
+     * @return a document section identifier
+     */
+    String section();
+
+    /**
+     * A quotation of relevant text from the section referenced by {@link #section()}.
+     *
+     * @return human-readable text from the references document and section.
+     */
+    String quote();
+}

smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/package-info.java

@@ -136,6 +136,14 @@
  * <td>List of disabled tests</td>
  * </tr>
  * <tr>
+ * <td>enabledSpecifications</td>
+ * <td>List of specifications for which to enable tests</td>
+ * </tr>
+ * <tr>
+ * <td>disabledSpecifications</td>
+ * <td>List of specificatinos for which to disable tests</td>
+ * </tr>
+ * <tr>
  * <td>defaultConnection</td>
  * <td>Nickname of the default connection</td>
  * </tr>
@@ -187,6 +195,20 @@
  * <p>
  * would run all tests defined in the <code>SoftwareInfoIntegrationTest</code> class.
  * </p>
+ * <p>
+ * Use <code>enabledSpecifications</code> to run all tests that assert implementation of functionality that is described
+ * in standards identified by the provided specification-reference.
+ * </p>
+ * <p>
+ * For example:
+ * </p>
+ *
+ * <pre>
+ * $ gradle integrationTest -Dsinttest.enabledSpecifications=XEP-0045
+ * </pre>
+ * <p>
+ * would run all tests that are annotated to verify functionality specified in XEP-0045: "Multi-User Chat".
+ * </p>
  * <h2>Overview of the components</h2>
  * <p>
  * Package <code>org.igniterealtime.smack.inttest</code>