feat: merge 4.2.7 into 4.2.x-rebase

.github/workflows/docs.yml

@@ -0,0 +1,41 @@
+name: Documentation
+
+on:
+  push:
+    branches:
+      - 4.2.x
+    paths:
+      - "docs/manual/**"
+  pull_request:
+    branches:
+      - 4.2.x
+    paths:
+      - "docs/manual/**"
+  workflow_dispatch:
+
+jobs:
+  deploy-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout GeoNetwork
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+      - name: mkdocs install
+        run: pip install --upgrade pip && pip install -r docs/manual/requirements.txt
+      - name: git configuration
+        run: git config user.name 'github-actions[bot]' && git config user.email 'github-actions[bot]@users.noreply.github.com'
+      - name: build docs without publishing them
+        if: ${{ github.event_name == 'pull_request' }}
+        working-directory: docs/manual
+        run: |
+          mike deploy --title "4.2 Stable" --no-redirect --update-aliases 4.2 stable
+      - name: deploy 4.2 docs to gh-pages branch
+        if: ${{ github.event_name != 'pull_request' }}
+        working-directory: docs/manual
+        run: |
+          mike deploy --push --title "4.2 Stable" --no-redirect --update-aliases 4.2 stable

.gitignore

@@ -14,6 +14,8 @@ git*.properties
 */target/
 **/.idea
 **/.settings
+.*/
+!.github
 #GeoNetwork*
 /geonetwork*
 camel-harvesters/wfsfeature-harvester/logs

CONTRIBUTING.md

@@ -11,13 +11,19 @@ Thank you for contributing to GeoNetwork:
 
 * Pull requests must be applied to `main`, before being backported.
 
+* Before merging a pull request, should be defined the following properties:
+
+  - Milestone to include the change.
+  - Add the label `changelog` when  the change is relevant to be added to the release changelog file .
+  - Add the label(s) to the backport to previous branch(es), when the change is a bug fix or if it is a small improvement that may be relevant to the backport.
+
 * Good housekeeping: Anytime you commit, try and clean the code around it to latest style guide. If you improve a function without comments: add comments. If you modify functionality that does not have tests: write a test. If you fix functionality without documentation: add documentation.
 
-* History: Clean commit messages and history: avoid big commits with hundreds of files, break commits commits up into understandable chunks, longer verbose commit messages are encouraged. Beware of reformatting and needless whitespace changes.
+* History: Clean commit messages and history: avoid big commits with hundreds of files, break commits up into understandable chunks, longer verbose commit messages are encouraged. Beware of reformatting and needless whitespace changes.
 
 * Draft: Use pull request *Draft** (or even the text "WIP") to identify work in progress.
 
-* Rebase: No merge commits with current branch: use rebase!
+* Rebase / Squash and merge: No merge commits with current branch, use Rebase or Squash and merge!
 
 * API Change: Please identify any API change or behavior changes in commit messages.
 

cachingxslt/pom.xml

@@ -31,7 +31,7 @@
   <parent>
     <groupId>org.geonetwork-opensource</groupId>
     <artifactId>geonetwork</artifactId>
-    <version>4.2.6-georchestra</version>
+    <version>4.2.7-georchestra</version>
   </parent>
 
 

code_quality/README.md

@@ -1,3 +1,3 @@
 The files in this directory are used for configuring IDEs or are used by the build tool to make certain code quality checks.
 
-For example findbugs-ecludes is used by both the build tool's findbugs implementation as well as IDES for running findbugs.
+For example, findbugs-excludes is used by both the build tool's findbugs implementation as well as IDES for running findbugs.

common/pom.xml

@@ -31,7 +31,7 @@
   <parent>
     <groupId>org.geonetwork-opensource</groupId>
     <artifactId>geonetwork</artifactId>
-    <version>4.2.6-georchestra</version>
+    <version>4.2.7-georchestra</version>
   </parent>
 
 

core/README.md

@@ -1,9 +1,9 @@
-The core module contains the core Geonetwork classes.  For example SearchManager for searching the metadata index, DataManager for saving
+The core module contains the core Geonetwork classes.  For example, EsSearchManager for searching the metadata index, DataManager for saving
 and loading Metadata.
 
-Services, Harvesters, etc... that are are plugins will usually depend on core and will make use of these core classes to implement their
+Services, Harvesters, etc... that are plugins will usually depend on core and will make use of these core classes to implement their
 services.
 
-Geonetwork is wired together via Spring-Dependency-Injection.  The critical classes (DataManager, SchemaManager, SearchManager, etc...) are
+Geonetwork is wired together via Spring-Dependency-Injection.  The critical classes (DataManager, SchemaManager, EsSearchManager, etc...) are
 all singleton beans in the application context and can be either injected into other beans or obtained via the ServiceContext.getBean
-method (or ServiceContext.getApplicationContext()).
+method (or ServiceContext.getApplicationContext()).

core/pom.xml

@@ -27,13 +27,13 @@
   <parent>
     <artifactId>geonetwork</artifactId>
     <groupId>org.geonetwork-opensource</groupId>
-    <version>4.2.6-georchestra</version>
+    <version>4.2.7-georchestra</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
   <artifactId>gn-core</artifactId>
   <name>GeoNetwork core</name>
-  <version>4.2.6-georchestra</version>
+  <version>4.2.7-georchestra</version>
 
   <dependencies>
     <dependency>

core/src/main/java/org/fao/geonet/kernel/AbstractSchematronValidator.java

@@ -47,7 +47,7 @@
 public class AbstractSchematronValidator {
 
     protected void runSchematron(String lang, Path schemaDir, List<MetadataValidation> validations, Element schemaTronXmlOut,
-                               int metadataId, Element md, ApplicableSchematron applicable) {
+                                 int metadataId, Element md, ApplicableSchematron applicable) {
         final ConfigurableApplicationContext applicationContext = ApplicationContextHolder.get();
         ThesaurusManager thesaurusManager = applicationContext.getBean(ThesaurusManager.class);
 
@@ -61,6 +61,9 @@ protected void runSchematron(String lang, Path schemaDir, List<MetadataValidatio
         report.setAttribute("dbident", String.valueOf(schematron.getId()), Edit.NAMESPACE);
         report.setAttribute("required", requirement.toString(), Edit.NAMESPACE);
 
+        MetadataValidationStatus metadataValidationStatus = null;
+        int invalidRules = 0;
+        int firedRules = 0;
         try {
             Map<String, Object> params = new HashMap<String, Object>();
             params.put("lang", lang);
@@ -74,28 +77,33 @@ protected void runSchematron(String lang, Path schemaDir, List<MetadataValidatio
                 // add results to persistent validation information
                 @SuppressWarnings("unchecked")
                 Iterator<Element> i = xmlReport.getDescendants(new ElementFilter("fired-rule", Geonet.Namespaces.SVRL));
-                int firedRules = Iterators.size(i);
+                firedRules = Iterators.size(i);
 
                 i = xmlReport.getDescendants(new ElementFilter("failed-assert", Geonet.Namespaces.SVRL));
-                int invalidRules = Iterators.size(i);
+                invalidRules = Iterators.size(i);
 
-                if (validations != null) {
-                    validations.add(new MetadataValidation().
-                        setId(new MetadataValidationId(metadataId, ruleId)).
-                        setStatus(invalidRules != 0 ? MetadataValidationStatus.INVALID : MetadataValidationStatus.VALID).
-                        setRequired(requirement == SchematronRequirement.REQUIRED).
-                        setNumTests(firedRules).
-                        setNumFailures(invalidRules));
-
-                }
+                metadataValidationStatus = invalidRules != 0 ? MetadataValidationStatus.INVALID : MetadataValidationStatus.VALID;
             }
         } catch (Exception e) {
             Log.error(Geonet.DATA_MANAGER, "WARNING: schematron xslt " + ruleId + " failed", e);
 
             // If an error occurs that prevents to verify schematron rules, add to show in report
             Element errorReport = new Element("schematronVerificationError", Edit.NAMESPACE);
-            errorReport.addContent("Schematron error ocurred, rules could not be verified: " + e.getMessage());
+            errorReport.addContent("Schematron error occurred, rules could not be verified: " + e.getMessage());
             report.addContent(errorReport);
+
+            // As the validation failed due to an exception lets identify the metadata as never validated.
+            metadataValidationStatus = MetadataValidationStatus.NEVER_CALCULATED;
+        } finally {
+            if (metadataValidationStatus != null && validations != null) {
+                validations.add(new MetadataValidation().
+                    setId(new MetadataValidationId(metadataId, ruleId)).
+                    setStatus(metadataValidationStatus).
+                    setRequired(requirement == SchematronRequirement.REQUIRED).
+                    setNumTests(firedRules).
+                    setNumFailures(invalidRules));
+
+            }
         }
 
         // -- append report to main XML report.

core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java

@@ -527,7 +527,11 @@ public void indexMetadata(final String metadataId,
 
                         // TODO: Check if ignore INSPIRE validation?
                         if (!type.equalsIgnoreCase("inspire")) {
-                            if (status == MetadataValidationStatus.INVALID && vi.isRequired()) {
+                            // If never validated and required then set status to never validated.
+                            if (status == MetadataValidationStatus.NEVER_CALCULATED && vi.isRequired()) {
+                                isValid = "-1";
+                            }
+                            if (status == MetadataValidationStatus.INVALID && vi.isRequired() && isValid != "-1") {
                                 isValid = "0";
                             }
                         } else {

core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java

@@ -153,7 +153,7 @@ public class EsSearchManager implements ISearchManager {
             .build();
 
         FIELDLIST_RELATED_SCRIPTED = ImmutableMap.<String, String>builder()
-            // ElasticSearch scripted field to get the first overview url. Scripted fields must return single values.
+            // Elasticsearch scripted field to get the first overview url. Scripted fields must return single values.
             .put("overview", "return params['_source'].overview == null ? [] : params['_source'].overview.stream().map(f -> f.url).findFirst().orElse('');")
             .build();
     }

core/src/main/java/org/fao/geonet/kernel/url/UrlChecker.java

@@ -1,5 +1,5 @@
 //=============================================================================
-//===	Copyright (C) 2001-2019 Food and Agriculture Organization of the
+//===	Copyright (C) 2001-2023 Food and Agriculture Organization of the
 //===	United Nations (FAO-UN), United Nations World Food Programme (WFP)
 //===	and United Nations Environment Programme (UNEP)
 //===
@@ -31,6 +31,7 @@
 import org.fao.geonet.constants.Geonet;
 import org.fao.geonet.domain.LinkStatus;
 import org.fao.geonet.kernel.setting.SettingManager;
+import org.fao.geonet.lib.Lib;
 import org.fao.geonet.lib.NetLib;
 import org.fao.geonet.utils.GeonetHttpRequestFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -149,8 +150,7 @@ public Void apply(@Nullable HttpClientBuilder originalConfig) {
                     Log.info(Geonet.GEONETWORK,"UrlChecker: cannot determine hostname from url: "+url);
                  }
                 //now we have hostname, we can configure proxy
-                NetLib netLib = new NetLib();
-                netLib.setupProxy(settingManager, originalConfig, hostname);
+                Lib.net.setupProxy(settingManager, originalConfig, hostname);
                 return null;
             }
         };

core/src/main/java/org/fao/geonet/lib/NetLib.java

@@ -1,5 +1,5 @@
 //=============================================================================
-//===	Copyright (C) 2001-2022 Food and Agriculture Organization of the
+//===	Copyright (C) 2001-2023 Food and Agriculture Organization of the
 //===	United Nations (FAO-UN), United Nations World Food Programme (WFP)
 //===	and United Nations Environment Programme (UNEP)
 //===
@@ -34,7 +34,6 @@
 import org.fao.geonet.GeonetContext;
 import org.fao.geonet.constants.Geonet;
 import org.fao.geonet.kernel.setting.SettingManager;
-import org.fao.geonet.kernel.setting.Settings;
 import org.fao.geonet.utils.Log;
 import org.fao.geonet.utils.XmlRequest;
 
@@ -51,6 +50,18 @@
 import jeeves.server.context.ServiceContext;
 
 public class NetLib {
+    private ProxyConfiguration proxyConfiguration;
+
+    public ProxyConfiguration getProxyConfiguration() {
+        return proxyConfiguration;
+    }
+
+    public NetLib() {
+        boolean isProxyConfiguredInSystemProperties = StringUtils.isNotBlank(System.getProperty(ProxyConfiguration.HTTP_PROXY_HOST)) ||
+            StringUtils.isNotBlank(System.getProperty(ProxyConfiguration.HTTPS_PROXY_HOST));
+
+        proxyConfiguration = new ProxyConfiguration(isProxyConfiguredInSystemProperties);
+    }
 
     public void setupProxy(ServiceContext context, XmlRequest req) {
         GeonetContext gc = (GeonetContext) context.getHandlerContext(Geonet.CONTEXT_NAME);
@@ -66,12 +77,14 @@ public void setupProxy(ServiceContext context, XmlRequest req) {
      */
 
     public void setupProxy(SettingManager sm, XmlRequest req) {
-        boolean enabled = sm.getValueAsBool(Settings.SYSTEM_PROXY_USE, false);
-        String host = sm.getValue(Settings.SYSTEM_PROXY_HOST);
-        String port = sm.getValue(Settings.SYSTEM_PROXY_PORT);
-        String username = sm.getValue(Settings.SYSTEM_PROXY_USERNAME);
-        String password = sm.getValue(Settings.SYSTEM_PROXY_PASSWORD);
-        String ignoreHostList = sm.getValue(Settings.SYSTEM_PROXY_IGNOREHOSTLIST);
+        proxyConfiguration.refresh(sm);
+
+        boolean enabled = proxyConfiguration.isEnabled();
+        String host = proxyConfiguration.getHost();
+        String port = proxyConfiguration.getPort();
+        String username = proxyConfiguration.getUsername();
+        String password = proxyConfiguration.getPassword();
+        String ignoreHostList = proxyConfiguration.getIgnoreHostList();
 
         if (!enabled) {
             req.setUseProxy(false);
@@ -108,12 +121,14 @@ public CredentialsProvider setupProxy(ServiceContext context, HttpClientBuilder
      * Setup proxy for http client
      */
     public CredentialsProvider setupProxy(SettingManager sm, HttpClientBuilder client, String requestHost) {
-        boolean enabled = sm.getValueAsBool(Settings.SYSTEM_PROXY_USE, false);
-        String host = sm.getValue(Settings.SYSTEM_PROXY_HOST);
-        String port = sm.getValue(Settings.SYSTEM_PROXY_PORT);
-        String username = sm.getValue(Settings.SYSTEM_PROXY_USERNAME);
-        String password = sm.getValue(Settings.SYSTEM_PROXY_PASSWORD);
-        String ignoreHostList = sm.getValue(Settings.SYSTEM_PROXY_IGNOREHOSTLIST);
+        proxyConfiguration.refresh(sm);
+
+        boolean enabled = proxyConfiguration.isEnabled();
+        String host = proxyConfiguration.getHost();
+        String port = proxyConfiguration.getPort();
+        String username = proxyConfiguration.getUsername();
+        String password = proxyConfiguration.getPassword();
+        String ignoreHostList = proxyConfiguration.getIgnoreHostList();
 
         CredentialsProvider provider = new BasicCredentialsProvider();
         if (enabled) {
@@ -153,30 +168,34 @@ public void setupProxy(ServiceContext context) {
      * Setup proxy for http client
      */
     public void setupProxy(SettingManager sm) {
-        boolean useProxy = sm.getValueAsBool(Settings.SYSTEM_PROXY_USE, false);
+        proxyConfiguration.refresh(sm);
 
-        if (useProxy) {
-            String host = sm.getValue(Settings.SYSTEM_PROXY_HOST);
-            String port = sm.getValue(Settings.SYSTEM_PROXY_PORT);
-            String username = sm.getValue(Settings.SYSTEM_PROXY_USERNAME);
-            String ignoreHostList = sm.getValue(Settings.SYSTEM_PROXY_IGNOREHOSTLIST);
+        // If the proxy is configured in the system properties,
+        // ignore the proxy configuration in the system settings.
+        if (proxyConfiguration.isProxyConfiguredInSystemProperties()) return;
+
+        if (proxyConfiguration.isEnabled()) {
+            String host = proxyConfiguration.getHost();
+            String port = proxyConfiguration.getPort();
+            String username = proxyConfiguration.getUsername();
+            String ignoreHostList = proxyConfiguration.getIgnoreHostList();
 
             Properties props = System.getProperties();
-            props.put("http.proxyHost", host);
-            props.put("http.proxyPort", port);
-            props.put("https.proxyHost", host);
-            props.put("https.proxyPort", port);
-            props.put("http.nonProxyHosts", ignoreHostList);
+            props.put(ProxyConfiguration.HTTP_PROXY_HOST, host);
+            props.put(ProxyConfiguration.HTTP_PROXY_PORT, port);
+            props.put(ProxyConfiguration.HTTPS_PROXY_HOST, host);
+            props.put(ProxyConfiguration.HTTPS_PROXY_PORT, port);
+            props.put(ProxyConfiguration.HTTP_NON_PROXY_HOSTS, ignoreHostList);
 
             if (username.trim().length() > 0) {
                 Log.error(Geonet.GEONETWORK, "Proxy credentials cannot be used");
             }
         } else {
-            System.clearProperty("http.proxyHost");
-            System.clearProperty("http.proxyPort");
-            System.clearProperty("https.proxyHost");
-            System.clearProperty("https.proxyPort");
-            System.clearProperty("http.nonProxyHosts");
+            System.clearProperty(ProxyConfiguration.HTTP_PROXY_HOST);
+            System.clearProperty(ProxyConfiguration.HTTP_PROXY_PORT);
+            System.clearProperty(ProxyConfiguration.HTTPS_PROXY_HOST);
+            System.clearProperty(ProxyConfiguration.HTTPS_PROXY_PORT);
+            System.clearProperty(ProxyConfiguration.HTTP_NON_PROXY_HOSTS);
         }
     }
 
@@ -189,12 +208,14 @@ public URLConnection setupProxy(ServiceContext context, URL url) throws IOExcept
         GeonetContext gc = (GeonetContext) context.getHandlerContext(Geonet.CONTEXT_NAME);
         SettingManager sm = gc.getBean(SettingManager.class);
 
-        boolean enabled = sm.getValueAsBool(Settings.SYSTEM_PROXY_USE, false);
-        String host = sm.getValue(Settings.SYSTEM_PROXY_HOST);
-        String port = sm.getValue(Settings.SYSTEM_PROXY_PORT);
-        String username = sm.getValue(Settings.SYSTEM_PROXY_USERNAME);
-        String password = sm.getValue(Settings.SYSTEM_PROXY_PASSWORD);
-        String ignoreHostList = sm.getValue(Settings.SYSTEM_PROXY_IGNOREHOSTLIST);
+        proxyConfiguration.refresh(sm);
+
+        boolean enabled = proxyConfiguration.isEnabled();
+        String host = proxyConfiguration.getHost();
+        String port = proxyConfiguration.getPort();
+        String username = proxyConfiguration.getUsername();
+        String password = proxyConfiguration.getPassword();
+        String ignoreHostList = proxyConfiguration.getIgnoreHostList();
 
         URLConnection conn = null;
         if (enabled) {