Merge branch 'feature/lustre-occupancy2' into dev

DAQAggregator_example.properties

@@ -66,6 +66,7 @@ f3.disk.url = http://es-cdaq.cms/sc/php/summarydisks.php
 f3.crashes.url = http://es-cdaq.cms/sc/php/resource_status.php
 f3.cpuload.url = http://cmsdaqfff/prod/sc/php/cpuusage.php
 f3.cpuload.type = 20% htcor(2x-x*x)
+f3.storagemanager.url = http://es-cdaq.cms/sc/php/lustre.php
 
 #
 # Flag indicating if flashlist discovery should be based on static catalog, by default based on dynamic

src/main/java/rcms/utilities/daqaggregator/DAQAggregator.java

@@ -286,12 +286,15 @@ public static Pair<MonitorManager, PersistorManager> initialize(RunMode runMode)
         String crashesUrl = Application.get().getProp(Settings.F3_CRASHES_URL);
         String cpuLoadUrl = Application.get().getProp(Settings.F3_CPU_LOAD_URL);
         String cpuLoadType = Application.get().getProp(Settings.F3_CPU_LOAD_TYPE);
+        String storageManagerUrl = Application.get().getProp(Settings.F3_STORAGE_MANAGER_URL);
         F3DataRetriever f3DataRetriever = null;
 
 
         if (f3Enabled && !"".equals(hltUrl) && !"".equals(diskUrl) && !"".equals(crashesUrl) && !"".equals(cpuLoadUrl) && !"".equals(cpuLoadType)) {
             logger.info("F3 monitoring is enabled and set to following urls: " + hltUrl + ", " + diskUrl);
-            f3DataRetriever = new F3DataRetriever(new Connector(false), hltUrl, diskUrl,crashesUrl, cpuLoadUrl, CpuLoadType.getByKey(cpuLoadType));
+            f3DataRetriever = new F3DataRetriever(new Connector(false), hltUrl, diskUrl,crashesUrl,
+                    cpuLoadUrl, CpuLoadType.getByKey(cpuLoadType),
+                    storageManagerUrl);
         } else if (f3Enabled) {
             throw new DAQException(DAQExceptionCode.MissingProperty, "Specify url for F3 data retrieval. Required: " + Settings.F3_DISK_URL.getKey() + ", " + Settings.F3_HLT_URL.getKey() + ", " + Settings.F3_CPU_LOAD_URL.getKey() + ", " + Settings.F3_CPU_LOAD_TYPE.getKey());
         } else {

src/main/java/rcms/utilities/daqaggregator/Settings.java

@@ -24,6 +24,7 @@ public enum Settings {
 	F3_CRASHES_URL("f3.crashes.url"),
 	F3_CPU_LOAD_URL("f3.cpuload.url"),
 	F3_CPU_LOAD_TYPE("f3.cpuload.type"),
+  F3_STORAGE_MANAGER_URL("f3.storagemanager.url"),
 
 	// settings for monitoring
 	MONITOR_SETUPNAME("monitor.setupName"),

src/main/java/rcms/utilities/daqaggregator/data/DAQ.java

@@ -83,6 +83,9 @@ public class DAQ implements FlashlistUpdatable {
 
 	private Boolean isLhcClockStable;
 
+	/** information about storage manager (e.g. disk occupancy) */
+	private StorageManager storageManager;
+
 	public BUSummary getBuSummary() {
 		return buSummary;
 	}
@@ -407,6 +410,7 @@ public int hashCode() {
 		result = prime * result + runNumber;
 		result = prime * result + sessionId;
 		result = prime * result + ((subSystems == null) ? 0 : subSystems.hashCode());
+		result = prime * result + ((storageManager == null) ? 0 : storageManager.hashCode());
 		return result;
 	}
 
@@ -485,6 +489,13 @@ public boolean equals(Object obj) {
 				return false;
 		} else if (!subSystems.equals(other.subSystems))
 			return false;
+
+		if (storageManager == null) {
+			if (other.storageManager != null)
+				return false;
+		} else if (!storageManager.equals(other.storageManager))
+			return false;
+
 		return true;
 	}
 
@@ -583,6 +594,10 @@ public void setLhcClockStable(Boolean lhcClockStable) {
 		isLhcClockStable = lhcClockStable;
 	}
 
+	public StorageManager getStorageManager() { return storageManager; }
+
+	public void setStorageManager(StorageManager storageManager) { this.storageManager = storageManager; }
+
 	@Override
 	public String toString() {
 		return "DAQ [sessionId=" + sessionId + ", dpsetPath=" + dpsetPath + ", runNumber=" + runNumber + ", runStart="

src/main/java/rcms/utilities/daqaggregator/data/StorageManager.java

@@ -0,0 +1,35 @@
+package rcms.utilities.daqaggregator.data;
+
+import java.util.Objects;
+
+/**
+ * class to keep information about storage manager occupancy (and more in the future).
+ */
+public class StorageManager {
+
+	/** fraction (0..1) of occupied disk space in the storage manager */
+	private Float occupancyFraction;
+
+	/** @return the fraction (0..1) of occupied disk space in the storage manager */
+	public Float getOccupancyFraction() {
+		return occupancyFraction;
+	}
+
+	public void setOccupancyFraction(Float occupancy) {
+		this.occupancyFraction = occupancy;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o) return true;
+		if (o == null || getClass() != o.getClass()) return false;
+		StorageManager that = (StorageManager) o;
+		return Objects.equals(occupancyFraction, that.occupancyFraction);
+	}
+
+	@Override
+	public int hashCode() {
+
+		return Objects.hash(occupancyFraction);
+	}
+}

src/main/java/rcms/utilities/daqaggregator/datasource/F3DataRetriever.java

@@ -18,6 +18,7 @@
 import rcms.utilities.daqaggregator.Application;
 import rcms.utilities.daqaggregator.ProxyManager;
 import rcms.utilities.daqaggregator.data.DAQ;
+import rcms.utilities.daqaggregator.data.StorageManager;
 
 /**
  * Retrieves date from F3
@@ -34,6 +35,9 @@ public class F3DataRetriever {
     private final String crashUrl;
     private final String cpuLoadUrl;
 
+    /** URL for retrieving storage manager occupancy from F3mon */
+    private final String storageManagerUrl;
+
     /** currently known CPU load types returned by F3mon (the strings
         corresponds to what F3mon uses in the returned results) */
     public static enum CpuLoadType {
@@ -68,14 +72,16 @@ public static CpuLoadType getByKey(String key) {
     private final CpuLoadType cpuLoadType;
 
     public F3DataRetriever(Connector connector, String hltUrl, String diskUrl, String crashUrl,
-            String cpuLoadUrl, CpuLoadType cpuLoadType) {
+            String cpuLoadUrl, CpuLoadType cpuLoadType,
+                           String storageManagerUrl) {
         this.mapper = new ObjectMapper();
         this.connector = connector;
         this.hltUrl = hltUrl;
         this.diskUrl = diskUrl;
         this.crashUrl = crashUrl;
         this.cpuLoadUrl = cpuLoadUrl + "?setup=cdaq&intlen=30&int=1";
         this.cpuLoadType = cpuLoadType;
+        this.storageManagerUrl = storageManagerUrl;
     }
 
     /**
@@ -84,7 +90,13 @@ public F3DataRetriever(Connector connector, String hltUrl, String diskUrl, Strin
      * @param args
      */
     public static void main(String[] args) {
-        F3DataRetriever f3dr = new F3DataRetriever(new Connector(false), "http://es-cdaq.cms/sc/php/stream_summary_last.php", "http://es-cdaq.cms/sc/php/summarydisks.php", "http://es-cdaq.cms/sc/php/resource_status.php", "http://cmsdaqfff/prod/sc/php/cpuusage.php", CpuLoadType.HTCORR_QUADRATIC);
+        F3DataRetriever f3dr = new F3DataRetriever(new Connector(false),
+                "http://es-cdaq.cms/sc/php/stream_summary_last.php",
+                "http://es-cdaq.cms/sc/php/summarydisks.php",
+                "http://es-cdaq.cms/sc/php/resource_status.php",
+                "http://cmsdaqfff/prod/sc/php/cpuusage.php",
+                CpuLoadType.HTCORR_QUADRATIC,
+                "http://es-cdaq.cms/sc/php/lustre.php");
         try {
             Application.initialize("DAQAggregator.properties");
             ProxyManager.get().startProxy();
@@ -98,6 +110,8 @@ public static void main(String[] args) {
 
             Float cpuLoad = f3dr.getCpuLoad();
             logger.info("cpu load: " + cpuLoad);
+
+            logger.info(String.format("storage manager occupancy: %.1f%%", f3dr.getStorageManager().getOccupancyFraction() * 100));
         } catch (JsonMappingException e) {
             e.printStackTrace();
         } catch (IOException e) {
@@ -113,14 +127,16 @@ public void dispatch(DAQ daq) {
         boolean hltSuccessful = dispatchHLT(daq);
         boolean crashSuccessful = dispatchCrashes(daq);
         boolean cpuUsageSuccessful = dispatchCpuLoad(daq);
-
+				boolean storageManagerSuccesful = dispatchStorageManager(daq);
+
         long end = System.currentTimeMillis();
 
-        if (diskSuccessful && hltSuccessful && crashSuccessful)
+        if (diskSuccessful && hltSuccessful && crashSuccessful && storageManagerSuccesful)
             logger.info("F3 data successfully retrieved and mapped in: " + (end - start) + "ms");
         else {
-            logger.warn("Problem retrieving F3 data [disk successful,hlt successful,crash successful,cpu usage successful]=[" + diskSuccessful + ","
-                    + hltSuccessful + "," +crashSuccessful+ "," + cpuUsageSuccessful + "]");
+            logger.warn("Problem retrieving F3 data [disk successful,hlt successful,crash successful,cpu usage successful, storage manager successful]=[" + diskSuccessful + ","
+                    + hltSuccessful + "," +crashSuccessful+ "," + cpuUsageSuccessful +
+                    "," + storageManagerSuccesful + "]");
         }
     }
 
@@ -159,6 +175,18 @@ protected boolean dispatchCrashes(DAQ daq) {
             return false;
     }
 
+    protected boolean dispatchStorageManager(DAQ daq) {
+        StorageManager sm = null;
+        try {
+            sm = getStorageManager();
+        } catch (IOException e) {
+            logger.warn("Could not retrieve Storage Manager occupancy from F3, IO exception: ", e);
+        }
+        daq.setStorageManager(sm);
+
+        return sm != null;
+    }
+
     protected Integer getCrashes() {
         Pair<Integer, List<String>> a = null;
         try {
@@ -379,6 +407,41 @@ public Float getCpuLoad() {
         return null;
     }
 
+    public StorageManager getStorageManager() throws IOException {
+
+        Pair<Integer, List<String>> a = connector.retrieveLines(storageManagerUrl);
+
+        List<String> result = a.getRight();
+
+        long count = result.size();
+        if (count == 1) {
+            JsonNode resultJson = mapper.readValue(result.get(0), JsonNode.class);
+
+            logger.debug(resultJson);
+            try {
+
+                // the php script returns the occupancy in percent, we convert it to a fraction
+                double occupancy = resultJson.get("occupancy_perc").asDouble() / 100.;
+
+                StorageManager sm = new StorageManager();
+                sm.setOccupancyFraction((float) occupancy);
+
+                return sm;
+
+            } catch (NoSuchElementException e) {
+
+                logger.warn("Cannot retrieve storage manager occupancy (no such element) from response: " + result.get(0));
+            } catch (NullPointerException e) {
+                logger.warn("Cannot retrieve storage manager occupancy from response: " + result.get(0));
+            }
+        } else {
+            logger.warn("Expected 1 node as a response but was " + count);
+        }
+
+        // problems getting the information
+        return null;
+    }
+
     public class DiskInfo {
         private Double ramdiskOccupancyFraction;
         private Integer ramdiskTotal;

src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverTest.java

@@ -28,7 +28,8 @@ public class F3DataRetrieverTest {
 	public void diskInfoTest() throws IOException {
 		logger.info("Test connection");
 		String fakeResponse = "{\"ramdisk_occ\":0,\"output_occ\":0.012734683851866,\"ramdisk_tot\":16370688,\"output_tot\":278240437}";
-		F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null,null,null, CpuLoadType.HTCORR_QUADRATIC);
+		F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null,null,null,
+						CpuLoadType.HTCORR_QUADRATIC, null);
 		DiskInfo di = f3dataRetriever.getDiskInfo();
 		Assert.assertEquals(new Double(0.012734683851866), di.getOutputOccupancyFraction());
 		Assert.assertEquals(new Integer(278240437), di.getOutputTotal());
@@ -39,31 +40,43 @@ public void diskInfoTest() throws IOException {
 	@Test
 	public void testHlt() throws IOException {
 		String fakeResponse = "{\"219\":{\"ALCALUMIPIXELS\":47.190047190047,\"ALCAPHISYM\":0,\"Calibration\":79.365079365079,\"DQM\":6.8640068640069,\"DQMCalibration\":7.9365079365079,\"DQMEventDisplay\":18.618618618619,\"DQMHistograms\":612.44101244101,\"EcalCalibration\":79.365079365079,\"Error\":0,\"ExpressCosmics\":19.004719004719,\"HLTRates\":612.44101244101,\"L1Rates\":612.44101244101,\"NanoDST\":5.7915057915058,\"Physics\":71.600171600172,\"RPCMON\":27.627627627628}}";
-		F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),"","",null, "", CpuLoadType.HTCORR_QUADRATIC);
+		F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),"","",null, "",
+						CpuLoadType.HTCORR_QUADRATIC, null);
 		Assert.assertEquals(new Double(71.600171600172d), f3dataRetriever.getHLToutputInfo(0).getEventRate(F3DataRetriever.PHYSICS_STREAM_NAME));
 	}
 
 	@Test
 	public void testHltOutputBandwidth() throws IOException {
 		String fakeResponse = "{\"58\":{\"ALCALUMIPIXELS\":127814.84341484,\"ALCAPHISYM\":0,\"Calibration\":2844043.7580438,\"DQM\":64910.424710425,\"DQMCalibration\":290670.87087087,\"DQMEventDisplay\":0,\"DQMHistograms\":0,\"EcalCalibration\":118622.05062205,\"Error\":0,\"ExpressCosmics\":181467.43886744,\"HLTRates\":41108.193908194,\"L1Rates\":254211.66881167,\"NanoDST\":4938.4384384384,\"Physics\":439103.56070356,\"RPCMON\":0}}";
-		F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null,null, null, CpuLoadType.HTCORR_QUADRATIC);
+		F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null,null, null,
+						CpuLoadType.HTCORR_QUADRATIC, null);
 		Assert.assertEquals(new Double(439103.56070356d), f3dataRetriever.getHLToutputInfo(0).getBandwidth(F3DataRetriever.PHYSICS_STREAM_NAME));
 	}
 
 	@Test
 	public void testHltCrashes() throws IOException {
 		String fakeResponse = "{\"last_run\":306155,\"num_BUs_with_last_run\":73,\"quarantinedRes\":0,\"crashedRes\":0,\"crashedOrQuarantinedRes\":0,\"crashes\":123,\"activeRes\":50728,\"activeResOldRuns\":0}";
-		F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null,null, null, CpuLoadType.HTCORR_QUADRATIC);
+		F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null,null, null,
+						CpuLoadType.HTCORR_QUADRATIC, null);
 		Assert.assertEquals(new Integer(123), f3dataRetriever.getCrashes());
 	}
 
 	@Test
 	public void testCpuLoad() throws IOException {
 		String fakeResponse = "{\"fusyscpu2\":[{\"name\":\"avg uncorr\",\"data\":[[1509988950000,0.16751412425305,954],[1509988980000,0.17668018867293,955]]},{\"name\":\"20% htcor\",\"data\":[[1509988950000,0.27917500538583,954],[1509988980000,0.29433205054159,955]]},{\"name\":\"20% htcor(2x-x*x)\",\"data\":[[1509988950000,0.300280062237,954],[1509988980000,0.31591453234005,955]]}]}";
-		F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null,null,null, CpuLoadType.HTCORR_QUADRATIC);
+		F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null,null,null,
+						CpuLoadType.HTCORR_QUADRATIC, null);
 		Assert.assertEquals(new Float(0.31591453234005), f3dataRetriever.getCpuLoad(), 1e-4);
 	}
 
+	@Test
+	public void testStorageManagerOccupancy() throws IOException {
+		String fakeResponse = "{\"occupancy_perc\":13}";
+		F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null,null,null,
+						CpuLoadType.HTCORR_QUADRATIC, null);
+		Assert.assertEquals(new Float(0.13), f3dataRetriever.getStorageManager().getOccupancyFraction(), 1e-4);
+	}
+
 	public class ConnectorFake extends Connector {
 
 		private final String response;