From 83ced0406731d98e152bcb6010ba5398048dfe8b Mon Sep 17 00:00:00 2001 From: Svenja Meyer Date: Thu, 18 Jan 2024 16:16:10 +0100 Subject: [PATCH] billing: Add option for json format Motivation: See GitHub issue #7421 https://github.com/dCache/dcache/issues/7421 Modification: An option to use json format was added. When it's set to true, BillingMessageSerializer is used to generate JSON output instead of text format. Further, if json format is set to true, there will be no header in the output file. Also some unit tests for json format were added. Result: Billing files can be written in json format. Acked-by: Tigran Patch: https://rb.dcache.org/r/14206/ Target: master Require-book: yes Require-notes: yes --- .../src/main/markdown/config-billing.md | 1 + .../services/billing/cells/BillingCell.java | 20 ++- .../dcache/services/billing/cells/billing.xml | 1 + .../billing/text/BillingJsonTest.java | 115 ++++++++++++++++++ skel/share/defaults/billing.properties | 6 + 5 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 modules/dcache/src/test/java/org/dcache/services/billing/text/BillingJsonTest.java diff --git a/docs/TheBook/src/main/markdown/config-billing.md b/docs/TheBook/src/main/markdown/config-billing.md index 1d41274462c..e1f6c17ab8b 100644 --- a/docs/TheBook/src/main/markdown/config-billing.md +++ b/docs/TheBook/src/main/markdown/config-billing.md @@ -25,6 +25,7 @@ If you installed dCache following the instructions in the Chapter [Installing dC ``` Use the property `billing.text.dir` to set the location of the log files and the property `billing.enable.text` to control whether the plain-text log files are generated. +To write the logs in JSON format instead of plain text, use the property `billing.format.json=true`. By default the log files are located in the directory `/var/lib/dcache/billing`. Under this directory the log files are diff --git a/modules/dcache/src/main/java/org/dcache/services/billing/cells/BillingCell.java b/modules/dcache/src/main/java/org/dcache/services/billing/cells/BillingCell.java index 7795123922a..27ff4183fd1 100644 --- a/modules/dcache/src/main/java/org/dcache/services/billing/cells/BillingCell.java +++ b/modules/dcache/src/main/java/org/dcache/services/billing/cells/BillingCell.java @@ -40,6 +40,7 @@ import java.util.stream.Collectors; import javax.annotation.PostConstruct; import org.dcache.cells.CellStub; +import org.dcache.notification.BillingMessageSerializerVisitor; import org.dcache.services.billing.text.StringTemplateInfoMessageVisitor; import org.dcache.util.Args; import org.dcache.util.Slf4jSTErrorListener; @@ -63,7 +64,7 @@ public final class BillingCell private static final Logger LOGGER = LoggerFactory.getLogger(BillingCell.class); - public static final String FORMAT_PREFIX = "billing.text.format."; + public static final String TEXT_FORMAT_PREFIX = "billing.text.format."; private final SimpleDateFormat _fileNameFormat = new SimpleDateFormat("yyyy.MM.dd"); @@ -87,6 +88,7 @@ public final class BillingCell private CellStub _poolManagerStub; private Path _logsDir; private boolean _enableText; + private boolean _jsonFormat; private boolean _flatTextDir; public BillingCell() { @@ -102,10 +104,10 @@ public void setEnvironment(final Map environment) { }; for (Map.Entry e : environment.entrySet()) { String key = e.getKey(); - if (key.startsWith(FORMAT_PREFIX)) { + if (key.startsWith(TEXT_FORMAT_PREFIX)) { String format = Formats.replaceKeywords(String.valueOf(e.getValue()), replaceable); String clazz = CaseFormat.LOWER_HYPHEN.to(CaseFormat.UPPER_CAMEL, - key.substring(FORMAT_PREFIX.length())); + key.substring(TEXT_FORMAT_PREFIX.length())); _formats.put(clazz, format); } } @@ -195,6 +197,11 @@ public void messageArrived(Object msg) { } private String getFormattedMessage(InfoMessage msg) { + if (_jsonFormat) { + BillingMessageSerializerVisitor visitor = new BillingMessageSerializerVisitor(); + msg.accept(visitor); + return new String(visitor.getData(), StandardCharsets.UTF_8); + } String format = _formats.get(msg.getClass().getSimpleName()); if (!Strings.isNullOrEmpty(format)) { try { @@ -352,6 +359,9 @@ private void log(Path path, String output) { } private String getFormatHeaders() { + if (_jsonFormat) { + return ""; + } return _formats.entrySet().stream() .map(e -> "## " + CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_HYPHEN, e.getKey()) + ' ' + e.getValue() + '\n') @@ -450,4 +460,8 @@ public void setEnableTxt(boolean enableText) { _enableText = enableText; } + public void setJsonFormat(boolean jsonFormat) { + _jsonFormat = jsonFormat; + } + } diff --git a/modules/dcache/src/main/resources/org/dcache/services/billing/cells/billing.xml b/modules/dcache/src/main/resources/org/dcache/services/billing/cells/billing.xml index bb380b0460c..856686ab660 100644 --- a/modules/dcache/src/main/resources/org/dcache/services/billing/cells/billing.xml +++ b/modules/dcache/src/main/resources/org/dcache/services/billing/cells/billing.xml @@ -39,6 +39,7 @@ class="org.dcache.services.billing.cells.BillingCell"> + diff --git a/modules/dcache/src/test/java/org/dcache/services/billing/text/BillingJsonTest.java b/modules/dcache/src/test/java/org/dcache/services/billing/text/BillingJsonTest.java new file mode 100644 index 00000000000..b196034f5d9 --- /dev/null +++ b/modules/dcache/src/test/java/org/dcache/services/billing/text/BillingJsonTest.java @@ -0,0 +1,115 @@ +package org.dcache.services.billing.text; + +import static org.junit.Assert.assertTrue; + +import diskCacheV111.util.PnfsId; +import diskCacheV111.vehicles.DoorRequestInfoMessage; +import diskCacheV111.vehicles.GenericStorageInfo; +import diskCacheV111.vehicles.MoverInfoMessage; +import diskCacheV111.vehicles.PoolHitInfoMessage; +import diskCacheV111.vehicles.RemoveFileInfoMessage; +import diskCacheV111.vehicles.StorageInfoMessage; +import diskCacheV111.vehicles.WarningPnfsFileInfoMessage; +import dmg.cells.nucleus.CellAddressCore; +import java.nio.charset.StandardCharsets; +import javax.security.auth.Subject; +import org.dcache.mock.ProtocolInfoBuilder; +import org.dcache.notification.BillingMessageSerializerVisitor; +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Test; + + +public class BillingJsonTest { + + private static final CellAddressCore cellAdressCore = new CellAddressCore( + "bccs_uib_no_023@nas023_bccs_uib_no_1Domain"); + private static final PnfsId pnfsid = new PnfsId("0000B706DD4045F346F2B90F882B706DA807"); + private static final BillingMessageSerializerVisitor visitor = new BillingMessageSerializerVisitor(); + GenericStorageInfo si = new GenericStorageInfo(); + + @Before + public void setup() { + si.setHsm("osm"); + si.setStorageClass("atlas:default"); + } + + @Test + public void testDoorRequestInfoMessageValidJson() { + DoorRequestInfoMessage msg = new DoorRequestInfoMessage(cellAdressCore); + msg.setSubject(new Subject()); + + msg.accept(visitor); + + assertTrue(isValidJson(new String(visitor.getData(), StandardCharsets.UTF_8))); + } + + @Test + public void testMoverInfoMessageValidJson() { + MoverInfoMessage msg = new MoverInfoMessage(cellAdressCore, pnfsid); + msg.setFileCreated(true); + msg.setFileSize(687926); + msg.setTransaction("remove"); + msg.setSubject(new Subject()); + msg.setStorageInfo(si); + msg.setTransferAttributes(256437, 2784, + ProtocolInfoBuilder.aProtocolInfo().withProtocol("GFtp") + .withIPAddress("109.105.124.147").build()); + + msg.accept(visitor); + + assertTrue(isValidJson(new String(visitor.getData(), StandardCharsets.UTF_8))); + } + + @Test + public void testPoolHitInfoMessageValidJson() { + PoolHitInfoMessage msg = new PoolHitInfoMessage(cellAdressCore, pnfsid); + + msg.accept(visitor); + + assertTrue(isValidJson(new String(visitor.getData(), StandardCharsets.UTF_8))); + } + + @Test + public void testRemoveFileInfoMessageValidJson() { + RemoveFileInfoMessage msg = new RemoveFileInfoMessage(cellAdressCore, pnfsid); + msg.setSubject(new Subject()); + + msg.setStorageInfo(si); + + msg.accept(visitor); + + assertTrue(isValidJson(new String(visitor.getData(), StandardCharsets.UTF_8))); + } + + @Test + public void testStorageInfoMessageValidJson() { + StorageInfoMessage msg = new StorageInfoMessage(cellAdressCore, pnfsid, true); + msg.setStorageInfo(si); + + msg.accept(visitor); + + assertTrue(isValidJson(new String(visitor.getData(), StandardCharsets.UTF_8))); + } + + @Test + public void testWarningPnfsFileInfoMessageValidJson() { + WarningPnfsFileInfoMessage msg = new WarningPnfsFileInfoMessage("TODO", cellAdressCore, + pnfsid, 0, "TODO"); + + msg.accept(visitor); + + assertTrue(isValidJson(new String(visitor.getData(), StandardCharsets.UTF_8))); + } + + private boolean isValidJson(String string) { + try { + new JSONObject(string); + return true; + } catch (JSONException e) { + return false; + } + } + +} diff --git a/skel/share/defaults/billing.properties b/skel/share/defaults/billing.properties index a62d7e23569..9d860a74d40 100644 --- a/skel/share/defaults/billing.properties +++ b/skel/share/defaults/billing.properties @@ -61,6 +61,12 @@ billing.cell.subscribe=${dcache.topic.billing} # (one-of?true|false)billing.enable.text = true +# ---- Use JSON format +# +# Changes the plain text output to JSON format if set to 'true'. +# +(one-of?true|false)billing.format.json = false + # ---- Directory for billing logs # # The directory within which the billing logs are to be written.