diff --git a/src/main/scala/webecho/dependencies/echostore/EchoStoreFileSystem.scala b/src/main/scala/webecho/dependencies/echostore/EchoStoreFileSystem.scala index 392fb77..66dc77e 100644 --- a/src/main/scala/webecho/dependencies/echostore/EchoStoreFileSystem.scala +++ b/src/main/scala/webecho/dependencies/echostore/EchoStoreFileSystem.scala @@ -23,7 +23,7 @@ import org.json4s.jackson.Serialization.write import org.slf4j.LoggerFactory import webecho.ServiceConfig import webecho.model.{EchoInfo, EchoWebSocket, EchoesInfo, OperationOrigin} -import webecho.tools.{JsonImplicits, UniqueIdentifiers} +import webecho.tools.{HashedIndexedFileStorageLive, JsonImplicits, UniqueIdentifiers} import java.io.{File, FileFilter, FilenameFilter} import java.time.Instant @@ -36,8 +36,8 @@ object EchoStoreFileSystem { // TODO not absolutely "thread safe" move to an actor based implementation // But not so bad as everything is based on distinct files... => immutable file content ;) class EchoStoreFileSystem(config: ServiceConfig) extends EchoStore with JsonImplicits { - private val logger = LoggerFactory.getLogger(getClass) - private val storeConfig = config.webEcho.behavior.fileSystemCache + private val logger = LoggerFactory.getLogger(getClass) + private val storeConfig = config.webEcho.behavior.fileSystemCache private val storeBaseDirectory = { val path = new File(storeConfig.path) if (!path.exists()) { @@ -70,13 +70,10 @@ class EchoStoreFileSystem(config: ServiceConfig) extends EchoStore with JsonImpl private def fsEntryFiles(uuid: UUID): Option[Array[File]] = { val entryFilter = new FilenameFilter { - override def accept(dir: File, name: String): Boolean = name.endsWith(".json") + override def accept(dir: File, name: String): Boolean = + name.endsWith(".data") || name.endsWith(".index") } - def encodedFileCreatedTimestamp(file: File): Long = { - file.getName.split("-", 2).head.toLongOption.getOrElse(0L) - } - - Option(fsEntryBaseDirectory(uuid).listFiles(entryFilter)).map(_.sortBy(f => -encodedFileCreatedTimestamp(f))) + Option(fsEntryBaseDirectory(uuid).listFiles(entryFilter)) } private def fsEntryUUIDs(): Iterable[UUID] = { @@ -115,15 +112,18 @@ class EchoStoreFileSystem(config: ServiceConfig) extends EchoStore with JsonImpl } override def echoInfo(uuid: UUID): Option[EchoInfo] = { - fsEntryFiles(uuid).map { files => + val dest = fsEntryBaseDirectory(uuid) + // TODO add caching to avoid systematic allocation + // TODO switch to effect system to take into account the Try + HashedIndexedFileStorageLive(dest.getAbsolutePath).toOption.map { storage => val origin = jsonRead(fsEntryInfo(uuid)).extractOpt[OperationOrigin] EchoInfo( - count = files.length, - lastUpdated = - files - .map(_.lastModified()) - .maxOption - .map(Instant.ofEpochMilli), + count = storage.size().toOption.getOrElse(0), + lastUpdated = storage + .lastUpdated() + .toOption + .flatten + .map(Instant.ofEpochMilli), origin = origin ) } @@ -154,30 +154,35 @@ class EchoStoreFileSystem(config: ServiceConfig) extends EchoStore with JsonImpl } override def echoAdd(uuid: UUID, origin: Option[OperationOrigin]): Unit = { - val dest = fsEntryBaseDirectory(uuid) - dest.mkdir() + val dest = fsEntryBaseDirectory(uuid) + // TODO add caching to avoid systematic allocation + // TODO switch to effect system to take into account the Try + val storage = HashedIndexedFileStorageLive(dest.getAbsolutePath).get jsonWrite(fsEntryInfo(uuid), decompose(origin)) } override def echoGet(uuid: UUID): Option[Iterator[JValue]] = { - fsEntryFiles(uuid).map { files => - files - .to(Iterator) - .map(jsonRead) + val dest = fsEntryBaseDirectory(uuid) + if (!dest.exists()) None else { + // TODO add caching to avoid systematic allocation + // TODO switch to effect system to take into account the Try + HashedIndexedFileStorageLive(dest.getAbsolutePath).toOption + .map { storage => + storage + .list(reverseOrder = true) + .get + .map(entry => parse(entry)) + } } } - private def makeEntryValueJsonFile(uuid: UUID) = { - val baseDir = fsEntryBaseDirectory(uuid) - val ts = System.currentTimeMillis() - val fileUUID = UniqueIdentifiers.randomUUID().toString - new File(baseDir, s"$ts-$fileUUID.json") - } - override def echoAddValue(uuid: UUID, value: JValue): Unit = { - // In fact just a new file with a timestamp encoded in its name... - val jsonFile = makeEntryValueJsonFile(uuid) - jsonWrite(jsonFile, value) + val dest = fsEntryBaseDirectory(uuid) + // TODO add caching to avoid systematic allocation + // TODO switch to effect system to take into account the Try + HashedIndexedFileStorageLive(dest.getAbsolutePath).foreach { storage => + storage.append(write(value)) + } } private def makeWebSocketJsonFile(entryUUID: UUID, uuid: UUID) = { diff --git a/src/main/scala/webecho/tools/HashedIndexedFileStorage.scala b/src/main/scala/webecho/tools/HashedIndexedFileStorage.scala index d78b2ac..24c98af 100644 --- a/src/main/scala/webecho/tools/HashedIndexedFileStorage.scala +++ b/src/main/scala/webecho/tools/HashedIndexedFileStorage.scala @@ -7,5 +7,6 @@ import scala.io.Codec trait HashedIndexedFileStorage { def list(reverseOrder: Boolean = false, fromEpoch: Option[Long] = None): Try[Iterator[String]] def append(data: String): Try[SHA] + def lastUpdated(): Try[Option[Long]] def size(): Try[Long] } diff --git a/src/main/scala/webecho/tools/HashedIndexedFileStorageLive.scala b/src/main/scala/webecho/tools/HashedIndexedFileStorageLive.scala index a28192f..fe4c2dc 100644 --- a/src/main/scala/webecho/tools/HashedIndexedFileStorageLive.scala +++ b/src/main/scala/webecho/tools/HashedIndexedFileStorageLive.scala @@ -148,15 +148,19 @@ private class HashedIndexedFileStorageLive( } yield dataIterator } - private def getCurrentLastEntrySHA(indexFile: RandomAccessFile): Option[Array[Byte]] = { + private def getIndexLastEntry(indexFile: RandomAccessFile): Option[IndexEntry] = { if (indexFile.length() == 0) None else { val offset = indexFile.length() - indexEntrySize val entry = indexReadEntry(indexFile, offset) - entry.toOption.map(_.sha.bytes) + entry.toOption } } + private def getIndexLastEntrySHA(indexFile: RandomAccessFile): Option[Array[Byte]] = { + getIndexLastEntry(indexFile).map(_.sha.bytes) + } + def append(data: String): Try[SHA] = { val bytes = data.getBytes(codec.charSet) if (bytes.length == 0) Failure(IllegalArgumentException("Input string is empty")) @@ -169,7 +173,7 @@ private class HashedIndexedFileStorageLive( dataIndex }.flatMap { dataIndex => Using(new RandomAccessFile(indexFile, "rwd")) { output => - val prevLastSHA = getCurrentLastEntrySHA(output) + val prevLastSHA = getIndexLastEntrySHA(output) val dataSHA = shaEngine.digest(bytes, prevLastSHA) val timestamp = System.currentTimeMillis() output.seek(output.length()) @@ -189,4 +193,9 @@ private class HashedIndexedFileStorageLive( } } + override def lastUpdated(): Try[Option[Timestamp]] = { + Using(new RandomAccessFile(indexFile, "r")) { indexFile => + getIndexLastEntry(indexFile).map(_.timestamp) + } + } } diff --git a/version.sbt b/version.sbt index 526b45c..dc4c4e4 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "1.2.9-SNAPSHOT" +ThisBuild / version := "1.3.0-SNAPSHOT"