Add metadata to the manifest

Makefile

@@ -121,6 +121,11 @@ testIntegration: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim testIntegration $(NIM_PARAMS) build.nims
 
+# Builds and runs the integration tests
+testRestApi: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim testRestApi $(NIM_PARAMS) build.nims
+
 # Builds and runs all tests (except for Taiko L2 tests)
 testAll: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \

build.nims

@@ -42,6 +42,10 @@ task testIntegration, "Run integration tests":
   buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true"
   test "testIntegration"
 
+task testRestApi, "Run rest api tests":
+  buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true"
+  test "testRestApi"
+
 task build, "build codex binary":
   codexTask()
 

codex/manifest/coders.nim

@@ -10,6 +10,7 @@
 # This module implements serialization and deserialization of Manifest
 
 import pkg/upraises
+import times
 
 push: {.upraises: [].}
 
@@ -59,6 +60,9 @@ proc encode*(manifest: Manifest): ?!seq[byte] =
   #     optional hcodec: MultiCodec = 5   # Multihash codec
   #     optional version: CidVersion = 6; # Cid version
   #     optional ErasureInfo erasure = 7; # erasure coding info
+  #     optional filename: string = 8;    # original filename
+  #     optional mimetype: string = 9;    # original mimetype
+  #     optional createdAt: string = 10;  # original createdAt
   #   }
   # ```
   #
@@ -70,6 +74,7 @@ proc encode*(manifest: Manifest): ?!seq[byte] =
   header.write(4, manifest.codec.uint32)
   header.write(5, manifest.hcodec.uint32)
   header.write(6, manifest.version.uint32)
+
   if manifest.protected:
     var erasureInfo = initProtoBuffer()
     erasureInfo.write(1, manifest.ecK.uint32)
@@ -90,6 +95,12 @@ proc encode*(manifest: Manifest): ?!seq[byte] =
     erasureInfo.finish()
     header.write(7, erasureInfo)
 
+  header.write(8, manifest.filename.string)
+  header.write(9, manifest.mimetype.string)
+
+  if manifest.createdAt > 0:
+    header.write(10, manifest.createdAt.uint64)
+
   pbNode.write(1, header) # set the treeCid as the data field
   pbNode.finish()
 
@@ -118,6 +129,9 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
     slotRoots: seq[seq[byte]]
     cellSize: uint32
     verifiableStrategy: uint32
+    filename: string
+    mimetype: string
+    createdAt: uint64
 
   # Decode `Header` message
   if pbNode.getField(1, pbHeader).isErr:
@@ -145,6 +159,18 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
   if pbHeader.getField(7, pbErasureInfo).isErr:
     return failure("Unable to decode `erasureInfo` from manifest!")
 
+  if pbHeader.getField(8, filename).isErr:
+     # The filename field is an optional data so we can ignore the error the decoding fails
+    discard
+
+  if pbHeader.getField(9, mimetype).isErr:
+    # The mimetype field is an optional data so we can ignore the error the decoding fails
+    discard
+
+  if pbHeader.getField(10, createdAt).isErr:
+    # The createdAt field is an optional data so we can ignore the error the decoding fails
+    discard
+
   let protected = pbErasureInfo.buffer.len > 0
   var verifiable = false
   if protected:
@@ -196,15 +222,21 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest =
         ecM = ecM.int,
         originalTreeCid = ? Cid.init(originalTreeCid).mapFailure,
         originalDatasetSize = originalDatasetSize.NBytes,
-        strategy = StrategyType(protectedStrategy))
+        strategy = StrategyType(protectedStrategy),
+        filename = filename.string,
+        mimetype = mimetype.string,
+        createdAt = createdAt.int64)
       else:
         Manifest.new(
           treeCid = treeCid,
           datasetSize = datasetSize.NBytes,
           blockSize = blockSize.NBytes,
           version = CidVersion(version),
           hcodec = hcodec.MultiCodec,
-          codec = codec.MultiCodec)
+          codec = codec.MultiCodec,
+          filename = filename.string,
+          mimetype = mimetype.string,
+          createdAt = createdAt.int64)
 
   ? self.verify()
 

codex/manifest/manifest.nim

@@ -10,6 +10,7 @@
 # This module defines all operations on Manifest
 
 import pkg/upraises
+import times
 
 push: {.upraises: [].}
 
@@ -36,6 +37,9 @@ type
     codec: MultiCodec                       # Dataset codec
     hcodec: MultiCodec                      # Multihash codec
     version: CidVersion                     # Cid version
+    filename {.serialize.}: string          # The filename of the content uploaded (optional)
+    mimetype {.serialize.}: string          # The mimetype of the content uploaded (optional)
+    createdAt {.serialize.}: int64          # The UTC creation timestamp in seconds
     case protected {.serialize.}: bool      # Protected datasets have erasure coded info
     of true:
       ecK: int                              # Number of blocks to encode
@@ -121,6 +125,14 @@ func verifiableStrategy*(self: Manifest): StrategyType =
 func numSlotBlocks*(self: Manifest): int =
   divUp(self.blocksCount, self.numSlots)
 
+func filename*(self: Manifest): string =
+  self.filename
+
+func mimetype*(self: Manifest): string =
+  self.mimetype
+
+func createdAt*(self: Manifest): int64 =
+  self.createdAt
 ############################################################
 # Operations on block list
 ############################################################
@@ -163,6 +175,9 @@ func `==`*(a, b: Manifest): bool =
   (a.hcodec == b.hcodec) and
   (a.codec == b.codec) and
   (a.protected == b.protected) and
+  (a.filename == b.filename) and
+  (a.mimetype == b.mimetype) and
+  (a.createdAt == b.createdAt) and
     (if a.protected:
       (a.ecK == b.ecK) and
       (a.ecM == b.ecM) and
@@ -188,6 +203,9 @@ func `$`*(self: Manifest): string =
     ", hcodec: " & $self.hcodec &
     ", codec: " & $self.codec &
     ", protected: " & $self.protected &
+    ", filename: " & $self.filename &
+    ", mimetype: " & $self.mimetype &
+    ", createdAt: " & $self.createdAt &
     (if self.protected:
       ", ecK: " & $self.ecK &
       ", ecM: " & $self.ecM &
@@ -214,7 +232,10 @@ func new*(
   version: CidVersion = CIDv1,
   hcodec = Sha256HashCodec,
   codec = BlockCodec,
-  protected = false): Manifest =
+  protected = false,
+  filename = "",
+  mimetype = "",
+  createdAt = now().utc.toTime.toUnix): Manifest =
 
   T(
     treeCid: treeCid,
@@ -223,7 +244,10 @@ func new*(
     version: version,
     codec: codec,
     hcodec: hcodec,
-    protected: protected)
+    protected: protected,
+    filename: filename,
+    mimetype: mimetype,
+    createdAt: createdAt)
 
 func new*(
   T: type Manifest,
@@ -247,7 +271,11 @@ func new*(
     ecK: ecK, ecM: ecM,
     originalTreeCid: manifest.treeCid,
     originalDatasetSize: manifest.datasetSize,
-    protectedStrategy: strategy)
+    protectedStrategy: strategy,
+    filename: manifest.filename,
+    mimetype: manifest.mimetype,
+    createdAt: manifest.createdAt
+    )
 
 func new*(
   T: type Manifest,
@@ -263,7 +291,10 @@ func new*(
     codec: manifest.codec,
     hcodec: manifest.hcodec,
     blockSize: manifest.blockSize,
-    protected: false)
+    protected: false,
+    filename: manifest.filename,
+    mimetype: manifest.mimetype,
+    createdAt: manifest.createdAt)
 
 func new*(
   T: type Manifest,
@@ -277,7 +308,10 @@ func new*(
   ecM: int,
   originalTreeCid: Cid,
   originalDatasetSize: NBytes,
-  strategy = SteppedStrategy): Manifest =
+  strategy = SteppedStrategy,
+  filename = "",
+  mimetype = "",
+  createdAt = now().utc.toTime.toUnix): Manifest =
 
   Manifest(
     treeCid: treeCid,
@@ -291,7 +325,10 @@ func new*(
     ecM: ecM,
     originalTreeCid: originalTreeCid,
     originalDatasetSize: originalDatasetSize,
-    protectedStrategy: strategy)
+    protectedStrategy: strategy,
+    filename: filename, 
+    mimetype: mimetype,
+    createdAt: createdAt)
 
 func new*(
   T: type Manifest,
@@ -329,7 +366,11 @@ func new*(
     verifyRoot: verifyRoot,
     slotRoots: @slotRoots,
     cellSize: cellSize,
-    verifiableStrategy: strategy)
+    verifiableStrategy: strategy,
+    filename: manifest.filename,
+    mimetype: manifest.mimetype,
+    createdAt: manifest.createdAt
+    )
 
 func new*(
   T: type Manifest,

codex/node.nim

@@ -297,6 +297,8 @@ proc retrieve*(
 proc store*(
   self: CodexNodeRef,
   stream: LPStream,
+  filename = "",
+  mimetype = "",
   blockSize = DefaultBlockSize): Future[?!Cid] {.async.} =
   ## Save stream contents as dataset with given blockSize
   ## to nodes's BlockStore, and return Cid of its manifest
@@ -355,7 +357,9 @@ proc store*(
     datasetSize = NBytes(chunker.offset),
     version = CIDv1,
     hcodec = hcodec,
-    codec = dataCodec)
+    codec = dataCodec,
+    filename = filename,
+    mimetype = mimetype)
 
   without manifestBlk =? await self.storeManifest(manifest), err:
     error "Unable to store manifest"
@@ -364,7 +368,9 @@ proc store*(
   info "Stored data", manifestCid = manifestBlk.cid,
                       treeCid = treeCid,
                       blocks = manifest.blocksCount,
-                      datasetSize = manifest.datasetSize
+                      datasetSize = manifest.datasetSize,
+                      filename = manifest.filename,
+                      mimetype = manifest.mimetype
 
   return manifestBlk.cid.success
 

codex/rest/api.nim

@@ -13,6 +13,8 @@ push: {.upraises: [].}
 
 
 import std/sequtils
+import mimetypes
+import os
 
 import pkg/questionable
 import pkg/questionable/results
@@ -122,6 +124,18 @@ proc setCorsHeaders(resp: HttpResponseRef, httpMethod: string, origin: string) =
   resp.setHeader("Access-Control-Allow-Methods", httpMethod & ", OPTIONS")
   resp.setHeader("Access-Control-Max-Age", "86400")
 
+proc getFilenameFromContentDisposition(contentDisposition: string): string =
+  if not("filename=" in contentDisposition):
+    return ""
+
+  let parts = contentDisposition.split("filename=\"")
+
+  if parts.len < 2:
+    return ""
+
+  let filename = parts[1].strip()
+  return filename[0..^2]
+
 proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRouter) =
   let allowedOrigin = router.allowedOrigin # prevents capture inside of api defintion
 
@@ -132,7 +146,7 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
 
       if corsOrigin =? allowedOrigin:
         resp.setCorsHeaders("POST", corsOrigin)
-        resp.setHeader("Access-Control-Allow-Headers", "content-type")
+        resp.setHeader("Access-Control-Allow-Headers", "content-type, content-disposition")
 
       resp.status = Http204
       await resp.sendBody("")
@@ -155,12 +169,29 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute
       #
       await request.handleExpect()
 
+      let mimetype: string = request.headers.getString(ContentTypeHeader)
+
+      if mimetype != "":
+        var m = newMimetypes()
+        let extension = m.getExt(mimetype, "")
+        if extension == "":
+            return RestApiResponse.error(Http422, "The MIME type is not valid.")
+
+      const ContentDispositionHeader = "Content-Disposition"
+      let contentDisposition = request.headers.getString(ContentDispositionHeader)
+      let filename = getFilenameFromContentDisposition(contentDisposition)
+
+      if filename != "" and not isValidFilename(filename):
+          return RestApiResponse.error(Http422, "The filename is not valid.")
+
+      # Here we could check if the extension matches the filename if needed
+
       let
         reader = bodyReader.get()
 
       try:
         without cid =? (
-          await node.store(AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)))), error:
+          await node.store(AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)), filename = filename, mimetype = mimetype)), error:
           error "Error uploading file", exc = error.msg
           return RestApiResponse.error(Http500, error.msg)
 

openapi.yaml

@@ -319,6 +319,18 @@ components:
         protected:
           type: boolean
           description: "Indicates if content is protected by erasure-coding"
+        filename:
+          type: string
+          description: "The original name of the uploaded content (optional)"
+          example: codex.png
+        mimetype:
+          type: string
+          description: "The original mimetype of the uploaded content (optional)"
+          example: image/png
+        createdAt:
+          type: int64
+          description: "The UTC creation timestamp in seconds"
+          example: 1729244192
 
     Space:
       type: object