Followers collection synchronization

CHANGES.md

@@ -8,6 +8,19 @@ Version 0.8.0
 
 To be released.
 
+ -  Implemented [followers collection synchronization mechanism][FEP-8fcf].
+
+     -  Added `RequestContext.sendActivity()` overload that takes `"followers"`
+        as the second parameter.
+     -  Added the second type parameter to `CollectionCallbackSetters`
+        interface.
+     -  Added the second type parameter to `CollectionDispatcher` type.
+     -  Added the fourth parameter to `CollectionDispatcher` type.
+     -  Added the second type parameter to `CollectionCounter` type.
+     -  Added the third parameter to `CollectionCounter` type.
+     -  Added the second type parameter to `CollectionCursor` type.
+     -  Added the third parameter to `CollectionCursor` type.
+
  -  Relaxed the required type for activity recipients. 
 
      -  Added `Recipient` interface.
@@ -16,6 +29,15 @@ To be released.
         since `Recipient` is a supertype of `Actor`, the existing code should
         work without any change.
 
+ -  Followers collection now has to consist of `Recipient` objects only.
+    (It could consist of `URL`s as well as `Actor`s before.)
+
+     -  The type of `Federation.setFollowersDispatcher()` method's second
+        parameter became `CollectionDispatcher<Recipient, TContextData, URL>`
+        (was `CollectionDispatcher<Actor | URL, TContextData>`).
+
+[FEP-8fcf]: https://codeberg.org/fediverse/fep/src/branch/main/fep/8fcf/fep-8fcf.md
+
 
 Version 0.7.0
 -------------
@@ -52,7 +74,7 @@ Released on April 23, 2024.
  -  Activity Vocabulary classes now have `typeId` static property.
 
  -  Dispatcher setters and inbox listener setters in `Federation` now take
-    a path as <code>`${string}{handle}${string}`</code> instead of `string`
+    a path as `` `${string}{handle}${string}` `` instead of `string`
     so that it is more type-safe.
 
  -  Added generalized object dispatchers.  [[#33]]

docs/manual/collections.md

@@ -354,26 +354,84 @@ Followers
 
 The followers collection is very similar to the following collection, but it's
 a collection of actors that are following the actor.  The followers collection
-also can consist of `Actor` objects or `URL` objects that represent the actors.
+has to consist of `Recipient` objects that represent the actors.
 
 The below example shows how to construct a followers collection:
 
 ~~~~ typescript
 federation
-  .setFollowersDispatcher("/users/{handle}/followers", async (ctx, handle, cursor) => {
-    // If a whole collection is requested, returns nothing as we prefer
-    // collection pages over the whole collection:
-    if (cursor == null) return null;
-    // Work with the database to find the actors that are following the actor
-    // (the below `getFollowersByUserHandle` is a hypothetical function):
-    const { users, nextCursor, last } = await getFollowersByUserHandle(
-      handle,
-      cursor === "" ? { limit: 10 } : { cursor, limit: 10 }
-    );
-    // Turn the users into `URL` objects:
-    const items = users.map(actor => actor.uri);
-    return { items, nextCursor: last ? null : nextCursor }
-  })
+  .setFollowersDispatcher(
+    "/users/{handle}/followers",
+    async (ctx, handle, cursor) => {
+      // If a whole collection is requested, returns nothing as we prefer
+      // collection pages over the whole collection:
+      if (cursor == null) return null;
+      // Work with the database to find the actors that are following the actor
+      // (the below `getFollowersByUserHandle` is a hypothetical function):
+      const { users, nextCursor, last } = await getFollowersByUserHandle(
+        handle,
+        cursor === "" ? { limit: 10 } : { cursor, limit: 10 }
+      );
+      // Turn the users into `Recipient` objects:
+      const items = users.map(actor => ({
+        id: new URL(actor.uri),
+        inboxId: new URL(actor.inboxUri),
+      }));
+      return { items, nextCursor: last ? null : nextCursor };
+    }
+  )
   // The first cursor is an empty string:
   .setFirstCursor(async (ctx, handle) => "");
 ~~~~
+
+> [!TIP]
+>
+> Every `Actor` object is also a `Recipient` object, so you can use the `Actor`
+> object as the `Recipient` object.
+
+### Filtering by server
+
+*This API is available since Fedify 0.8.0.*
+
+The followers collection can be filtered by the base URI of the actor URIs.
+It can be useful to filter by a remote server to synchronize the followers
+collection with it.
+
+> [!TIP]
+> However, the filtering is optional, and you can skip it if you don't need
+> [followers collection synchronization](./send.md#followers-collection-synchronization).
+
+In order to filter the followers collection by the server, you need to let
+your followers collection dispatcher be aware of the fourth argument:
+the base URI of the actor URIs to filter in.  The base URI consists of
+the protocol, the authority (the host and the port), and the root path of
+the actor URIs.  When the base URI is not `null`, the dispatcher should
+return only the actors whose URIs start with the base URI.  If the base URI
+is `null`, the dispatcher should return all the actors.
+
+The following example shows how to filter the followers collection by the
+server:
+
+~~~~ typescript {8-11}
+federation
+  .setFollowersDispatcher(
+    "/users/{handle}/followers",
+    async (ctx, handle, cursor, baseUri) => {
+      // Work with the database to find the actors that are following the actor
+      // (the below `getFollowersByUserHandle` is a hypothetical function):
+      let users = await getFollowersByUserHandle(handle);
+      // Filter the actors by the base URI:
+      if (baseUri != null) {
+        users = users.filter(actor => actor.uri.href.startsWith(baseUri.href));
+      }
+      // Turn the users into `URL` objects:
+      const items = users.map(actor => actor.uri);
+      return { items };
+    }
+  );
+~~~~
+
+> [!NOTE]
+> In the above example, we filter the actors in memory, but in the real
+> world, you should filter the actors in the database query to improve the
+> performance.

docs/manual/log.md

@@ -89,8 +89,8 @@ The `configure()` function takes an object with three properties:
 
 `loggers.level` (optional)
 :   The `level` property is a string that specifies the log level.  The log
-    level can be one of the following: `"trace"`, `"debug"`, `"info"`,
-    `"warning"`, `"error"`, or `"fatal"`.
+    level can be one of the following: `"debug"`, `"info"`, `"warning"`,
+    `"error"`, or `"fatal"`.
 
 
 Categories
@@ -117,6 +117,11 @@ The `"fedify"` category is used for everything related to the Fedify library.
 The `["fedify", "federation"]` category is used for logging federation-related
 messages.
 
+### `["fedify", "federation", "collection"]`
+
+The `["fedify", "federation", "collection"]` category is used for logging
+messages related to collections (e.g., outbox, followers, following).
+
 ### `["fedify", "federation", "inbox"]`
 
 The `["fedify", "federation", "inbox"]` category is used for logging messages

docs/manual/send.md

@@ -177,6 +177,60 @@ async function sendNote(
 [shared inbox delivery]: https://www.w3.org/TR/activitypub/#shared-inbox-delivery
 
 
+Followers collection synchronization
+------------------------------------
+
+*This API is available since Fedify 0.8.0.*
+
+> [!NOTE]
+> For efficiency, you should implement
+> [filtering-by-server](./collections.md#filtering-by-server) of
+> the followers collection, otherwise the synchronization may be slow.
+
+If an activity needs to be delivered to only followers of the sender through
+the shared inbox, the server of the recipients has to be aware of the list of
+followers residing on the server.  However, synchronizing the followers
+collection every time an activity is sent is inefficient. To solve this problem,
+Mastodon, etc., use a mechanism called [followers collection
+synchronization][FEP-8fcf].
+
+The idea is to send a digest of the followers collection with the activity
+so that the recipient server can check if it needs to resynchronize
+the followers collection.  Fedify provides a way to include the digest
+of the followers collection in the activity delivery request by specifying
+the recipients parameter of the `~Context.sendActivity()` method as
+the `"followers"` string:
+
+~~~~ typescript
+await ctx.sendActivity(
+  { handle: senderHandle },
+  "followers",  // [!code highlight]
+  new Create({
+    actor: ctx.getActorUri(senderHandle),
+    to: ctx.getFollowersUri(senderHandle),
+    object: new Note({
+      attribution: ctx.getActorUri(senderHandle),
+      to: ctx.getFollowersUri(senderHandle),
+    }),
+  }),
+  { preferSharedInbox: true },  // [!code highlight]
+);
+~~~~
+
+If you specify the `"followers"` string as the recipients parameter,
+it automatically sends the activity to the sender's followers and includes
+the digest of the followers collection in the payload.
+
+> [!NOTE]
+> The `to` and `cc` properties of an `Activity` and its `object` should be set
+> to the followers collection IRI to ensure that the activity is visible to
+> the followers.  If you set the `to` and `cc` properties to
+> the `PUBLIC_COLLECTION`, the activity is visible to everyone regardless of
+> the recipients parameter.
+
+[FEP-8fcf]: https://codeberg.org/fediverse/fep/src/branch/main/fep/8fcf/fep-8fcf.md
+
+
 Error handling
 --------------
 

examples/blog/.gitignore

@@ -7,5 +7,7 @@
 
 # Fresh build directory
 _fresh/
+# Log files
+log.jsonl
 # npm dependencies
 node_modules/

examples/blog/deno.json

@@ -29,6 +29,7 @@
     "jsxImportSource": "preact"
   },
   "unstable": [
+    "fs",
     "kv"
   ]
 }

examples/blog/federation/mod.ts

@@ -24,6 +24,7 @@ import {
   countFollowers,
   getFollowers,
   removeFollower,
+  toRecipient,
 } from "../models/follower.ts";
 import { countPosts, getPost, getPosts, toArticle } from "../models/post.ts";
 import { openKv } from "../models/kv.ts";
@@ -281,7 +282,7 @@ federation
         cursor === "" ? undefined : cursor,
       );
       return {
-        items: followers.map((f) => new URL(f.id)),
+        items: followers.map(toRecipient),
         nextCursor,
       };
     },

examples/blog/import_map.g.json

@@ -26,8 +26,8 @@
     "@std/fs": "jsr:@std/fs@^0.220.1",
     "@std/http/negotiation": "jsr:@std/http@^0.220.1/negotiation",
     "@std/json/common": "jsr:@std/json@^0.220.1/common",
-    "@std/path": "jsr:@std/path@^0.220.1",
-    "@std/semver": "jsr:@std/semver@^0.220.1",
+    "@std/path": "jsr:@std/path@^0.224.0",
+    "@std/semver": "jsr:@std/semver@^0.224.0",
     "@std/testing": "jsr:@std/testing@^0.220.1",
     "@std/text": "jsr:@std/text@^0.220.1",
     "@std/url": "jsr:@std/url@^0.220.1",
@@ -40,7 +40,8 @@
     "$fresh/": "https://deno.land/x/[email protected]/",
     "@preact/signals": "https://esm.sh/*@preact/[email protected]",
     "@preact/signals-core": "https://esm.sh/*@preact/[email protected]",
-    "@std/dotenv/load": "jsr:@std/dotenv@^0.220.1/load",
+    "@std/dotenv/load": "jsr:@std/dotenv@^0.224.0/load",
+    "@std/encoding/hex": "jsr:@std/encoding@^0.224.0/hex",
     "markdown-it": "npm:markdown-it@^14.0.0",
     "preact": "https://esm.sh/[email protected]",
     "preact/": "https://esm.sh/[email protected]/",

examples/blog/import_map.json

@@ -4,9 +4,10 @@
     "@js-temporal/polyfill": "npm:@js-temporal/polyfill@^0.4.4",
     "@preact/signals": "https://esm.sh/*@preact/[email protected]",
     "@preact/signals-core": "https://esm.sh/*@preact/[email protected]",
-    "@std/dotenv/load": "jsr:@std/dotenv@^0.220.1/load",
-    "@std/path": "jsr:@std/path@^0.220.1",
-    "@std/semver": "jsr:@std/semver@^0.220.1",
+    "@std/dotenv/load": "jsr:@std/dotenv@^0.224.0/load",
+    "@std/encoding/hex": "jsr:@std/encoding@^0.224.0/hex",
+    "@std/path": "jsr:@std/path@^0.224.0",
+    "@std/semver": "jsr:@std/semver@^0.224.0",
     "@fedify/fedify": "../../mod.ts",
     "@fedify/fedify/federation": "../../federation/mod.ts",
     "@fedify/fedify/httpsig": "../../httpsig/mod.ts",

examples/blog/loggers.ts

@@ -1,23 +1,35 @@
-import { configure, getConsoleSink, type LogLevel } from "@logtape/logtape";
+import {
+  configure,
+  getConsoleSink,
+  getFileSink,
+  type LogLevel,
+} from "@logtape/logtape";
 
 await configure({
-  sinks: { console: getConsoleSink() },
+  sinks: {
+    console: getConsoleSink(),
+    file: getFileSink("log.jsonl", {
+      formatter(log) {
+        return JSON.stringify(log) + "\n";
+      },
+    }),
+  },
   filters: {},
   loggers: [
     {
       category: "fedify",
       level: (Deno.env.get("FEDIFY_LOG") as LogLevel | undefined) ?? "debug",
-      sinks: ["console"],
+      sinks: ["console", "file"],
     },
     {
       category: "blog",
       level: (Deno.env.get("BLOG_LOG") as LogLevel | undefined) ?? "debug",
-      sinks: ["console"],
+      sinks: ["console", "file"],
     },
     {
       category: ["logtape", "meta"],
       level: "warning",
-      sinks: ["console"],
+      sinks: ["console", "file"],
     },
   ],
 });

examples/blog/models/follower.ts

@@ -1,9 +1,4 @@
-import {
-  Actor,
-  ActorTypeName,
-  Endpoints,
-  getActorClassByTypeName,
-} from "@fedify/fedify/vocab";
+import { ActorTypeName, Recipient } from "@fedify/fedify/vocab";
 import { openKv } from "./kv.ts";
 
 export interface Follower {
@@ -66,21 +61,12 @@ export async function countFollowers(): Promise<bigint> {
   return (record?.value as bigint | null) ?? 0n;
 }
 
-export async function getFollowersAsActors(): Promise<Actor[]> {
-  const kv = await openKv();
-  const actors: Actor[] = [];
-  for await (const f of kv.list<Follower>({ prefix: ["follower"] })) {
-    const cls = getActorClassByTypeName(f.value.typeName);
-    const actor = new cls({
-      id: new URL(f.value.id),
-      inbox: new URL(f.value.inbox),
-      endpoints: new Endpoints({
-        sharedInbox: f.value.sharedInbox
-          ? new URL(f.value.sharedInbox)
-          : undefined,
-      }),
-    });
-    actors.push(actor);
-  }
-  return actors;
+export function toRecipient(follower: Follower): Recipient {
+  return {
+    id: new URL(follower.id),
+    inboxId: new URL(follower.inbox),
+    endpoints: {
+      sharedInbox: follower.sharedInbox ? new URL(follower.sharedInbox) : null,
+    },
+  };
 }