Skip to content

Commit

Permalink
add partitions to json format
Browse files Browse the repository at this point in the history
and fix missing replica setting too.

also remove partitions from shape hash value much like we exclude the
table info
  • Loading branch information
magnetised committed Dec 20, 2024
1 parent 8aafed5 commit 3c45c27
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 7 deletions.
26 changes: 20 additions & 6 deletions packages/sync-service/lib/electric/shapes/shape.ex
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,13 @@ defmodule Electric.Shapes.Shape do
root_table_id: non_neg_integer(),
where: String.t(),
selected_columns: [String.t(), ...] | nil,
table_info: [json_table_list(), ...]
table_info: [json_table_list(), ...],
partitions: %{[String.t()] => [String.t()]},
replica: String.t()
}

def hash(%__MODULE__{} = shape), do: shape |> Map.drop([:table_info]) |> :erlang.phash2()
def hash(%__MODULE__{} = shape),
do: shape |> Map.drop([:table_info, :partitions]) |> :erlang.phash2()

def generate_id(%__MODULE__{} = shape) do
hash = hash(shape)
Expand Down Expand Up @@ -342,7 +345,9 @@ defmodule Electric.Shapes.Shape do
root_table_id: root_table_id,
where: where,
selected_columns: selected_columns,
table_info: table_info
table_info: table_info,
partitions: partitions,
replica: replica
} = shape

query =
Expand All @@ -362,7 +367,9 @@ defmodule Electric.Shapes.Shape do
Enum.map(table_info, fn {{schema, name}, columns} ->
[[schema, name], json_safe_columns(columns)]
end)
)
),
partitions: Enum.map(partitions, fn {{kk, kv}, {vk, vv}} -> [[kk, kv], [vk, vv]] end),
replica: to_string(replica)
}
end

Expand All @@ -385,7 +392,9 @@ defmodule Electric.Shapes.Shape do
"root_table_id" => root_table_id,
"where" => where,
"selected_columns" => selected_columns,
"table_info" => info
"table_info" => info,
"partitions" => partitions,
"replica" => replica
} = map

table_info =
Expand All @@ -407,13 +416,18 @@ defmodule Electric.Shapes.Shape do
root_table_id: root_table_id,
where: where,
selected_columns: selected_columns,
table_info: table_info
table_info: table_info,
partitions: Map.new(partitions, fn [[kk, kv], [vk, vv]] -> {{kk, kv}, {vk, vv}} end),
replica: replica_from_json(replica)
}
end

defp column_info_from_json({"type_id", [id, mod]}), do: {:type_id, {id, mod}}
defp column_info_from_json({"type", type}), do: {:type, String.to_atom(type)}
defp column_info_from_json({key, value}), do: {String.to_atom(key), value}

defp replica_from_json("full"), do: :full
defp replica_from_json("default"), do: :default
end

defimpl Inspect, for: Electric.Shapes.Shape do
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -575,7 +575,7 @@ defmodule Electric.ShapeCache.StorageImplimentationsTest do
storage.initialise(shape_opts)
storage.set_shape_definition(@shape, shape_opts)

assert 2274 = Electric.ShapeCache.Storage.get_total_disk_usage({storage, opts})
assert 2310 = Electric.ShapeCache.Storage.get_total_disk_usage({storage, opts})
end
end
end
Expand Down
39 changes: 39 additions & 0 deletions packages/sync-service/test/electric/shapes/shape_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,45 @@ defmodule Electric.Shapes.ShapeTest do
assert {:ok, json} = Jason.encode(shape)
assert ^shape = Jason.decode!(json) |> Shape.from_json_safe!()
end

test "should serialise shape replica setting" do
shape = %Electric.Shapes.Shape{
root_table: {"public", "foo"},
root_table_id: 1,
table_info: %{
{"public", "foo"} => %{
columns: [],
pk: ["first", "second", "third"]
}
},
where: nil,
replica: :full
}

assert {:ok, json} = Jason.encode(shape)
assert ^shape = Jason.decode!(json) |> Shape.from_json_safe!()
end

test "should serialise shape on partitioned table" do
shape = %Electric.Shapes.Shape{
root_table: {"public", "foo"},
root_table_id: 1,
table_info: %{
{"public", "foo"} => %{
columns: [],
pk: ["first", "second", "third"]
}
},
where: nil,
partitions: %{
{"public", "foo_1"} => {"public", "foo"},
{"public", "foo_2"} => {"public", "foo"}
}
}

assert {:ok, json} = Jason.encode(shape)
assert ^shape = Jason.decode!(json) |> Shape.from_json_safe!()
end
end

def load_column_info({"public", "table"}, _),
Expand Down

0 comments on commit 3c45c27

Please sign in to comment.