Fix array decoding to handle quoted elements

.github/workflows/test.yaml

@@ -35,22 +35,18 @@ jobs:
 
       - uses: actions/checkout@v4
 
-      - uses: erlef/setup-beam@v1
-        id: setup-elixir
-        with:
-          version-type: strict
-          version-file: .tool-versions
+      - uses: jdx/mise-action@v2
 
       - name: Cache Deps
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: deps
           key: ${{ runner.os }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
           restore-keys: |
             ${{ runner.os }}-mix-
 
       - name: Cache Build
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: _build
           key: ${{ runner.os }}-build-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
@@ -76,22 +72,18 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: erlef/setup-beam@v1
-        id: setup-elixir
-        with:
-          version-type: strict
-          version-file: .tool-versions
+      - uses: jdx/mise-action@v2
 
       - name: Cache Deps
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: deps
           key: ${{ runner.os }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
           restore-keys: |
             ${{ runner.os }}-mix-
 
       - name: Cache Build
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: _build
           key: ${{ runner.os }}-build-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
@@ -111,22 +103,18 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: erlef/setup-beam@v1
-        id: setup-elixir
-        with:
-          version-type: strict
-          version-file: .tool-versions
+      - uses: jdx/mise-action@v2
 
       - name: Cache Deps
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: deps
           key: ${{ runner.os }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
           restore-keys: |
             ${{ runner.os }}-mix-
 
       - name: Cache Build
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: _build
           key: ${{ runner.os }}-build-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
@@ -143,22 +131,18 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: erlef/setup-beam@v1
-        id: setup-elixir
-        with:
-          version-type: strict
-          version-file: .tool-versions
+      - uses: jdx/mise-action@v2
 
       - name: Cache Deps
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: deps
           key: ${{ runner.os }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
           restore-keys: |
             ${{ runner.os }}-mix-
 
       - name: Cache Build
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: _build
           key: ${{ runner.os }}-build-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
@@ -177,27 +161,23 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: erlef/setup-beam@v1
-        id: setup-elixir
-        with:
-          version-type: strict
-          version-file: .tool-versions
+      - uses: jdx/mise-action@v2
 
       - name: Cache Deps
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: deps
           key: ${{ runner.os }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
           restore-keys: |
             ${{ runner.os }}-mix-
 
       - name: Cache Build
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: _build
           key: ${{ runner.os }}-build-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
           restore-keys: |
             ${{ runner.os }}-build-
 
       - run: mix do deps.get, compile
-      - run: mix sobelow --config
+      - run: mix sobelow --config

.tool-versions

@@ -1,2 +1,2 @@
-elixir 1.18.2-otp-27
-erlang 27.1.3
+elixir 1.19.5-otp-28
+erlang 28.4.1

lib/live_sync/replication.ex

@@ -281,9 +281,7 @@ defmodule LiveSync.Replication do
 
   defp load_value({:array, type}, value) do
     value
-    |> String.replace_leading("{", "")
-    |> String.replace_trailing("}", "")
-    |> String.split(",")
+    |> parse_array()
     |> Enum.map(&load_value(type, &1))
   end
 
@@ -308,6 +306,41 @@ defmodule LiveSync.Replication do
     Ecto.Type.cast!(type, value)
   end
 
+  # Postgres delivers arrays in its text output format, e.g.
+  #   {"a, b","c",NULL,plain}
+  # Any element containing a comma, quote, brace, backslash or whitespace (or an
+  # empty string) is double-quoted, with `"` and `\` backslash-escaped inside,
+  # and an unquoted NULL means nil. Splitting on `,` corrupts quoted elements, so
+  # we walk the literal, returning raw strings (or nil) for load_value/2 to cast.
+  defp parse_array("{" <> rest), do: parse_array(rest, [])
+
+  defp parse_array("}" <> _rest, acc), do: Enum.reverse(acc)
+  defp parse_array("," <> rest, acc), do: parse_array(rest, acc)
+
+  defp parse_array(<<?", rest::binary>>, acc) do
+    {value, rest} = parse_quoted(rest, [])
+    parse_array(rest, [value | acc])
+  end
+
+  defp parse_array(binary, acc) do
+    {value, rest} = parse_unquoted(binary, [])
+    parse_array(rest, [value | acc])
+  end
+
+  defp parse_quoted(<<?\\, char::utf8, rest::binary>>, acc), do: parse_quoted(rest, [<<char::utf8>> | acc])
+  defp parse_quoted(<<?", rest::binary>>, acc), do: {IO.iodata_to_binary(Enum.reverse(acc)), rest}
+  defp parse_quoted(<<char::utf8, rest::binary>>, acc), do: parse_quoted(rest, [<<char::utf8>> | acc])
+
+  # a nil element is only ever an unquoted NULL; a real "NULL" string is quoted
+  defp parse_unquoted(<<char, _rest::binary>> = binary, acc) when char in [?,, ?}] do
+    case IO.iodata_to_binary(Enum.reverse(acc)) do
+      "NULL" -> {nil, binary}
+      value -> {value, binary}
+    end
+  end
+
+  defp parse_unquoted(<<char::utf8, rest::binary>>, acc), do: parse_unquoted(rest, [<<char::utf8>> | acc])
+
   @epoch DateTime.to_unix(~U[2000-01-01 00:00:00Z], :microsecond)
   defp current_time, do: System.os_time(:microsecond) - @epoch
 

lib/live_sync/replication_test.exs

@@ -10,7 +10,7 @@ defmodule LiveSync.ReplicationTest do
   @moduletag cleanup: ["ignored", "examples"]
 
   setup do
-    LiveSync.start_link(repo: LiveSync.Repo, otp_app: :live_sync)
+    LiveSync.start_link(repo: Repo, otp_app: :live_sync)
     :ok
   end
 
@@ -38,20 +38,20 @@ defmodule LiveSync.ReplicationTest do
 
     assert_receive {:live_sync,
                     [
-                      delete: %LiveSync.Example{id: delete_id, name: nil}
+                      delete: %Example{id: delete_id, name: nil}
                     ]}
 
     assert_receive {:live_sync,
                     [
-                      update: %LiveSync.Example{
+                      update: %Example{
                         id: update_id,
                         name: "more replication",
                         enabled: true,
                         input: %{"key" => "value"},
                         embed_one: %EmbedOne{name: "embed"},
                         embed_many: [%EmbedMany{name: "many"}]
                       },
-                      insert: %LiveSync.Example{
+                      insert: %Example{
                         id: insert_id,
                         name: "replication",
                         enabled: false,
@@ -65,4 +65,37 @@ defmodule LiveSync.ReplicationTest do
     assert id == update_id
     assert id == delete_id
   end
+
+  test "decodes array columns whose elements need quoting" do
+    Replication.subscribe("live_sync:1")
+
+    # Postgres quotes array elements that contain commas, quotes, braces,
+    # backslashes or whitespace, and represents an unquoted NULL for nil.
+    tags = [
+      "DELETE FROM t RETURNING id, name, tags",
+      ~s(has "double" quotes),
+      "back\\slash",
+      "{braced}",
+      "",
+      # a quoted "NULL" string must stay a string, distinct from a real NULL (nil)
+      "NULL",
+      nil,
+      "plain"
+    ]
+
+    {:ok, id} =
+      Repo.transaction(fn ->
+        # insert the struct directly so changeset empty-value filtering doesn't
+        # drop the empty-string element we want to exercise the decoder against
+        example = Repo.insert!(%Example{organization_id: 1, name: "arrays", tags: tags})
+        Repo.update!(change(example, name: "updated arrays"))
+        example.id
+      end)
+
+    assert_receive {:live_sync,
+                    [
+                      update: %Example{id: ^id, name: "updated arrays", tags: ^tags},
+                      insert: %Example{id: ^id, name: "arrays", tags: ^tags}
+                    ]}
+  end
 end

lib/live_sync/socket_test.exs

@@ -6,7 +6,7 @@ defmodule LiveSync.SocketTest do
   alias LiveSync.Repo
 
   setup do
-    LiveSync.start_link(repo: LiveSync.Repo, otp_app: :live_sync)
+    LiveSync.start_link(repo: Repo, otp_app: :live_sync)
     :ok
   end
 

lib/test_helper.exs

@@ -13,6 +13,7 @@ CREATE TABLE examples (
   parent_id bytea REFERENCES examples(id),
   organization_id integer,
   input jsonb,
+  tags text[],
   embed_one jsonb,
   embed_many jsonb
 );

mix.exs

@@ -4,19 +4,14 @@ defmodule LiveSync.MixProject do
   def project do
     [
       app: :live_sync,
-      version: "0.1.10",
+      version: "0.1.11",
       elixir: "~> 1.17",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,
       deps: deps(),
       package: package(),
       test_paths: ["lib"],
-      test_coverage: [tool: ExCoveralls],
-      preferred_cli_env: [
-        coveralls: :test,
-        "coveralls.html": :test,
-        "coveralls.json": :test
-      ]
+      test_coverage: [tool: ExCoveralls]
     ]
   end
 
@@ -29,6 +24,16 @@ defmodule LiveSync.MixProject do
     ]
   end
 
+  def cli do
+    [
+      preferred_envs: [
+        coveralls: :test,
+        "coveralls.html": :test,
+        "coveralls.json": :test
+      ]
+    ]
+  end
+
   defp package do
     [
       maintainers: ["Michael St Clair"],
@@ -50,6 +55,7 @@ defmodule LiveSync.MixProject do
       {:ex_doc, ">= 0.0.0", only: :dev, runtime: false},
       {:excoveralls, "~> 0.10", only: :test},
       {:floki, ">= 0.30.0", only: :test},
+      {:lazy_html, ">= 0.1.0", only: :test},
       {:mix_audit, "~> 2.0", only: [:dev, :test], runtime: false},
       {:sobelow, "~> 0.13", only: [:dev, :test], runtime: false},
       {:styler, "~> 1.0", only: [:dev, :test], runtime: false}