Should backends be prefix aware?

[tpendragon]

Hi! I'm in the midst of writing an Ecto storage backend (just about there) and one thing I ran into is that the Backend's #put_object keys include the prefix as part of the key. That makes a ton of sense in the context of an object store, but for Ecto it means I have to do something like this for list_all_objects_stream:

  def list_all_objects_stream(%{repo: repo} = _state, prefix, opts) when is_binary(prefix) do
    {error_handler, _stream_opts} =
      Keyword.pop(opts, :error_handler, fn reason -> raise inspect(reason) end)

    query =
      from(o in Object,
        where: like(o.key, ^"#{prefix}%"),
        order_by: [asc: o.key]
      )

    page_size = 500

    # Need to page through postgres basically.
    try do
      Stream.unfold(:start, fn
        :done ->
          nil

        cursor ->
          page_query =
            case cursor do
              :start -> query |> limit(^page_size)
              last_key -> query |> where([o], o.key > ^last_key) |> limit(^page_size)
            end

          case repo.all(page_query) do
            [] ->
              nil

            rows ->
              next_cursor =
                if length(rows) < page_size, do: :done, else: List.last(rows).key

              {rows, next_cursor}
          end
      end)
      |> Stream.flat_map(& &1)
      |> Stream.map(fn %Object{key: key, body: body, version: version} ->
        case decode_body(body) do
          {:ok, decoded} ->
            %{key: key, etag: Integer.to_string(version), body: decoded}

          {:error, reason} ->
            case error_handler.({:decode_failed, key, reason}) do
              :halt -> throw(:halt_stream)
              _ -> nil
            end
        end
      end)
      |> Stream.reject(&is_nil/1)
    rescue
      error ->
        case error_handler.(error) do
          _ -> Stream.map([], & &1)
        end
    end
  end

If the contract was something like

  def get_object(state, key, prefix, _opts)

or even if keys were structs that contained the prefix, then I could just add that as a column to my database.

Also I'd really really love if there was some sort of shared end to end testing suite I could hook up to my store to make sure it worked - I'm kinda wingin' it right now.