Support values lists (#552)

Co-authored-by: narrowtux <narrow.m@gmail.com>

Author: greg-rychlewski

.github/workflows/ci.yml

@@ -47,7 +47,7 @@ jobs:
       - uses: earthly/actions-setup@v1
       - uses: actions/checkout@v3
       - name: test ecto_sql
-        run: earthly -P --ci --build-arg ELIXIR_BASE=${{matrix.elixirbase}} --build-arg POSTGRES=${{matrix.postgres}} +integration-test-mysql
+        run: earthly -P --ci --build-arg ELIXIR_BASE=${{matrix.elixirbase}} --build-arg MYSQL=${{matrix.mysql}} +integration-test-mysql
   test-mssql:
     name: mssql integration test
     runs-on: ubuntu-latest

Earthfile

@@ -71,7 +71,7 @@ integration-test-postgres:
     ELSE IF [ "$POSTGRES" = "15.0" ]
         # for 15.0 we need an upgraded version of pg_dump;
         # alpine 3.17 does not come with the postgres 15 client by default;
-        # we must first update the public keys for the packages because they 
+        # we must first update the public keys for the packages because they
         # might have been rotated since our image was built
         RUN apk add -X https://dl-cdn.alpinelinux.org/alpine/v3.17/main -u alpine-keys
         RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.17/main' >> /etc/apk/repositories

integration_test/myxql/test_helper.exs

@@ -108,7 +108,7 @@ excludes = [
 if Version.match?(version, ">= 8.0.0") do
   ExUnit.configure(exclude: excludes)
 else
-  ExUnit.configure(exclude: [:rename_column | excludes])
+  ExUnit.configure(exclude: [:values_list, :rename_column | excludes])
 end
 
 :ok = Ecto.Migrator.up(TestRepo, 0, Ecto.Integration.Migration, log: false)

lib/ecto/adapters/myxql/connection.ex

@@ -704,6 +704,10 @@ if Code.ensure_loaded?(MyXQL) do
       |> parens_for_select
     end
 
+    defp expr({:values, _, [types, _idx, num_rows]}, _, query) do
+      [?(, values_list(types, num_rows, query), ?)]
+    end
+
     defp expr({:literal, _, [literal]}, _sources, _query) do
       quote_name(literal)
     end
@@ -832,6 +836,23 @@ if Code.ensure_loaded?(MyXQL) do
       error!(query, "unsupported expression: #{inspect(expr)}")
     end
 
+    defp values_list(types, num_rows, query) do
+      rows = Enum.to_list(1..num_rows)
+
+      [
+        "VALUES ",
+        intersperse_map(rows, ?,, fn _ ->
+          ["ROW(", values_expr(types, query), ?)]
+        end)
+      ]
+    end
+
+    defp values_expr(types, query) do
+      intersperse_map(types, ?,, fn {_field, type} ->
+        ["CAST(", ??, " AS ", ecto_cast_to_db(type, query), ?)]
+      end)
+    end
+
     defp interval(count, "millisecond", sources, query) do
       ["INTERVAL (", expr(count, sources, query) | " * 1000) microsecond"]
     end
@@ -870,6 +891,9 @@ if Code.ensure_loaded?(MyXQL) do
         {:fragment, _, _} ->
           {nil, as_prefix ++ [?f | Integer.to_string(pos)], nil}
 
+        {:values, _, _}  ->
+          {nil, as_prefix ++ [?v | Integer.to_string(pos)], nil}
+
         {table, schema, prefix} ->
           name = as_prefix ++ [create_alias(table) | Integer.to_string(pos)]
           {quote_table(prefix, table), name, schema}
@@ -1330,6 +1354,7 @@ if Code.ensure_loaded?(MyXQL) do
 
     defp get_source(query, sources, ix, source) do
       {expr, name, _schema} = elem(sources, ix)
+      name = maybe_add_column_names(source, name)
       {expr || expr(source, sources, query), name}
     end
 
@@ -1340,6 +1365,13 @@ if Code.ensure_loaded?(MyXQL) do
       end
     end
 
+    defp maybe_add_column_names({:values, _, [types, _, _]}, name) do
+      fields = Keyword.keys(types)
+      [name, ?\s, ?(, quote_names(fields), ?)]
+    end
+
+    defp maybe_add_column_names(_, name), do: name
+
     defp quote_name(name) when is_atom(name) do
       quote_name(Atom.to_string(name))
     end

lib/ecto/adapters/postgres/connection.ex

@@ -893,6 +893,10 @@ if Code.ensure_loaded?(Postgrex) do
       |> parens_for_select
     end
 
+    defp expr({:values, _, [types, idx, num_rows]}, _, _query) do
+      [?(, values_list(types, idx + 1, num_rows), ?)]
+    end
+
     defp expr({:literal, _, [literal]}, _sources, _query) do
       quote_name(literal)
     end
@@ -1038,6 +1042,25 @@ if Code.ensure_loaded?(Postgrex) do
       [?(, expr(expr, sources, query), "#>'{", path, "}')"]
     end
 
+    defp values_list(types, idx, num_rows) do
+      rows = Enum.to_list(1..num_rows)
+
+      [
+        "VALUES ",
+        intersperse_reduce(rows, ?,, idx, fn _, idx ->
+          {value, idx} = values_expr(types, idx)
+          {[?(, value, ?)], idx}
+        end)
+        |> elem(0)
+      ]
+    end
+
+    defp values_expr(types, idx) do
+      intersperse_reduce(types, ?,, idx, fn {_field, type}, idx ->
+        {[?$ , Integer.to_string(idx), ?:, ?: | tagged_to_db(type)], idx + 1}
+      end)
+    end
+
     defp type_unless_typed(%Ecto.Query.Tagged{}, _type), do: []
     defp type_unless_typed(_, type), do: [?:, ?: | type]
 
@@ -1102,6 +1125,9 @@ if Code.ensure_loaded?(Postgrex) do
         {:fragment, _, _} ->
           {nil, as_prefix ++ [?f | Integer.to_string(pos)], nil}
 
+        {:values, _, _}  ->
+          {nil, as_prefix ++ [?v | Integer.to_string(pos)], nil}
+
         {table, schema, prefix} ->
           name = as_prefix ++ [create_alias(table) | Integer.to_string(pos)]
           {quote_table(prefix, table), name, schema}
@@ -1702,6 +1728,7 @@ if Code.ensure_loaded?(Postgrex) do
 
     defp get_source(query, sources, ix, source) do
       {expr, name, _schema} = elem(sources, ix)
+      name = maybe_add_column_names(source, name)
       {expr || expr(source, sources, query), name}
     end
 
@@ -1712,6 +1739,13 @@ if Code.ensure_loaded?(Postgrex) do
       end
     end
 
+    defp maybe_add_column_names({:values, _, [types, _, _]}, name) do
+      fields = Keyword.keys(types)
+      [name, ?\s, ?(, quote_names(fields), ?)]
+    end
+
+    defp maybe_add_column_names(_, name), do: name
+
     defp quote_qualified_name(name, sources, ix) do
       {_, source, _} = elem(sources, ix)
       [source, ?. | quote_name(name)]

lib/ecto/adapters/tds/connection.ex

@@ -791,6 +791,10 @@ if Code.ensure_loaded?(Tds) do
       |> parens_for_select
     end
 
+    defp expr({:values, _, [types, idx, num_rows]}, _, _query) do
+      [?(, values_list(types, idx + 1, num_rows), ?)]
+    end
+
     defp expr({:literal, _, [literal]}, _sources, _query) do
       quote_name(literal)
     end
@@ -939,6 +943,25 @@ if Code.ensure_loaded?(Tds) do
       error!(query, "unsupported MSSQL expressions: `#{inspect(field)}`")
     end
 
+    defp values_list(types, idx, num_rows) do
+      rows = Enum.to_list(1..num_rows)
+
+      [
+        "VALUES ",
+        intersperse_reduce(rows, ?,, idx, fn _, idx ->
+          {value, idx} = values_expr(types, idx)
+          {[?(, value, ?)], idx}
+        end)
+        |> elem(0)
+      ]
+    end
+
+    defp values_expr(types, idx) do
+      intersperse_reduce(types, ?,, idx, fn {_field, type}, idx ->
+        {["CAST(", ?@ , Integer.to_string(idx), " AS ", column_type(type, []), ?)], idx + 1}
+      end)
+    end
+
     defp op_to_binary({op, _, [_, _]} = expr, sources, query) when op in @binary_ops do
       paren_expr(expr, sources, query)
     end
@@ -1002,6 +1025,9 @@ if Code.ensure_loaded?(Tds) do
         {:fragment, _, _} ->
           {nil, as_prefix ++ [?f | Integer.to_string(pos)], nil}
 
+        {:values, _, _}  ->
+          {nil, as_prefix ++ [?v | Integer.to_string(pos)], nil}
+
         {table, model, prefix} ->
           name = as_prefix ++ [create_alias(table) | Integer.to_string(pos)]
           {quote_table(prefix, table), name, model}
@@ -1563,6 +1589,7 @@ if Code.ensure_loaded?(Tds) do
 
     defp get_source(query, sources, ix, source) do
       {expr, name, _schema} = elem(sources, ix)
+      name = maybe_add_column_names(source, name)
       {expr || expr(source, sources, query), name}
     end
 
@@ -1573,6 +1600,13 @@ if Code.ensure_loaded?(Tds) do
       end
     end
 
+    defp maybe_add_column_names({:values, _, [types, _, _]}, name) do
+      fields = Keyword.keys(types)
+      [name, ?\s, ?(, quote_names(fields), ?)]
+    end
+
+    defp maybe_add_column_names(_, name), do: name
+
     defp quote_name(name) when is_atom(name) do
       quote_name(Atom.to_string(name))
     end

mix.lock

@@ -5,7 +5,7 @@
   "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
   "deep_merge": {:hex, :deep_merge, "0.2.0", "c1050fa2edf4848b9f556fba1b75afc66608a4219659e3311d9c9427b5b680b3", [:mix], [], "hexpm", "e3bf435a54ed27b0ba3a01eb117ae017988804e136edcbe8a6a14c310daa966e"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.33", "3c3fd9673bb5dcc9edc28dd90f50c87ce506d1f71b70e3de69aa8154bc695d44", [:mix], [], "hexpm", "2d526833729b59b9fdb85785078697c72ac5e5066350663e5be6a1182da61b8f"},
-  "ecto": {:git, "https://github.com/elixir-ecto/ecto.git", "eb03f45b999e2bf67ffae92811233eb6e56dba55", []},
+  "ecto": {:git, "https://github.com/elixir-ecto/ecto.git", "743ce048cfee927e20dba72f37d58eb52c3535a9", []},
   "ex_doc": {:hex, :ex_doc, "0.30.5", "aa6da96a5c23389d7dc7c381eba862710e108cee9cfdc629b7ec021313900e9e", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "88a1e115dcb91cefeef7e22df4a6ebbe4634fbf98b38adcbc25c9607d6d9d8e6"},
   "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"},
   "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"},

test/ecto/adapters/myxql_test.exs

@@ -1184,6 +1184,67 @@ defmodule Ecto.Adapters.MyXQLTest do
     assert query == ~s{DELETE FROM `schema` WHERE `x` IS NULL AND `y` = ?}
   end
 
+  # Values List
+
+  test "values list: all" do
+    uuid = Ecto.UUID.generate()
+    values = [%{bid: uuid, num: 1}, %{num: 2, bid: uuid}]
+    types = %{bid: Ecto.UUID, num: :integer}
+
+    query =
+      from(v1 in values(values, types),
+        join: v2 in values(values, types),
+        on: v1.bid == v2.bid,
+        select: v2,
+        where: v1.num == ^2
+      )
+      |> plan()
+      |> all()
+
+    assert query ==
+             ~s{SELECT v1.`bid`, v1.`num` } <>
+             ~s{FROM (VALUES ROW(CAST(? AS binary(16)),CAST(? AS unsigned)),ROW(CAST(? AS binary(16)),CAST(? AS unsigned))) AS v0 (`bid`,`num`) } <>
+             ~s{INNER JOIN (VALUES ROW(CAST(? AS binary(16)),CAST(? AS unsigned)),ROW(CAST(? AS binary(16)),CAST(? AS unsigned))) AS v1 (`bid`,`num`) ON v0.`bid` = v1.`bid` } <>
+             ~s{WHERE (v0.`num` = ?)}
+  end
+
+  test "values list: delete_all" do
+    uuid = Ecto.UUID.generate()
+    values = [%{bid: uuid, num: 1}, %{num: 2, bid: uuid}]
+    types = %{bid: Ecto.UUID, num: :integer}
+
+    query =
+      from(s in "schema", join: v in values(values, types), on: s.x == v.num, where: v.num == ^2)
+      |> plan(:delete_all)
+      |> delete_all()
+
+    assert query ==
+             ~s{DELETE s0.* FROM `schema` AS s0 } <>
+             ~s{INNER JOIN (VALUES ROW(CAST(? AS binary(16)),CAST(? AS unsigned)),ROW(CAST(? AS binary(16)),CAST(? AS unsigned))) AS v1 (`bid`,`num`) } <>
+             ~s{ON s0.`x` = v1.`num` WHERE (v1.`num` = ?)}
+  end
+
+  test "values list: update_all" do
+    uuid = Ecto.UUID.generate()
+    values = [%{bid: uuid, num: 1}, %{num: 2, bid: uuid}]
+    types = %{bid: Ecto.UUID, num: :integer}
+
+    query =
+      from(s in "schema",
+        join: v in values(values, types),
+        on: s.x == v.num,
+        where: v.num == ^2,
+        update: [set: [y: v.num]]
+      )
+      |> plan(:update_all)
+      |> update_all()
+
+    assert query ==
+             ~s{UPDATE `schema` AS s0, } <>
+               ~s{(VALUES ROW(CAST(? AS binary(16)),CAST(? AS unsigned)),ROW(CAST(? AS binary(16)),CAST(? AS unsigned))) AS v1 (`bid`,`num`) } <>
+               ~s{SET s0.`y` = v1.`num` WHERE (s0.`x` = v1.`num`) AND (v1.`num` = ?)}
+  end
+
   # DDL
 
   import Ecto.Migration, only: [table: 1, table: 2, index: 2, index: 3,

test/ecto/adapters/postgres_test.exs

@@ -1485,6 +1485,67 @@ defmodule Ecto.Adapters.PostgresTest do
     assert query == ~s{DELETE FROM "prefix"."schema" WHERE "x" IS NULL AND "y" = $1}
   end
 
+  # Values List
+
+  test "values list: all" do
+    uuid = Ecto.UUID.generate()
+    values = [%{bid: uuid, num: 1}, %{num: 2, bid: uuid}]
+    types = %{bid: Ecto.UUID, num: :integer}
+
+    query =
+      from(v1 in values(values, types),
+        join: v2 in values(values, types),
+        on: v1.bid == v2.bid,
+        select: v2,
+        where: v1.num == ^2
+      )
+      |> plan()
+      |> all()
+
+    assert query ==
+             ~s{SELECT v1."bid", v1."num" } <>
+             ~s{FROM (VALUES ($1::uuid,$2::bigint),($3::uuid,$4::bigint)) AS v0 ("bid","num") } <>
+             ~s{INNER JOIN (VALUES ($5::uuid,$6::bigint),($7::uuid,$8::bigint)) AS v1 ("bid","num") ON v0."bid" = v1."bid" } <>
+             ~s{WHERE (v0."num" = $9)}
+  end
+
+  test "values list: delete_all" do
+    uuid = Ecto.UUID.generate()
+    values = [%{bid: uuid, num: 1}, %{num: 2, bid: uuid}]
+    types = %{bid: Ecto.UUID, num: :integer}
+
+    query =
+      from(s in "schema", join: v in values(values, types), on: s.x == v.num, where: v.num == ^2)
+      |> plan(:delete_all)
+      |> delete_all()
+
+    assert query ==
+             ~s{DELETE FROM "schema" AS s0 } <>
+             ~s{USING (VALUES ($1::uuid,$2::bigint),($3::uuid,$4::bigint)) AS v1 ("bid","num") } <>
+             ~s{WHERE (s0."x" = v1."num") AND (v1."num" = $5)}
+  end
+
+  test "values list: update_all" do
+    uuid = Ecto.UUID.generate()
+    values = [%{bid: uuid, num: 1}, %{num: 2, bid: uuid}]
+    types = %{bid: Ecto.UUID, num: :integer}
+
+    query =
+      from(s in "schema",
+        join: v in values(values, types),
+        on: s.x == v.num,
+        where: v.num == ^2,
+        update: [set: [y: v.num]]
+      )
+      |> plan(:update_all)
+      |> update_all()
+
+    assert query ==
+             ~s{UPDATE "schema" AS s0 SET "y" = v1."num" } <>
+               ~s{FROM (VALUES ($1::uuid,$2::bigint),($3::uuid,$4::bigint)) AS v1 ("bid","num") } <>
+               ~s{WHERE (s0."x" = v1."num") AND (v1."num" = $5)}
+  end
+
   # DDL
 
   alias Ecto.Migration.Reference