From 2ab86081667d879574d7be11c6aa2750f908cd6d Mon Sep 17 00:00:00 2001 From: hawkyre Date: Fri, 11 Jul 2025 12:51:10 +0200 Subject: [PATCH 01/34] multipart requests --- lib/ch/connection.ex | 35 +++++++++++++++++++++++++++++++++-- mix.exs | 1 + mix.lock | 2 ++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/lib/ch/connection.ex b/lib/ch/connection.ex index 40970ae..9e247c4 100644 --- a/lib/ch/connection.ex +++ b/lib/ch/connection.ex @@ -2,6 +2,7 @@ defmodule Ch.Connection do @moduledoc false use DBConnection require Logger + alias Multipart.Part alias Ch.{Error, Query, Result} alias Mint.HTTP1, as: HTTP @@ -214,7 +215,9 @@ defmodule Ch.Connection do def handle_execute(%Query{command: :insert} = query, params, opts, conn) do conn = maybe_reconnect(conn) - {query_params, extra_headers, body} = params + + multipart_request = Keyword.get(opts, :multipart_request, true) + {query_params, extra_headers, body} = parse_params(params, multipart_request) path = path(conn, query_params, opts) headers = headers(conn, extra_headers, opts) @@ -233,7 +236,9 @@ defmodule Ch.Connection do def handle_execute(query, params, opts, conn) do conn = maybe_reconnect(conn) - {query_params, extra_headers, body} = params + + multipart_request = Keyword.get(opts, :multipart_request, true) + {query_params, extra_headers, body} = parse_params(params, multipart_request) path = path(conn, query_params, opts) headers = headers(conn, extra_headers, opts) @@ -379,6 +384,32 @@ defmodule Ch.Connection do end end + @spec parse_params(tuple, boolean) :: tuple + defp parse_params({query_params, headers, body}, true = _multipart?) when is_binary(body) do + body = to_string(body) + + multipart = + query_params + |> Enum.reduce(Multipart.new(), fn {k, v}, acc -> + Multipart.add_part(acc, Part.text_field(v, k)) + end) + |> Multipart.add_part(Part.text_field(body, "query")) + + content_length = Multipart.content_length(multipart) + content_type = Multipart.content_type(multipart, "multipart/form-data") + + multipart_headers = [ + {"Content-Type", content_type}, + {"Content-Length", to_string(content_length)} + ] + + {[], headers ++ multipart_headers, Multipart.body_binary(multipart)} + end + + defp parse_params(params, _) do + params + end + defp get_header(headers, key) do case List.keyfind(headers, key, 0) do {_, value} -> value diff --git a/mix.exs b/mix.exs index dd11049..c8f704e 100644 --- a/mix.exs +++ b/mix.exs @@ -41,6 +41,7 @@ defmodule Ch.MixProject do {:db_connection, "~> 2.0"}, {:jason, "~> 1.0"}, {:decimal, "~> 2.0"}, + {:multipart, "~> 0.4.0"}, {:ecto, "~> 3.13.0", optional: true}, {:benchee, "~> 1.0", only: [:bench]}, {:dialyxir, "~> 1.0", only: [:dev], runtime: false}, diff --git a/mix.lock b/mix.lock index 0e57a37..94448ac 100644 --- a/mix.lock +++ b/mix.lock @@ -13,7 +13,9 @@ "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, + "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, + "multipart": {:hex, :multipart, "0.4.0", "634880a2148d4555d050963373d0e3bbb44a55b2badd87fa8623166172e9cda0", [:mix], [{:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}], "hexpm", "3c5604bc2fb17b3137e5d2abdf5dacc2647e60c5cc6634b102cf1aef75a06f0a"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "statistex": {:hex, :statistex, "1.1.0", "7fec1eb2f580a0d2c1a05ed27396a084ab064a40cfc84246dbfb0c72a5c761e5", [:mix], [], "hexpm", "f5950ea26ad43246ba2cce54324ac394a4e7408fdcf98b8e230f503a0cba9cf5"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, From 76274b77363f1414033d4b5eb9fef57d3480ad3b Mon Sep 17 00:00:00 2001 From: hawkyre Date: Fri, 11 Jul 2025 12:52:54 +0200 Subject: [PATCH 02/34] to false --- lib/ch/connection.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ch/connection.ex b/lib/ch/connection.ex index 9e247c4..46c3600 100644 --- a/lib/ch/connection.ex +++ b/lib/ch/connection.ex @@ -216,7 +216,7 @@ defmodule Ch.Connection do def handle_execute(%Query{command: :insert} = query, params, opts, conn) do conn = maybe_reconnect(conn) - multipart_request = Keyword.get(opts, :multipart_request, true) + multipart_request = Keyword.get(opts, :multipart_request, false) {query_params, extra_headers, body} = parse_params(params, multipart_request) path = path(conn, query_params, opts) @@ -237,7 +237,7 @@ defmodule Ch.Connection do def handle_execute(query, params, opts, conn) do conn = maybe_reconnect(conn) - multipart_request = Keyword.get(opts, :multipart_request, true) + multipart_request = Keyword.get(opts, :multipart_request, false) {query_params, extra_headers, body} = parse_params(params, multipart_request) path = path(conn, query_params, opts) From 828133c5fabfad10e13ccec9dde49b4830c10519 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Fri, 11 Jul 2025 13:11:05 +0200 Subject: [PATCH 03/34] update docs and interface --- README.md | 20 ++++++++++++++++++++ lib/ch.ex | 1 + lib/ch/connection.ex | 8 ++++---- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 22de38c..6222ed5 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,26 @@ settings = [async_insert: 1] Ch.query!(pid, "SHOW SETTINGS LIKE 'async_insert'", [], settings: settings) ``` +#### Sending request as multipart + +```elixir +{:ok, pid} = Ch.start_link() + +Ch.query!(pid, "CREATE TABLE IF NOT EXISTS ch_demo(id UInt64) ENGINE Null") + +{:ok, %Ch.Result{rows: [[0], [1], [2]]}} = + Ch.query(pid, "SELECT * FROM system.numbers LIMIT {$0:UInt8}", [3], multipart: true) + +{:ok, %Ch.Result{rows: [[0], [1], [2]]}} = + Ch.query(pid, "SELECT * FROM system.numbers LIMIT {limit:UInt8}", %{"limit" => 3}, multipart: true) + +%Ch.Result{num_rows: 2} = + Ch.query!(pid, "INSERT INTO ch_demo(id) VALUES ({$0:UInt8}), ({$1:UInt32})", [0, 1], multipart: true) +``` + +Adding the `multipart: true` option to a query sends its Clickhouse request as a multipart request. +This encodes all the request data inside the body, and leaves the URL empty. + ## Caveats #### NULL in RowBinary diff --git a/lib/ch.ex b/lib/ch.ex index 52a5f6b..758627f 100644 --- a/lib/ch.ex +++ b/lib/ch.ex @@ -76,6 +76,7 @@ defmodule Ch do * `:headers` - Custom HTTP headers for the request * `:format` - Custom response format for the request * `:decode` - Whether to automatically decode the response + * `:multipart` - Whether to use multipart/form-data encoding for the request. Not supported for RowBinary inserts or streaming. * [`DBConnection.connection_option()`](https://hexdocs.pm/db_connection/DBConnection.html#t:connection_option/0) """ diff --git a/lib/ch/connection.ex b/lib/ch/connection.ex index 46c3600..351a5d7 100644 --- a/lib/ch/connection.ex +++ b/lib/ch/connection.ex @@ -216,8 +216,8 @@ defmodule Ch.Connection do def handle_execute(%Query{command: :insert} = query, params, opts, conn) do conn = maybe_reconnect(conn) - multipart_request = Keyword.get(opts, :multipart_request, false) - {query_params, extra_headers, body} = parse_params(params, multipart_request) + multipart = Keyword.get(opts, :multipart, false) + {query_params, extra_headers, body} = parse_params(params, multipart) path = path(conn, query_params, opts) headers = headers(conn, extra_headers, opts) @@ -237,8 +237,8 @@ defmodule Ch.Connection do def handle_execute(query, params, opts, conn) do conn = maybe_reconnect(conn) - multipart_request = Keyword.get(opts, :multipart_request, false) - {query_params, extra_headers, body} = parse_params(params, multipart_request) + multipart = Keyword.get(opts, :multipart, false) + {query_params, extra_headers, body} = parse_params(params, multipart) path = path(conn, query_params, opts) headers = headers(conn, extra_headers, opts) From 23db2b9056f7c4a590c92697fa8cbf3473f23286 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Fri, 11 Jul 2025 13:13:16 +0200 Subject: [PATCH 04/34] remove to string --- lib/ch/connection.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/ch/connection.ex b/lib/ch/connection.ex index 351a5d7..76679f5 100644 --- a/lib/ch/connection.ex +++ b/lib/ch/connection.ex @@ -386,8 +386,6 @@ defmodule Ch.Connection do @spec parse_params(tuple, boolean) :: tuple defp parse_params({query_params, headers, body}, true = _multipart?) when is_binary(body) do - body = to_string(body) - multipart = query_params |> Enum.reduce(Multipart.new(), fn {k, v}, acc -> From 66020ff9cf1df99bce5a45b4437f41ef0bcb13ac Mon Sep 17 00:00:00 2001 From: hawkyre Date: Tue, 29 Jul 2025 09:49:24 +0200 Subject: [PATCH 05/34] custom multipart and slight refactor --- lib/ch/connection.ex | 33 +---------- lib/ch/encode/multipart.ex | 86 ++++++++++++++++++++++++++++ lib/ch/encode/parameters.ex | 109 ++++++++++++++++++++++++++++++++++++ lib/ch/query.ex | 104 +--------------------------------- mix.exs | 1 - 5 files changed, 200 insertions(+), 133 deletions(-) create mode 100644 lib/ch/encode/multipart.ex create mode 100644 lib/ch/encode/parameters.ex diff --git a/lib/ch/connection.ex b/lib/ch/connection.ex index 76679f5..40970ae 100644 --- a/lib/ch/connection.ex +++ b/lib/ch/connection.ex @@ -2,7 +2,6 @@ defmodule Ch.Connection do @moduledoc false use DBConnection require Logger - alias Multipart.Part alias Ch.{Error, Query, Result} alias Mint.HTTP1, as: HTTP @@ -215,9 +214,7 @@ defmodule Ch.Connection do def handle_execute(%Query{command: :insert} = query, params, opts, conn) do conn = maybe_reconnect(conn) - - multipart = Keyword.get(opts, :multipart, false) - {query_params, extra_headers, body} = parse_params(params, multipart) + {query_params, extra_headers, body} = params path = path(conn, query_params, opts) headers = headers(conn, extra_headers, opts) @@ -236,9 +233,7 @@ defmodule Ch.Connection do def handle_execute(query, params, opts, conn) do conn = maybe_reconnect(conn) - - multipart = Keyword.get(opts, :multipart, false) - {query_params, extra_headers, body} = parse_params(params, multipart) + {query_params, extra_headers, body} = params path = path(conn, query_params, opts) headers = headers(conn, extra_headers, opts) @@ -384,30 +379,6 @@ defmodule Ch.Connection do end end - @spec parse_params(tuple, boolean) :: tuple - defp parse_params({query_params, headers, body}, true = _multipart?) when is_binary(body) do - multipart = - query_params - |> Enum.reduce(Multipart.new(), fn {k, v}, acc -> - Multipart.add_part(acc, Part.text_field(v, k)) - end) - |> Multipart.add_part(Part.text_field(body, "query")) - - content_length = Multipart.content_length(multipart) - content_type = Multipart.content_type(multipart, "multipart/form-data") - - multipart_headers = [ - {"Content-Type", content_type}, - {"Content-Length", to_string(content_length)} - ] - - {[], headers ++ multipart_headers, Multipart.body_binary(multipart)} - end - - defp parse_params(params, _) do - params - end - defp get_header(headers, key) do case List.keyfind(headers, key, 0) do {_, value} -> value diff --git a/lib/ch/encode/multipart.ex b/lib/ch/encode/multipart.ex new file mode 100644 index 0000000..25fa42e --- /dev/null +++ b/lib/ch/encode/multipart.ex @@ -0,0 +1,86 @@ +defmodule Ch.Encode.Multipart do + @moduledoc false + + alias Ch.Encode.Parameters + + @doc """ + Encodes a query statement and params into a multipart request. + """ + @spec encode(iodata, map, [Ch.query_option()]) :: + {list, Mint.Types.headers(), iodata} + def encode(statement, params, opts) do + types = Keyword.get(opts, :types) + default_format = if types, do: "RowBinary", else: "RowBinaryWithNamesAndTypes" + format = Keyword.get(opts, :format) || default_format + + boundary = "ChFormBoundary" <> Base.url_encode64(:crypto.strong_rand_bytes(24)) + content_type = "multipart/form-data; boundary=\"#{boundary}\"" + enc_boundary = "--#{boundary}\r\n" + + multipart = + params + |> multipart_params(enc_boundary) + |> add_multipart_part("query", statement, enc_boundary) + |> then(&[&1 | "--#{boundary}--\r\n"]) + + headers = [{"x-clickhouse-format", format}, {"content-type", content_type} | headers(opts)] + + {_no_query_params = [], headers, multipart} + end + + defp multipart_params(params, boundary) when is_map(params) do + multipart_named_params(Map.to_list(params), boundary, []) + end + + defp multipart_params(params, boundary) when is_list(params) do + multipart_positional_params(params, 0, boundary, []) + end + + defp multipart_named_params([{name, value} | params], boundary, acc) do + acc = + add_multipart_part( + acc, + "param_" <> URI.encode_www_form(name), + Parameters.encode(value), + boundary + ) + + multipart_named_params(params, boundary, acc) + end + + defp multipart_named_params([], _boundary, acc), do: acc + + defp multipart_positional_params([value | params], idx, boundary, acc) do + acc = + add_multipart_part( + acc, + "param_$" <> Integer.to_string(idx), + Parameters.encode(value), + boundary + ) + + multipart_positional_params(params, idx + 1, boundary, acc) + end + + defp multipart_positional_params([], _idx, _boundary, acc), do: acc + + @compile inline: [add_multipart_part: 4] + defp add_multipart_part(multipart, name, value, boundary) do + part = [ + boundary, + "content-disposition: form-data; name=\"", + name, + "\"\r\n\r\n", + value, + "\r\n" + ] + + case multipart do + [] -> part + _ -> [multipart | part] + end + end + + @spec headers(Keyword.t()) :: Mint.Types.headers() + defp headers(opts), do: Keyword.get(opts, :headers, []) +end diff --git a/lib/ch/encode/parameters.ex b/lib/ch/encode/parameters.ex new file mode 100644 index 0000000..818f4fb --- /dev/null +++ b/lib/ch/encode/parameters.ex @@ -0,0 +1,109 @@ +defmodule Ch.Encode.Parameters do + @moduledoc false + + @doc """ + Encodes a map/list of parameters into a list of clickhouse parameter tuples. + + The format is `[{"param_", ""}, ...]`. + """ + @spec encode_many(map | [term]) :: [{String.t(), String.t()}] + def encode_many(params) when is_map(params) do + Enum.map(params, fn {k, v} -> {"param_#{k}", encode(v)} end) + end + + def encode_many(params) when is_list(params) do + params + |> Enum.with_index() + |> Enum.map(fn {v, idx} -> {"param_$#{idx}", encode(v)} end) + end + + @doc """ + Encodes a clickhouse parameter to a string. + """ + @spec encode(term) :: binary + def encode(n) when is_integer(n), do: Integer.to_string(n) + def encode(f) when is_float(f), do: Float.to_string(f) + + # TODO possibly speed up + # For more info see + # https://clickhouse.com/docs/en/interfaces/http#tabs-in-url-parameters + # "escaped" format is the same as https://clickhouse.com/docs/en/interfaces/formats#tabseparated-data-formatting + def encode(b) when is_binary(b) do + escape_param([{"\\", "\\\\"}, {"\t", "\\\t"}, {"\n", "\\\n"}], b) + end + + def encode(b) when is_boolean(b), do: Atom.to_string(b) + def encode(%Decimal{} = d), do: Decimal.to_string(d, :normal) + def encode(%Date{} = date), do: Date.to_iso8601(date) + def encode(%NaiveDateTime{} = naive), do: NaiveDateTime.to_iso8601(naive) + + def encode(%DateTime{microsecond: microsecond} = dt) do + dt = DateTime.shift_zone!(dt, "Etc/UTC") + + case microsecond do + {val, precision} when val > 0 and precision > 0 -> + size = round(:math.pow(10, precision)) + unix = DateTime.to_unix(dt, size) + seconds = div(unix, size) + fractional = rem(unix, size) + + IO.iodata_to_binary([ + Integer.to_string(seconds), + ?., + String.pad_leading(Integer.to_string(fractional), precision, "0") + ]) + + _ -> + dt |> DateTime.to_unix(:second) |> Integer.to_string() + end + end + + def encode(tuple) when is_tuple(tuple) do + IO.iodata_to_binary([?(, encode_array_params(Tuple.to_list(tuple)), ?)]) + end + + def encode(a) when is_list(a) do + IO.iodata_to_binary([?[, encode_array_params(a), ?]]) + end + + def encode(m) when is_map(m) do + IO.iodata_to_binary([?{, encode_map_params(Map.to_list(m)), ?}]) + end + + defp encode_array_params([last]), do: encode_array_param(last) + + defp encode_array_params([s | rest]) do + [encode_array_param(s), ?, | encode_array_params(rest)] + end + + defp encode_array_params([] = empty), do: empty + + defp encode_map_params([last]), do: encode_map_param(last) + + defp encode_map_params([kv | rest]) do + [encode_map_param(kv), ?, | encode_map_params(rest)] + end + + defp encode_map_params([] = empty), do: empty + + defp encode_array_param(s) when is_binary(s) do + [?', escape_param([{"'", "''"}, {"\\", "\\\\"}], s), ?'] + end + + defp encode_array_param(%s{} = param) when s in [Date, NaiveDateTime] do + [?', encode(param), ?'] + end + + defp encode_array_param(v), do: encode(v) + + defp encode_map_param({k, v}) do + [encode_array_param(k), ?:, encode_array_param(v)] + end + + defp escape_param([{pattern, replacement} | escapes], param) do + param = String.replace(param, pattern, replacement) + escape_param(escapes, param) + end + + defp escape_param([], param), do: param +end diff --git a/lib/ch/query.ex b/lib/ch/query.ex index f5a9805..c1d9159 100644 --- a/lib/ch/query.ex +++ b/lib/ch/query.ex @@ -72,6 +72,7 @@ end defimpl DBConnection.Query, for: Ch.Query do alias Ch.{Query, Result, RowBinary} + alias Ch.Encode.{Multipart, Parameters} @spec parse(Query.t(), [Ch.query_option()]) :: Query.t() def parse(query, _opts), do: query @@ -123,15 +124,12 @@ defimpl DBConnection.Query, for: Ch.Query do {_query_params = [], headers(opts), [statement, ?\n | data]} true -> - {query_params(params), headers(opts), statement} + {Parameters.encode_many(params), headers(opts), statement} end end def encode(%Query{statement: statement}, params, opts) do - types = Keyword.get(opts, :types) - default_format = if types, do: "RowBinary", else: "RowBinaryWithNamesAndTypes" - format = Keyword.get(opts, :format) || default_format - {query_params(params), [{"x-clickhouse-format", format} | headers(opts)], statement} + Multipart.encode(statement, params, opts) end defp format_row_binary?(statement) when is_binary(statement) do @@ -207,102 +205,6 @@ defimpl DBConnection.Query, for: Ch.Query do end end - defp query_params(params) when is_map(params) do - Enum.map(params, fn {k, v} -> {"param_#{k}", encode_param(v)} end) - end - - defp query_params(params) when is_list(params) do - params - |> Enum.with_index() - |> Enum.map(fn {v, idx} -> {"param_$#{idx}", encode_param(v)} end) - end - - defp encode_param(n) when is_integer(n), do: Integer.to_string(n) - defp encode_param(f) when is_float(f), do: Float.to_string(f) - - # TODO possibly speed up - # For more info see - # https://clickhouse.com/docs/en/interfaces/http#tabs-in-url-parameters - # "escaped" format is the same as https://clickhouse.com/docs/en/interfaces/formats#tabseparated-data-formatting - defp encode_param(b) when is_binary(b) do - escape_param([{"\\", "\\\\"}, {"\t", "\\\t"}, {"\n", "\\\n"}], b) - end - - defp encode_param(b) when is_boolean(b), do: Atom.to_string(b) - defp encode_param(%Decimal{} = d), do: Decimal.to_string(d, :normal) - defp encode_param(%Date{} = date), do: Date.to_iso8601(date) - defp encode_param(%NaiveDateTime{} = naive), do: NaiveDateTime.to_iso8601(naive) - - defp encode_param(%DateTime{microsecond: microsecond} = dt) do - dt = DateTime.shift_zone!(dt, "Etc/UTC") - - case microsecond do - {val, precision} when val > 0 and precision > 0 -> - size = round(:math.pow(10, precision)) - unix = DateTime.to_unix(dt, size) - seconds = div(unix, size) - fractional = rem(unix, size) - - IO.iodata_to_binary([ - Integer.to_string(seconds), - ?., - String.pad_leading(Integer.to_string(fractional), precision, "0") - ]) - - _ -> - dt |> DateTime.to_unix(:second) |> Integer.to_string() - end - end - - defp encode_param(tuple) when is_tuple(tuple) do - IO.iodata_to_binary([?(, encode_array_params(Tuple.to_list(tuple)), ?)]) - end - - defp encode_param(a) when is_list(a) do - IO.iodata_to_binary([?[, encode_array_params(a), ?]]) - end - - defp encode_param(m) when is_map(m) do - IO.iodata_to_binary([?{, encode_map_params(Map.to_list(m)), ?}]) - end - - defp encode_array_params([last]), do: encode_array_param(last) - - defp encode_array_params([s | rest]) do - [encode_array_param(s), ?, | encode_array_params(rest)] - end - - defp encode_array_params([] = empty), do: empty - - defp encode_map_params([last]), do: encode_map_param(last) - - defp encode_map_params([kv | rest]) do - [encode_map_param(kv), ?, | encode_map_params(rest)] - end - - defp encode_map_params([] = empty), do: empty - - defp encode_array_param(s) when is_binary(s) do - [?', escape_param([{"'", "''"}, {"\\", "\\\\"}], s), ?'] - end - - defp encode_array_param(%s{} = param) when s in [Date, NaiveDateTime] do - [?', encode_param(param), ?'] - end - - defp encode_array_param(v), do: encode_param(v) - - defp encode_map_param({k, v}) do - [encode_array_param(k), ?:, encode_array_param(v)] - end - - defp escape_param([{pattern, replacement} | escapes], param) do - param = String.replace(param, pattern, replacement) - escape_param(escapes, param) - end - - defp escape_param([], param), do: param - @spec headers(Keyword.t()) :: Mint.Types.headers() defp headers(opts), do: Keyword.get(opts, :headers, []) end diff --git a/mix.exs b/mix.exs index c8f704e..dd11049 100644 --- a/mix.exs +++ b/mix.exs @@ -41,7 +41,6 @@ defmodule Ch.MixProject do {:db_connection, "~> 2.0"}, {:jason, "~> 1.0"}, {:decimal, "~> 2.0"}, - {:multipart, "~> 0.4.0"}, {:ecto, "~> 3.13.0", optional: true}, {:benchee, "~> 1.0", only: [:bench]}, {:dialyxir, "~> 1.0", only: [:dev], runtime: false}, From 482a42fd9ef538462c4e0ee0323fea9a4fa54068 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Tue, 29 Jul 2025 09:53:24 +0200 Subject: [PATCH 06/34] merge --- lib/ch/encode/parameters.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/ch/encode/parameters.ex b/lib/ch/encode/parameters.ex index 818f4fb..0ac0828 100644 --- a/lib/ch/encode/parameters.ex +++ b/lib/ch/encode/parameters.ex @@ -33,9 +33,11 @@ defmodule Ch.Encode.Parameters do end def encode(b) when is_boolean(b), do: Atom.to_string(b) + def encode(nil), do: "\\N" def encode(%Decimal{} = d), do: Decimal.to_string(d, :normal) def encode(%Date{} = date), do: Date.to_iso8601(date) def encode(%NaiveDateTime{} = naive), do: NaiveDateTime.to_iso8601(naive) + def encode(%Time{} = time), do: Time.to_iso8601(time) def encode(%DateTime{microsecond: microsecond} = dt) do dt = DateTime.shift_zone!(dt, "Etc/UTC") @@ -90,6 +92,8 @@ defmodule Ch.Encode.Parameters do [?', escape_param([{"'", "''"}, {"\\", "\\\\"}], s), ?'] end + defp encode_array_param(nil), do: "null" + defp encode_array_param(%s{} = param) when s in [Date, NaiveDateTime] do [?', encode(param), ?'] end From 5191d07e060c2b0bd6f0a8925f6b413fa9467953 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Tue, 29 Jul 2025 10:01:05 +0200 Subject: [PATCH 07/34] docs --- README.md | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index f044dfe..0f0b1f5 100644 --- a/README.md +++ b/README.md @@ -163,23 +163,8 @@ settings = [async_insert: 1] #### Sending request as multipart -```elixir -{:ok, pid} = Ch.start_link() - -Ch.query!(pid, "CREATE TABLE IF NOT EXISTS ch_demo(id UInt64) ENGINE Null") - -{:ok, %Ch.Result{rows: [[0], [1], [2]]}} = - Ch.query(pid, "SELECT * FROM system.numbers LIMIT {$0:UInt8}", [3], multipart: true) - -{:ok, %Ch.Result{rows: [[0], [1], [2]]}} = - Ch.query(pid, "SELECT * FROM system.numbers LIMIT {limit:UInt8}", %{"limit" => 3}, multipart: true) - -%Ch.Result{num_rows: 2} = - Ch.query!(pid, "INSERT INTO ch_demo(id) VALUES ({$0:UInt8}), ({$1:UInt32})", [0, 1], multipart: true) -``` - -Adding the `multipart: true` option to a query sends its Clickhouse request as a multipart request. -This encodes all the request data inside the body, and leaves the URL empty. +SELECT queries will be automatically sent as multipart requests. +INSERT queries and streams are treated normally. ## Caveats From 22207b1a9f0a4b3f45ed20407615a87c524a4154 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Tue, 29 Jul 2025 10:03:14 +0200 Subject: [PATCH 08/34] doc title --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0f0b1f5..0bece3e 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ settings = [async_insert: 1] Ch.query!(pid, "SHOW SETTINGS LIKE 'async_insert'", [], settings: settings) ``` -#### Sending request as multipart +#### Multipart requests SELECT queries will be automatically sent as multipart requests. INSERT queries and streams are treated normally. From d21d7abf2a7280cc2526d56c1d0df92ba5c738e0 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Tue, 29 Jul 2025 10:04:55 +0200 Subject: [PATCH 09/34] more cleanup --- lib/ch.ex | 1 - mix.lock | 2 -- 2 files changed, 3 deletions(-) diff --git a/lib/ch.ex b/lib/ch.ex index 7346345..8b12e2c 100644 --- a/lib/ch.ex +++ b/lib/ch.ex @@ -76,7 +76,6 @@ defmodule Ch do * `:headers` - Custom HTTP headers for the request * `:format` - Custom response format for the request * `:decode` - Whether to automatically decode the response - * `:multipart` - Whether to use multipart/form-data encoding for the request. Not supported for RowBinary inserts or streaming. * [`DBConnection.connection_option()`](https://hexdocs.pm/db_connection/DBConnection.html#t:connection_option/0) """ diff --git a/mix.lock b/mix.lock index 94448ac..0e57a37 100644 --- a/mix.lock +++ b/mix.lock @@ -13,9 +13,7 @@ "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, - "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, - "multipart": {:hex, :multipart, "0.4.0", "634880a2148d4555d050963373d0e3bbb44a55b2badd87fa8623166172e9cda0", [:mix], [{:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}], "hexpm", "3c5604bc2fb17b3137e5d2abdf5dacc2647e60c5cc6634b102cf1aef75a06f0a"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "statistex": {:hex, :statistex, "1.1.0", "7fec1eb2f580a0d2c1a05ed27396a084ab064a40cfc84246dbfb0c72a5c761e5", [:mix], [], "hexpm", "f5950ea26ad43246ba2cce54324ac394a4e7408fdcf98b8e230f503a0cba9cf5"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, From 3bee9258c615827b3d64d670ee5fed0e8047b2c5 Mon Sep 17 00:00:00 2001 From: hawkyre Date: Mon, 18 Aug 2025 15:03:07 +0200 Subject: [PATCH 10/34] send settings as params --- lib/ch/encode/multipart.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ch/encode/multipart.ex b/lib/ch/encode/multipart.ex index 25fa42e..c058fde 100644 --- a/lib/ch/encode/multipart.ex +++ b/lib/ch/encode/multipart.ex @@ -10,6 +10,7 @@ defmodule Ch.Encode.Multipart do {list, Mint.Types.headers(), iodata} def encode(statement, params, opts) do types = Keyword.get(opts, :types) + settings = Keyword.get(opts, :settings, []) default_format = if types, do: "RowBinary", else: "RowBinaryWithNamesAndTypes" format = Keyword.get(opts, :format) || default_format @@ -25,7 +26,7 @@ defmodule Ch.Encode.Multipart do headers = [{"x-clickhouse-format", format}, {"content-type", content_type} | headers(opts)] - {_no_query_params = [], headers, multipart} + {settings, headers, multipart} end defp multipart_params(params, boundary) when is_map(params) do From 33ecfaf1fee0a55698c335257c9ce5d01a9d0414 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Aug 2025 08:39:22 +0900 Subject: [PATCH 11/34] Bump actions/checkout from 4 to 5 (#270) Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/bench.yml | 2 +- .github/workflows/spellcheck.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index aa08d9e..a9d40fa 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -27,7 +27,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - id: beam uses: erlef/setup-beam@v1 diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index ff0ab78..3d39fe8 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,7 +9,7 @@ jobs: codespell: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: codespell-project/actions-codespell@v2 with: check_filenames: true @@ -18,5 +18,5 @@ jobs: typos: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: crate-ci/typos@master diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 06b25d0..05b2a6a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,7 +62,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - id: beam uses: erlef/setup-beam@v1 @@ -87,7 +87,7 @@ jobs: format: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: erlef/setup-beam@v1 with: elixir-version: 1 From 574c14bfcc540a398183b5e43cac7a3ffdbf88ab Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:22:38 +0900 Subject: [PATCH 12/34] fix version check (#274) --- lib/ch/connection.ex | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/ch/connection.ex b/lib/ch/connection.ex index f94b50a..b84846a 100644 --- a/lib/ch/connection.ex +++ b/lib/ch/connection.ex @@ -21,7 +21,7 @@ defmodule Ch.Connection do case DBConnection.Query.decode(handshake, responses, _opts = []) do %Result{rows: [[1, version]]} -> conn = - if version >= "24.10" do + if parse_version(version) >= parse_version("24.10") do settings = HTTP.get_private(conn, :settings, []) |> Keyword.put_new(:input_format_binary_read_json_as_string, 1) @@ -51,6 +51,17 @@ defmodule Ch.Connection do end end + defp parse_version(version) do + version + |> String.split(".") + |> Enum.flat_map(fn segment -> + case Integer.parse(segment) do + {int, _rest} -> [int] + :error -> [] + end + end) + end + @impl true @spec ping(conn) :: {:ok, conn} | {:disconnect, Mint.Types.error() | Error.t(), conn} def ping(conn) do From a018df865801eecb288773ee9ee222e76efab0db Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:23:35 +0900 Subject: [PATCH 13/34] changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f48d1a..6845417 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- fix version check for adding JSON settings https://github.com/plausible/ch/pull/274 + ## 0.5.4 (2025-07-22) - allow `nil` in params https://github.com/plausible/ch/pull/268 From ecc8a19f7002125548cee5942796d051e2e14551 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:24:25 +0900 Subject: [PATCH 14/34] add older ClickHouse to CI --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05b2a6a..66f32da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,6 +45,9 @@ jobs: otp: 27.3.1 clickhouse: 24.12.2.29 timezone: UTC + - elixir: 1.18 + otp: 28 + clickhouse: 24.5.4.49 services: clickhouse: From cac0abd2c0ff245fb20079b70c84453c15a36399 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:27:06 +0900 Subject: [PATCH 15/34] tag 'json as string' test as json --- test/ch/connection_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/ch/connection_test.exs b/test/ch/connection_test.exs index b9003ed..394dcb6 100644 --- a/test/ch/connection_test.exs +++ b/test/ch/connection_test.exs @@ -593,6 +593,7 @@ defmodule Ch.ConnectionTest do assert_raise ArgumentError, fn -> Ch.query!(conn, "SELECT o FROM json") end end + @tag :json test "json as string", %{conn: conn} do # after v25 ClickHouse started rendering numbers in JSON as strings [[version]] = Ch.query!(conn, "select version()").rows From a59d141a1ff1ff468188e2335e1ad84b929d2e8a Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:29:49 +0900 Subject: [PATCH 16/34] release v0.5.5 --- CHANGELOG.md | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6845417..88b6a72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 0.5.5 (2025-08-26) - fix version check for adding JSON settings https://github.com/plausible/ch/pull/274 diff --git a/mix.exs b/mix.exs index 48ff301..e0dc05c 100644 --- a/mix.exs +++ b/mix.exs @@ -2,7 +2,7 @@ defmodule Ch.MixProject do use Mix.Project @source_url "https://github.com/plausible/ch" - @version "0.5.4" + @version "0.5.5" def project do [ From e0ccd54e435028314f17d25aea1b2bc5eb48c111 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:30:08 +0900 Subject: [PATCH 17/34] update deps --- mix.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.lock b/mix.lock index 0e57a37..4abb55a 100644 --- a/mix.lock +++ b/mix.lock @@ -3,11 +3,11 @@ "db_connection": {:hex, :db_connection, "2.8.0", "64fd82cfa6d8e25ec6660cea73e92a4cbc6a18b31343910427b702838c4b33b2", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "008399dae5eee1bf5caa6e86d204dcb44242c82b1ed5e22c881f2c34da201b15"}, "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, - "dialyxir": {:hex, :dialyxir, "1.4.5", "ca1571ac18e0f88d4ab245f0b60fa31ff1b12cbae2b11bd25d207f865e8ae78a", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b0fb08bb8107c750db5c0b324fa2df5ceaa0f9307690ee3c1f6ba5b9eb5d35c3"}, + "dialyxir": {:hex, :dialyxir, "1.4.6", "7cca478334bf8307e968664343cbdb432ee95b4b68a9cba95bdabb0ad5bdfd9a", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "8cf5615c5cd4c2da6c501faae642839c8405b49f8aa057ad4ae401cb808ef64d"}, "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "ecto": {:hex, :ecto, "3.13.2", "7d0c0863f3fc8d71d17fc3ad3b9424beae13f02712ad84191a826c7169484f01", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "669d9291370513ff56e7b7e7081b7af3283d02e046cf3d403053c557894a0b3e"}, "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, - "ex_doc": {:hex, :ex_doc, "0.38.2", "504d25eef296b4dec3b8e33e810bc8b5344d565998cd83914ffe1b8503737c02", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "732f2d972e42c116a70802f9898c51b54916e542cc50968ac6980512ec90f42b"}, + "ex_doc": {:hex, :ex_doc, "0.38.3", "ddafe36b8e9fe101c093620879f6604f6254861a95133022101c08e75e6c759a", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "ecaa785456a67f63b4e7d7f200e8832fa108279e7eb73fd9928e7e66215a01f9"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, From b3edc2fd4c4022b0aed9fe5a6f8069575b506c89 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:33:09 +0900 Subject: [PATCH 18/34] comment on why older version in ci --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 66f32da..8dbc3fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,6 +45,8 @@ jobs: otp: 27.3.1 clickhouse: 24.12.2.29 timezone: UTC + # some older pre-JSON ClickHouse version + # https://github.com/plausible/ch/issues/273 - elixir: 1.18 otp: 28 clickhouse: 24.5.4.49 From 27e11d8b0a9246b1c53106c2d0749ff6c3f6d5a2 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 09:34:30 +0900 Subject: [PATCH 19/34] shorter cache key --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8dbc3fa..f2e1dd7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,10 +80,10 @@ jobs: path: | deps _build - key: test-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ github.head_ref || github.ref }}-${{ hashFiles('**/mix.lock') }} + key: test-${{ steps.beam.outputs.elixir-version }}-${{ github.head_ref || github.ref }}-${{ hashFiles('**/mix.lock') }} restore-keys: | - test-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ github.head_ref || github.ref }}- - test-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-refs/heads/master- + test-${{ steps.beam.outputs.elixir-version }}-${{ github.head_ref || github.ref }}- + test-${{ steps.beam.outputs.elixir-version }}-refs/heads/master- - run: mix deps.get --only $MIX_ENV - run: mix compile --warnings-as-errors From 116322b9fd077cba2c21c107c6af133b3dd13c03 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 14:47:22 +0900 Subject: [PATCH 20/34] fix internal type ordering in Variant (#275) * fix internal type ordering in Variant * cleanup * link pr --- CHANGELOG.md | 4 ++++ lib/ch/types.ex | 2 +- test/ch/variant_test.exs | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88b6a72..ba42b84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- fix internal type ordering in Variant https://github.com/plausible/ch/pull/275 + ## 0.5.5 (2025-08-26) - fix version check for adding JSON settings https://github.com/plausible/ch/pull/274 diff --git a/lib/ch/types.ex b/lib/ch/types.ex index de29ba2..e86b16c 100644 --- a/lib/ch/types.ex +++ b/lib/ch/types.ex @@ -502,7 +502,7 @@ defmodule Ch.Types do defp named_columns_to_types([], acc), do: acc defp build_variant(types) do - Enum.sort_by(types, &__MODULE__.encode/1) + Enum.sort_by(types, fn t -> IO.iodata_to_binary(encode(t)) end) end # TODO '', \' diff --git a/test/ch/variant_test.exs b/test/ch/variant_test.exs index 04b5d8e..23a2c57 100644 --- a/test/ch/variant_test.exs +++ b/test/ch/variant_test.exs @@ -19,6 +19,24 @@ defmodule Ch.VariantTest do [["Hello, World!"]] end + # https://github.com/plausible/ch/issues/272 + test "ordering internal types", %{conn: conn} do + test = %{ + "'hello'" => "hello", + "-10" => -10, + "true" => true, + "map('hello', null::Nullable(String))" => %{"hello" => nil}, + "map('hello', 'world'::Nullable(String))" => %{"hello" => "world"} + } + + for {value, expected} <- test do + assert Ch.query!( + conn, + "select #{value}::Variant(String, Int32, Bool, Map(String, Nullable(String)))" + ).rows == [[expected]] + end + end + test "with a table", %{conn: conn} do # https://clickhouse.com/docs/sql-reference/data-types/variant#creating-variant Ch.query!(conn, """ From d8cb624409678529dd9ca8839cd96d5baefd0ccc Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Aug 2025 14:48:11 +0900 Subject: [PATCH 21/34] release v0.5.6 --- CHANGELOG.md | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba42b84..75810b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 0.5.6 (2025-08-26) - fix internal type ordering in Variant https://github.com/plausible/ch/pull/275 diff --git a/mix.exs b/mix.exs index e0dc05c..4f86353 100644 --- a/mix.exs +++ b/mix.exs @@ -2,7 +2,7 @@ defmodule Ch.MixProject do use Mix.Project @source_url "https://github.com/plausible/ch" - @version "0.5.5" + @version "0.5.6" def project do [ From b28de4ee794d76d255b3be5164c7a80056e3871f Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Fri, 2 Jan 2026 15:01:25 +0300 Subject: [PATCH 22/34] fewer changes --- .github/workflows/test.yml | 3 + CHANGELOG.md | 45 ++++++++ lib/ch.ex | 3 +- lib/ch/encode/multipart.ex | 87 ---------------- lib/ch/encode/parameters.ex | 113 -------------------- lib/ch/query.ex | 198 ++++++++++++++++++++++++++++++++++-- test/ch/multipart_test.exs | 13 +++ 7 files changed, 255 insertions(+), 207 deletions(-) delete mode 100644 lib/ch/encode/multipart.ex delete mode 100644 lib/ch/encode/parameters.ex create mode 100644 test/ch/multipart_test.exs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f46e5dc..9ffb996 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,6 +97,9 @@ jobs: if: ${{ matrix.lint }} - run: mix test --include slow + - run: mix test --include slow --include multipart + env: + CH_MULTIPART: true - name: Restore PLTs cache if: ${{ matrix.dialyzer }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c7406c..fc45c91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,50 @@ # Changelog +## Unreleased + +- added support for `multipart/form-data` in queries: https://github.com/plausible/ch/pull/290 -- which allows bypassing URL length limits sometimes imposed by reverse proxies when sending queries with many parameters. + + ⚠️ This is currently **opt-in** per query ⚠️ + + Global support for the entire connection pool is planned for a future release. + + **Usage** + + Pass `multipart: true` in the options list for `Ch.query/4` + + ```elixir + # Example usage + Ch.query(pool, "SELECT {a:String}, {b:String}", %{"a" => "A", "b" => "B"}, multipart: true) + ``` + +
+ View raw request format reference + + ```http + POST / HTTP/1.1 + content-length: 387 + host: localhost:8123 + user-agent: ch/0.6.2-dev + x-clickhouse-format: RowBinaryWithNamesAndTypes + content-type: multipart/form-data; boundary="ChFormBoundaryZZlfchKTcd8ToWjEvn66i3lAxNJ_T9dw" + + --ChFormBoundaryZZlfchKTcd8ToWjEvn66i3lAxNJ_T9dw + content-disposition: form-data; name="param_a" + + A + --ChFormBoundaryZZlfchKTcd8ToWjEvn66i3lAxNJ_T9dw + content-disposition: form-data; name="param_b" + + B + --ChFormBoundaryZZlfchKTcd8ToWjEvn66i3lAxNJ_T9dw + content-disposition: form-data; name="query" + + select {a:String}, {b:String} + --ChFormBoundaryZZlfchKTcd8ToWjEvn66i3lAxNJ_T9dw-- + ``` + +
+ ## 0.6.1 (2025-12-04) - handle disconnect during stream https://github.com/plausible/ch/pull/283 diff --git a/lib/ch.ex b/lib/ch.ex index 8b12e2c..e63dd12 100644 --- a/lib/ch.ex +++ b/lib/ch.ex @@ -31,7 +31,6 @@ defmodule Ch do * `:password` - User password * `:settings` - Keyword list of ClickHouse settings * `:timeout` - HTTP receive timeout in milliseconds - * `:transport_opts` - options to be given to the transport being used. See `Mint.HTTP1.connect/4` for more info * [`DBConnection.start_option()`](https://hexdocs.pm/db_connection/DBConnection.html#t:start_option/0) """ @@ -59,6 +58,7 @@ defmodule Ch do # TODO remove | {:encode, boolean} | {:decode, boolean} + | {:multipart, boolean} | DBConnection.connection_option() @doc """ @@ -76,6 +76,7 @@ defmodule Ch do * `:headers` - Custom HTTP headers for the request * `:format` - Custom response format for the request * `:decode` - Whether to automatically decode the response + * `:multipart` - Whether to send the query as multipart/form-data * [`DBConnection.connection_option()`](https://hexdocs.pm/db_connection/DBConnection.html#t:connection_option/0) """ diff --git a/lib/ch/encode/multipart.ex b/lib/ch/encode/multipart.ex deleted file mode 100644 index c058fde..0000000 --- a/lib/ch/encode/multipart.ex +++ /dev/null @@ -1,87 +0,0 @@ -defmodule Ch.Encode.Multipart do - @moduledoc false - - alias Ch.Encode.Parameters - - @doc """ - Encodes a query statement and params into a multipart request. - """ - @spec encode(iodata, map, [Ch.query_option()]) :: - {list, Mint.Types.headers(), iodata} - def encode(statement, params, opts) do - types = Keyword.get(opts, :types) - settings = Keyword.get(opts, :settings, []) - default_format = if types, do: "RowBinary", else: "RowBinaryWithNamesAndTypes" - format = Keyword.get(opts, :format) || default_format - - boundary = "ChFormBoundary" <> Base.url_encode64(:crypto.strong_rand_bytes(24)) - content_type = "multipart/form-data; boundary=\"#{boundary}\"" - enc_boundary = "--#{boundary}\r\n" - - multipart = - params - |> multipart_params(enc_boundary) - |> add_multipart_part("query", statement, enc_boundary) - |> then(&[&1 | "--#{boundary}--\r\n"]) - - headers = [{"x-clickhouse-format", format}, {"content-type", content_type} | headers(opts)] - - {settings, headers, multipart} - end - - defp multipart_params(params, boundary) when is_map(params) do - multipart_named_params(Map.to_list(params), boundary, []) - end - - defp multipart_params(params, boundary) when is_list(params) do - multipart_positional_params(params, 0, boundary, []) - end - - defp multipart_named_params([{name, value} | params], boundary, acc) do - acc = - add_multipart_part( - acc, - "param_" <> URI.encode_www_form(name), - Parameters.encode(value), - boundary - ) - - multipart_named_params(params, boundary, acc) - end - - defp multipart_named_params([], _boundary, acc), do: acc - - defp multipart_positional_params([value | params], idx, boundary, acc) do - acc = - add_multipart_part( - acc, - "param_$" <> Integer.to_string(idx), - Parameters.encode(value), - boundary - ) - - multipart_positional_params(params, idx + 1, boundary, acc) - end - - defp multipart_positional_params([], _idx, _boundary, acc), do: acc - - @compile inline: [add_multipart_part: 4] - defp add_multipart_part(multipart, name, value, boundary) do - part = [ - boundary, - "content-disposition: form-data; name=\"", - name, - "\"\r\n\r\n", - value, - "\r\n" - ] - - case multipart do - [] -> part - _ -> [multipart | part] - end - end - - @spec headers(Keyword.t()) :: Mint.Types.headers() - defp headers(opts), do: Keyword.get(opts, :headers, []) -end diff --git a/lib/ch/encode/parameters.ex b/lib/ch/encode/parameters.ex deleted file mode 100644 index 0ac0828..0000000 --- a/lib/ch/encode/parameters.ex +++ /dev/null @@ -1,113 +0,0 @@ -defmodule Ch.Encode.Parameters do - @moduledoc false - - @doc """ - Encodes a map/list of parameters into a list of clickhouse parameter tuples. - - The format is `[{"param_", ""}, ...]`. - """ - @spec encode_many(map | [term]) :: [{String.t(), String.t()}] - def encode_many(params) when is_map(params) do - Enum.map(params, fn {k, v} -> {"param_#{k}", encode(v)} end) - end - - def encode_many(params) when is_list(params) do - params - |> Enum.with_index() - |> Enum.map(fn {v, idx} -> {"param_$#{idx}", encode(v)} end) - end - - @doc """ - Encodes a clickhouse parameter to a string. - """ - @spec encode(term) :: binary - def encode(n) when is_integer(n), do: Integer.to_string(n) - def encode(f) when is_float(f), do: Float.to_string(f) - - # TODO possibly speed up - # For more info see - # https://clickhouse.com/docs/en/interfaces/http#tabs-in-url-parameters - # "escaped" format is the same as https://clickhouse.com/docs/en/interfaces/formats#tabseparated-data-formatting - def encode(b) when is_binary(b) do - escape_param([{"\\", "\\\\"}, {"\t", "\\\t"}, {"\n", "\\\n"}], b) - end - - def encode(b) when is_boolean(b), do: Atom.to_string(b) - def encode(nil), do: "\\N" - def encode(%Decimal{} = d), do: Decimal.to_string(d, :normal) - def encode(%Date{} = date), do: Date.to_iso8601(date) - def encode(%NaiveDateTime{} = naive), do: NaiveDateTime.to_iso8601(naive) - def encode(%Time{} = time), do: Time.to_iso8601(time) - - def encode(%DateTime{microsecond: microsecond} = dt) do - dt = DateTime.shift_zone!(dt, "Etc/UTC") - - case microsecond do - {val, precision} when val > 0 and precision > 0 -> - size = round(:math.pow(10, precision)) - unix = DateTime.to_unix(dt, size) - seconds = div(unix, size) - fractional = rem(unix, size) - - IO.iodata_to_binary([ - Integer.to_string(seconds), - ?., - String.pad_leading(Integer.to_string(fractional), precision, "0") - ]) - - _ -> - dt |> DateTime.to_unix(:second) |> Integer.to_string() - end - end - - def encode(tuple) when is_tuple(tuple) do - IO.iodata_to_binary([?(, encode_array_params(Tuple.to_list(tuple)), ?)]) - end - - def encode(a) when is_list(a) do - IO.iodata_to_binary([?[, encode_array_params(a), ?]]) - end - - def encode(m) when is_map(m) do - IO.iodata_to_binary([?{, encode_map_params(Map.to_list(m)), ?}]) - end - - defp encode_array_params([last]), do: encode_array_param(last) - - defp encode_array_params([s | rest]) do - [encode_array_param(s), ?, | encode_array_params(rest)] - end - - defp encode_array_params([] = empty), do: empty - - defp encode_map_params([last]), do: encode_map_param(last) - - defp encode_map_params([kv | rest]) do - [encode_map_param(kv), ?, | encode_map_params(rest)] - end - - defp encode_map_params([] = empty), do: empty - - defp encode_array_param(s) when is_binary(s) do - [?', escape_param([{"'", "''"}, {"\\", "\\\\"}], s), ?'] - end - - defp encode_array_param(nil), do: "null" - - defp encode_array_param(%s{} = param) when s in [Date, NaiveDateTime] do - [?', encode(param), ?'] - end - - defp encode_array_param(v), do: encode(v) - - defp encode_map_param({k, v}) do - [encode_array_param(k), ?:, encode_array_param(v)] - end - - defp escape_param([{pattern, replacement} | escapes], param) do - param = String.replace(param, pattern, replacement) - escape_param(escapes, param) - end - - defp escape_param([], param), do: param -end diff --git a/lib/ch/query.ex b/lib/ch/query.ex index 0e1cd78..dec9746 100644 --- a/lib/ch/query.ex +++ b/lib/ch/query.ex @@ -1,8 +1,15 @@ defmodule Ch.Query do @moduledoc "Query struct wrapping the SQL statement." - defstruct [:statement, :command, :encode, :decode] + defstruct [:statement, :command, :encode, :decode, :multipart] + @dialyzer :no_improper_lists - @type t :: %__MODULE__{statement: iodata, command: command, encode: boolean, decode: boolean} + @type t :: %__MODULE__{ + statement: iodata, + command: command, + encode: boolean, + decode: boolean, + multipart: boolean + } @doc false @spec build(iodata, [Ch.query_option()]) :: t @@ -10,7 +17,15 @@ defmodule Ch.Query do command = Keyword.get(opts, :command) || extract_command(statement) encode = Keyword.get(opts, :encode, true) decode = Keyword.get(opts, :decode, true) - %__MODULE__{statement: statement, command: command, encode: encode, decode: decode} + multipart = Keyword.get(opts, :multipart, false) + + %__MODULE__{ + statement: statement, + command: command, + encode: encode, + decode: decode, + multipart: multipart + } end statements = [ @@ -73,7 +88,6 @@ end defimpl DBConnection.Query, for: Ch.Query do alias Ch.{Query, Result, RowBinary} - alias Ch.Encode.{Multipart, Parameters} @spec parse(Query.t(), [Ch.query_option()]) :: Query.t() def parse(query, _opts), do: query @@ -125,12 +139,84 @@ defimpl DBConnection.Query, for: Ch.Query do {_query_params = [], headers(opts), [statement, ?\n | data]} true -> - {Parameters.encode_many(params), headers(opts), statement} + {query_params(params), headers(opts), statement} end end + def encode(%Query{multipart: true, statement: statement}, params, opts) do + types = Keyword.get(opts, :types) + default_format = if types, do: "RowBinary", else: "RowBinaryWithNamesAndTypes" + format = Keyword.get(opts, :format) || default_format + + boundary = "ChFormBoundary" <> Base.url_encode64(:crypto.strong_rand_bytes(24)) + content_type = "multipart/form-data; boundary=\"#{boundary}\"" + enc_boundary = "--#{boundary}\r\n" + multipart = multipart_params(params, enc_boundary) + multipart = add_multipart_part(multipart, "query", statement, enc_boundary) + multipart = [multipart | "--#{boundary}--\r\n"] + + {_no_query_params = [], + [{"x-clickhouse-format", format}, {"content-type", content_type} | headers(opts)], multipart} + end + def encode(%Query{statement: statement}, params, opts) do - Multipart.encode(statement, params, opts) + types = Keyword.get(opts, :types) + default_format = if types, do: "RowBinary", else: "RowBinaryWithNamesAndTypes" + format = Keyword.get(opts, :format) || default_format + {query_params(params), [{"x-clickhouse-format", format} | headers(opts)], statement} + end + + defp multipart_params(params, boundary) when is_map(params) do + multipart_named_params(Map.to_list(params), boundary, []) + end + + defp multipart_params(params, boundary) when is_list(params) do + multipart_positional_params(params, 0, boundary, []) + end + + defp multipart_named_params([{name, value} | params], boundary, acc) do + acc = + add_multipart_part( + acc, + "param_" <> URI.encode_www_form(name), + encode_param(value), + boundary + ) + + multipart_named_params(params, boundary, acc) + end + + defp multipart_named_params([], _boundary, acc), do: acc + + defp multipart_positional_params([value | params], idx, boundary, acc) do + acc = + add_multipart_part( + acc, + "param_$" <> Integer.to_string(idx), + encode_param(value), + boundary + ) + + multipart_positional_params(params, idx + 1, boundary, acc) + end + + defp multipart_positional_params([], _idx, _boundary, acc), do: acc + + @compile inline: [add_multipart_part: 4] + defp add_multipart_part(multipart, name, value, boundary) do + part = [ + boundary, + "content-disposition: form-data; name=\"", + name, + "\"\r\n\r\n", + value, + "\r\n" + ] + + case multipart do + [] -> part + _ -> [multipart | part] + end end defp format_row_binary?(statement) when is_binary(statement) do @@ -206,6 +292,106 @@ defimpl DBConnection.Query, for: Ch.Query do end end + defp query_params(params) when is_map(params) do + Enum.map(params, fn {k, v} -> {"param_#{k}", encode_param(v)} end) + end + + defp query_params(params) when is_list(params) do + params + |> Enum.with_index() + |> Enum.map(fn {v, idx} -> {"param_$#{idx}", encode_param(v)} end) + end + + defp encode_param(n) when is_integer(n), do: Integer.to_string(n) + defp encode_param(f) when is_float(f), do: Float.to_string(f) + + # TODO possibly speed up + # For more info see + # https://clickhouse.com/docs/en/interfaces/http#tabs-in-url-parameters + # "escaped" format is the same as https://clickhouse.com/docs/en/interfaces/formats#tabseparated-data-formatting + defp encode_param(b) when is_binary(b) do + escape_param([{"\\", "\\\\"}, {"\t", "\\\t"}, {"\n", "\\\n"}], b) + end + + defp encode_param(b) when is_boolean(b), do: Atom.to_string(b) + defp encode_param(nil), do: "\\N" + defp encode_param(%Decimal{} = d), do: Decimal.to_string(d, :normal) + defp encode_param(%Date{} = date), do: Date.to_iso8601(date) + defp encode_param(%NaiveDateTime{} = naive), do: NaiveDateTime.to_iso8601(naive) + defp encode_param(%Time{} = time), do: Time.to_iso8601(time) + + defp encode_param(%DateTime{microsecond: microsecond} = dt) do + dt = DateTime.shift_zone!(dt, "Etc/UTC") + + case microsecond do + {val, precision} when val > 0 and precision > 0 -> + size = round(:math.pow(10, precision)) + unix = DateTime.to_unix(dt, size) + seconds = div(unix, size) + fractional = rem(unix, size) + + IO.iodata_to_binary([ + Integer.to_string(seconds), + ?., + String.pad_leading(Integer.to_string(fractional), precision, "0") + ]) + + _ -> + dt |> DateTime.to_unix(:second) |> Integer.to_string() + end + end + + defp encode_param(tuple) when is_tuple(tuple) do + IO.iodata_to_binary([?(, encode_array_params(Tuple.to_list(tuple)), ?)]) + end + + defp encode_param(a) when is_list(a) do + IO.iodata_to_binary([?[, encode_array_params(a), ?]]) + end + + defp encode_param(m) when is_map(m) do + IO.iodata_to_binary([?{, encode_map_params(Map.to_list(m)), ?}]) + end + + defp encode_array_params([last]), do: encode_array_param(last) + + defp encode_array_params([s | rest]) do + [encode_array_param(s), ?, | encode_array_params(rest)] + end + + defp encode_array_params([] = empty), do: empty + + defp encode_map_params([last]), do: encode_map_param(last) + + defp encode_map_params([kv | rest]) do + [encode_map_param(kv), ?, | encode_map_params(rest)] + end + + defp encode_map_params([] = empty), do: empty + + defp encode_array_param(s) when is_binary(s) do + [?', escape_param([{"'", "''"}, {"\\", "\\\\"}], s), ?'] + end + + defp encode_array_param(nil), do: "null" + + defp encode_array_param(%s{} = param) when s in [Date, NaiveDateTime] do + [?', encode_param(param), ?'] + end + + defp encode_array_param(v), do: encode_param(v) + + defp encode_map_param({k, v}) do + [encode_array_param(k), ?:, encode_array_param(v)] + end + + defp escape_param([{pattern, replacement} | escapes], param) do + param = String.replace(param, pattern, replacement) + escape_param(escapes, param) + end + + defp escape_param([], param), do: param + @spec headers(Keyword.t()) :: Mint.Types.headers() defp headers(opts), do: Keyword.get(opts, :headers, []) end diff --git a/test/ch/multipart_test.exs b/test/ch/multipart_test.exs new file mode 100644 index 0000000..1913e6c --- /dev/null +++ b/test/ch/multipart_test.exs @@ -0,0 +1,13 @@ +defmodule Ch.MultipartTest do + use ExUnit.Case, async: true + + setup do + {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} + end + + test "sends multipart", %{conn: conn} do + assert Ch.query!(conn, "SELECT {a:String}, {b:String}", %{"a" => "A", "b" => "B"}, + multipart: true + ).rows == [["A", "B"]] + end +end From fbe507228411efe54d7e4986e401b9ecbe6726f4 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Fri, 2 Jan 2026 15:01:56 +0300 Subject: [PATCH 23/34] eh --- .github/workflows/test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9ffb996..f46e5dc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,9 +97,6 @@ jobs: if: ${{ matrix.lint }} - run: mix test --include slow - - run: mix test --include slow --include multipart - env: - CH_MULTIPART: true - name: Restore PLTs cache if: ${{ matrix.dialyzer }} From ae596c10dbcb144eb889b6c201dfac1cc587632a Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Fri, 2 Jan 2026 15:02:41 +0300 Subject: [PATCH 24/34] eh x2 --- lib/ch.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ch.ex b/lib/ch.ex index e63dd12..989a991 100644 --- a/lib/ch.ex +++ b/lib/ch.ex @@ -31,6 +31,7 @@ defmodule Ch do * `:password` - User password * `:settings` - Keyword list of ClickHouse settings * `:timeout` - HTTP receive timeout in milliseconds + * `:transport_opts` - options to be given to the transport being used. See `Mint.HTTP1.connect/4` for more info * [`DBConnection.start_option()`](https://hexdocs.pm/db_connection/DBConnection.html#t:start_option/0) """ From 6761c333af913da9e827f3f1173646eb1d20c116 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Fri, 2 Jan 2026 15:10:50 +0300 Subject: [PATCH 25/34] readme --- README.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1faecaa..2b773b4 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,24 @@ Note on datetime encoding in query parameters: - `%NaiveDateTime{}` is encoded as text to make it assume the column's or ClickHouse server's timezone - `%DateTime{}` is encoded as unix timestamp and is treated as UTC timestamp by ClickHouse +#### Select rows (lots of params, reverse proxy) + +For queries with many parameters the resulting URL can become too long for some reverse proxies, resulting in a `414 Request-URI Too Large` error. + +To avoid this, you can use the `multipart: true` option to send the query and parameters in the request body. + +```elixir +{:ok, pid} = Ch.start_link() + +# Moves parameters from the URL to a multipart/form-data body +%Ch.Result{rows: [[[1, 2, 3 | _rest]]]} = + Ch.query!(pid, "SELECT {ids:Array(UInt64)}", %{"ids" => Enum.to_list(1..10_000)}, multipart: true) +``` + +> [!NOTE] +> +> `multipart: true` is currently required on each individual query. Support for pool-wide configuration is planned for a future release. + #### Insert rows ```elixir @@ -161,11 +179,6 @@ settings = [async_insert: 1] Ch.query!(pid, "SHOW SETTINGS LIKE 'async_insert'", [], settings: settings) ``` -#### Multipart requests - -SELECT queries will be automatically sent as multipart requests. -INSERT queries and streams are treated normally. - ## Caveats #### NULL in RowBinary From be1f04acf701067cb83f15524b298922d155a22f Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Fri, 2 Jan 2026 15:13:24 +0300 Subject: [PATCH 26/34] dialyzer --- lib/ch/query.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ch/query.ex b/lib/ch/query.ex index dec9746..d6d077a 100644 --- a/lib/ch/query.ex +++ b/lib/ch/query.ex @@ -1,7 +1,6 @@ defmodule Ch.Query do @moduledoc "Query struct wrapping the SQL statement." defstruct [:statement, :command, :encode, :decode, :multipart] - @dialyzer :no_improper_lists @type t :: %__MODULE__{ statement: iodata, @@ -87,6 +86,7 @@ defmodule Ch.Query do end defimpl DBConnection.Query, for: Ch.Query do + @dialyzer :no_improper_lists alias Ch.{Query, Result, RowBinary} @spec parse(Query.t(), [Ch.query_option()]) :: Query.t() From a86ffff626f3d85cd3612798c8e6c45aa1a92371 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Fri, 2 Jan 2026 15:21:18 +0300 Subject: [PATCH 27/34] a few more tests --- test/ch/multipart_test.exs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/ch/multipart_test.exs b/test/ch/multipart_test.exs index 1913e6c..3d7cd7e 100644 --- a/test/ch/multipart_test.exs +++ b/test/ch/multipart_test.exs @@ -6,8 +6,18 @@ defmodule Ch.MultipartTest do end test "sends multipart", %{conn: conn} do - assert Ch.query!(conn, "SELECT {a:String}, {b:String}", %{"a" => "A", "b" => "B"}, - multipart: true - ).rows == [["A", "B"]] + sql = "SELECT {a:String}, {b:String}" + params = %{"a" => "A", "b" => "B"} + + assert %Ch.Result{rows: [["A", "B"]]} = + Ch.query!(conn, sql, params, multipart: true) + end + + test "sends positional parameters correctly", %{conn: conn} do + sql = "SELECT {$0:String}, {$1:Int32}" + params = ["pos0", 42] + + assert %Ch.Result{rows: [["pos0", 42]]} = + Ch.query!(conn, sql, params, multipart: true) end end From 0f910e67199624a5d09bd80e07670d6224bedf9f Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Fri, 2 Jan 2026 16:01:05 +0300 Subject: [PATCH 28/34] more tests --- test/ch/connection_test.exs | 871 +++++++++++++++++++++--------------- test/ch/multipart_test.exs | 23 - 2 files changed, 499 insertions(+), 395 deletions(-) delete mode 100644 test/ch/multipart_test.exs diff --git a/test/ch/connection_test.exs b/test/ch/connection_test.exs index 29a0265..1e59c6b 100644 --- a/test/ch/connection_test.exs +++ b/test/ch/connection_test.exs @@ -1,62 +1,81 @@ defmodule Ch.ConnectionTest do - use ExUnit.Case + use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] alias Ch.RowBinary setup do {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} end - test "select without params", %{conn: conn} do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = Ch.query(conn, "select 1") + defp query( + %{conn: conn, query_options: default_options}, + sql, + params \\ [], + custom_options \\ [] + ) do + Ch.query(conn, sql, params, Keyword.merge(default_options, custom_options)) end - test "select with types", %{conn: conn} do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = Ch.query(conn, "select 1", [], types: ["UInt8"]) + defp query!( + %{conn: conn, query_options: default_options}, + sql, + params \\ [], + custom_options \\ [] + ) do + Ch.query!(conn, sql, params, Keyword.merge(default_options, custom_options)) end - test "select with params", %{conn: conn} do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = Ch.query(conn, "select {a:UInt8}", %{"a" => 1}) + test "select without params", ctx do + assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select 1") + end + + test "select with types", ctx do + assert {:ok, %{num_rows: 1, rows: [[1]]}} = + query(ctx, "select 1", [], types: ["UInt8"]) + end + + test "select with params", ctx do + assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:UInt8}", %{"a" => 1}) assert {:ok, %{num_rows: 1, rows: [[true]]}} = - Ch.query(conn, "select {b:Bool}", %{"b" => true}) + query(ctx, "select {b:Bool}", %{"b" => true}) assert {:ok, %{num_rows: 1, rows: [[false]]}} = - Ch.query(conn, "select {b:Bool}", %{"b" => false}) + query(ctx, "select {b:Bool}", %{"b" => false}) assert {:ok, %{num_rows: 1, rows: [[nil]]}} = - Ch.query(conn, "select {n:Nullable(Nothing)}", %{"n" => nil}) + query(ctx, "select {n:Nullable(Nothing)}", %{"n" => nil}) assert {:ok, %{num_rows: 1, rows: [[1.0]]}} = - Ch.query(conn, "select {a:Float32}", %{"a" => 1.0}) + query(ctx, "select {a:Float32}", %{"a" => 1.0}) assert {:ok, %{num_rows: 1, rows: [["a&b=c"]]}} = - Ch.query(conn, "select {a:String}", %{"a" => "a&b=c"}) + query(ctx, "select {a:String}", %{"a" => "a&b=c"}) assert {:ok, %{num_rows: 1, rows: [["a\n"]]}} = - Ch.query(conn, "select {a:String}", %{"a" => "a\n"}) + query(ctx, "select {a:String}", %{"a" => "a\n"}) assert {:ok, %{num_rows: 1, rows: [["a\t"]]}} = - Ch.query(conn, "select {a:String}", %{"a" => "a\t"}) + query(ctx, "select {a:String}", %{"a" => "a\t"}) assert {:ok, %{num_rows: 1, rows: [[["a\tb"]]]}} = - Ch.query(conn, "select {a:Array(String)}", %{"a" => ["a\tb"]}) + query(ctx, "select {a:Array(String)}", %{"a" => ["a\tb"]}) assert {:ok, %{num_rows: 1, rows: [[[true, false]]]}} = - Ch.query(conn, "select {a:Array(Bool)}", %{"a" => [true, false]}) + query(ctx, "select {a:Array(Bool)}", %{"a" => [true, false]}) assert {:ok, %{num_rows: 1, rows: [[["a", nil, "b"]]]}} = - Ch.query(conn, "select {a:Array(Nullable(String))}", %{"a" => ["a", nil, "b"]}) + query(ctx, "select {a:Array(Nullable(String))}", %{"a" => ["a", nil, "b"]}) assert {:ok, %{num_rows: 1, rows: [row]}} = - Ch.query(conn, "select {a:Decimal(9,4)}", %{"a" => Decimal.new("2000.333")}) + query(ctx, "select {a:Decimal(9,4)}", %{"a" => Decimal.new("2000.333")}) assert row == [Decimal.new("2000.3330")] assert {:ok, %{num_rows: 1, rows: [[~D[2022-01-01]]]}} = - Ch.query(conn, "select {a:Date}", %{"a" => ~D[2022-01-01]}) + query(ctx, "select {a:Date}", %{"a" => ~D[2022-01-01]}) assert {:ok, %{num_rows: 1, rows: [[~D[2022-01-01]]]}} = - Ch.query(conn, "select {a:Date32}", %{"a" => ~D[2022-01-01]}) + query(ctx, "select {a:Date32}", %{"a" => ~D[2022-01-01]}) naive_noon = ~N[2022-01-01 12:00:00] @@ -64,7 +83,7 @@ defmodule Ch.ConnectionTest do # see https://clickhouse.com/docs/en/sql-reference/data-types/datetime # https://kb.altinity.com/altinity-kb-queries-and-syntax/time-zones/ assert {:ok, %{num_rows: 1, rows: [[naive_datetime]], headers: headers}} = - Ch.query(conn, "select {naive:DateTime}", %{"naive" => naive_noon}) + query(ctx, "select {naive:DateTime}", %{"naive" => naive_noon}) # to make this test pass for contributors with non UTC timezone we perform the same steps as ClickHouse # i.e. we give server timezone to the naive datetime and shift it to UTC before comparing with the result @@ -78,17 +97,17 @@ defmodule Ch.ConnectionTest do # when the timezone information is provided in the type, we don't need to rely on server timezone assert {:ok, %{num_rows: 1, rows: [[bkk_datetime]]}} = - Ch.query(conn, "select {$0:DateTime('Asia/Bangkok')}", [naive_noon]) + query(ctx, "select {$0:DateTime('Asia/Bangkok')}", [naive_noon]) assert bkk_datetime == DateTime.from_naive!(naive_noon, "Asia/Bangkok") assert {:ok, %{num_rows: 1, rows: [[~U[2022-01-01 12:00:00Z]]]}} = - Ch.query(conn, "select {$0:DateTime('UTC')}", [naive_noon]) + query(ctx, "select {$0:DateTime('UTC')}", [naive_noon]) naive_noon_ms = ~N[2022-01-01 12:00:00.123] assert {:ok, %{num_rows: 1, rows: [[naive_datetime]]}} = - Ch.query(conn, "select {$0:DateTime64(3)}", [naive_noon_ms]) + query(ctx, "select {$0:DateTime64(3)}", [naive_noon_ms]) assert NaiveDateTime.compare( naive_datetime, @@ -99,60 +118,60 @@ defmodule Ch.ConnectionTest do ) == :eq assert {:ok, %{num_rows: 1, rows: [[["a", "b'", "\\'c"]]]}} = - Ch.query(conn, "select {a:Array(String)}", %{"a" => ["a", "b'", "\\'c"]}) + query(ctx, "select {a:Array(String)}", %{"a" => ["a", "b'", "\\'c"]}) assert {:ok, %{num_rows: 1, rows: [[["a\n", "b\tc"]]]}} = - Ch.query(conn, "select {a:Array(String)}", %{"a" => ["a\n", "b\tc"]}) + query(ctx, "select {a:Array(String)}", %{"a" => ["a\n", "b\tc"]}) assert {:ok, %{num_rows: 1, rows: [[[1, 2, 3]]]}} = - Ch.query(conn, "select {a:Array(UInt8)}", %{"a" => [1, 2, 3]}) + query(ctx, "select {a:Array(UInt8)}", %{"a" => [1, 2, 3]}) assert {:ok, %{num_rows: 1, rows: [[[[1], [2, 3], []]]]}} = - Ch.query(conn, "select {a:Array(Array(UInt8))}", %{"a" => [[1], [2, 3], []]}) + query(ctx, "select {a:Array(Array(UInt8))}", %{"a" => [[1], [2, 3], []]}) uuid = "9B29BD20-924C-4DE5-BDB3-8C2AA1FCE1FC" uuid_bin = uuid |> String.replace("-", "") |> Base.decode16!() assert {:ok, %{num_rows: 1, rows: [[^uuid_bin]]}} = - Ch.query(conn, "select {a:UUID}", %{"a" => uuid}) + query(ctx, "select {a:UUID}", %{"a" => uuid}) # TODO # assert {:ok, %{num_rows: 1, rows: [[^uuid_bin]]}} = - # Ch.query(conn, "select {a:UUID}", %{"a" => uuid_bin}) + # query(ctx, "select {a:UUID}", %{"a" => uuid_bin}) # pseudo-positional bind - assert {:ok, %{num_rows: 1, rows: [[1]]}} = Ch.query(conn, "select {$0:UInt8}", [1]) + assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {$0:UInt8}", [1]) end - test "utc datetime query param encoding", %{conn: conn} do + test "utc datetime query param encoding", ctx do utc = ~U[2021-01-01 12:00:00Z] msk = DateTime.new!(~D[2021-01-01], ~T[15:00:00], "Europe/Moscow") - naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(conn)) |> DateTime.to_naive() + naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(ctx.conn)) |> DateTime.to_naive() - assert Ch.query!(conn, "select {$0:DateTime} as d, toString(d)", [utc]).rows == + assert query!(ctx, "select {$0:DateTime} as d, toString(d)", [utc]).rows == [[~N[2021-01-01 12:00:00], to_string(naive)]] - assert Ch.query!(conn, "select {$0:DateTime('UTC')} as d, toString(d)", [utc]).rows == + assert query!(ctx, "select {$0:DateTime('UTC')} as d, toString(d)", [utc]).rows == [[utc, "2021-01-01 12:00:00"]] - assert Ch.query!(conn, "select {$0:DateTime('Europe/Moscow')} as d, toString(d)", [utc]).rows == + assert query!(ctx, "select {$0:DateTime('Europe/Moscow')} as d, toString(d)", [utc]).rows == [[msk, "2021-01-01 15:00:00"]] end - test "non-utc datetime query param encoding", %{conn: conn} do + test "non-utc datetime query param encoding", ctx do jp = DateTime.shift_zone!(~U[2021-01-01 12:34:56Z], "Asia/Tokyo") assert inspect(jp) == "#DateTime<2021-01-01 21:34:56+09:00 JST Asia/Tokyo>" assert [[utc, jp]] = - Ch.query!(conn, "select {$0:DateTime('UTC')}, {$0:DateTime('Asia/Tokyo')}", [jp]).rows + query!(ctx, "select {$0:DateTime('UTC')}, {$0:DateTime('Asia/Tokyo')}", [jp]).rows assert inspect(utc) == "~U[2021-01-01 12:34:56Z]" assert inspect(jp) == "#DateTime<2021-01-01 21:34:56+09:00 JST Asia/Tokyo>" end - test "non-utc datetime rowbinary encoding", %{conn: conn} do - Ch.query!( - conn, + test "non-utc datetime rowbinary encoding", ctx do + query!( + ctx, "create table ch_non_utc_datetimes(name String, datetime DateTime) engine Memory" ) @@ -168,13 +187,12 @@ defmodule Ch.ConnectionTest do rows = [["taipei", taipei], ["tokyo", tokyo], ["vienna", vienna]] - Ch.query!(conn, "insert into ch_non_utc_datetimes(name, datetime) format RowBinary", rows, + query!(ctx, "insert into ch_non_utc_datetimes(name, datetime) format RowBinary", rows, types: ["String", "DateTime"] ) result = - conn - |> Ch.query!("select name, cast(datetime as DateTime('UTC')) from ch_non_utc_datetimes") + query!(ctx, "select name, cast(datetime as DateTime('UTC')) from ch_non_utc_datetimes") |> Map.fetch!(:rows) |> Map.new(fn [name, datetime] -> {name, datetime} end) @@ -183,57 +201,61 @@ defmodule Ch.ConnectionTest do assert result["vienna"] == ~U[2024-12-21 05:35:19Z] end - test "utc datetime64 query param encoding", %{conn: conn} do + test "utc datetime64 query param encoding", ctx do utc = ~U[2021-01-01 12:00:00.123456Z] msk = DateTime.new!(~D[2021-01-01], ~T[15:00:00.123456], "Europe/Moscow") - naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(conn)) |> DateTime.to_naive() + naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(ctx.conn)) |> DateTime.to_naive() - assert Ch.query!(conn, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == + assert query!(ctx, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == [[~N[2021-01-01 12:00:00.123456], to_string(naive)]] - assert Ch.query!(conn, "select {$0:DateTime64(6, 'UTC')} as d, toString(d)", [utc]).rows == + assert query!(ctx, "select {$0:DateTime64(6, 'UTC')} as d, toString(d)", [utc]).rows == [[utc, "2021-01-01 12:00:00.123456"]] - assert Ch.query!(conn, "select {$0:DateTime64(6,'Europe/Moscow')} as d, toString(d)", [utc]).rows == + assert query!(ctx, "select {$0:DateTime64(6,'Europe/Moscow')} as d, toString(d)", [utc]).rows == [[msk, "2021-01-01 15:00:00.123456"]] end - test "utc datetime64 zero microseconds query param encoding", %{conn: conn} do + test "utc datetime64 zero microseconds query param encoding", ctx do # this test case guards against a previous bug where DateTimes with a microsecond value of 0 and precision > 0 would # get encoded as a val like "1.6095024e9" which ClickHouse would be unable to parse to a DateTime. utc = ~U[2021-01-01 12:00:00.000000Z] - naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(conn)) |> DateTime.to_naive() + naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(ctx.conn)) |> DateTime.to_naive() - assert Ch.query!(conn, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == + assert query!(ctx, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == [[~N[2021-01-01 12:00:00.000000], to_string(naive)]] end - test "utc datetime64 microseconds with more precision than digits", %{conn: conn} do + test "utc datetime64 microseconds with more precision than digits", ctx do # this test case guards against a previous bug where DateTimes with a microsecond value of with N digits # and a precision > N would be encoded with a space like `234235234. 234123` utc = ~U[2024-05-26 20:00:46.099856Z] - naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(conn)) |> DateTime.to_naive() + naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(ctx.conn)) |> DateTime.to_naive() - assert Ch.query!(conn, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == + assert query!(ctx, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == [[~N[2024-05-26 20:00:46.099856Z], to_string(naive)]] end - test "select with options", %{conn: conn} do + test "select with options", ctx do assert {:ok, %{num_rows: 1, rows: [["async_insert", "Bool", "1"]]}} = - Ch.query(conn, "show settings like 'async_insert'", [], settings: [async_insert: 1]) + query(ctx, "show settings like 'async_insert'", [], settings: [async_insert: 1]) assert {:ok, %{num_rows: 1, rows: [["async_insert", "Bool", "0"]]}} = - Ch.query(conn, "show settings like 'async_insert'", [], settings: [async_insert: 0]) + query(ctx, "show settings like 'async_insert'", [], settings: [async_insert: 0]) end - test "create", %{conn: conn} do + test "create", ctx do assert {:ok, %{command: :create, num_rows: nil, rows: [], data: []}} = - Ch.query(conn, "create table create_example(a UInt8) engine = Memory") + query(ctx, "create table create_example(a UInt8) engine = Memory") + + on_exit(fn -> + Ch.Test.query("drop table create_example", [], database: Ch.Test.database()) + end) end - test "create with options", %{conn: conn} do + test "create with options", ctx do assert {:error, %Ch.Error{code: 164, message: message}} = - Ch.query(conn, "create table create_example(a UInt8) engine = Memory", [], + query(ctx, "create table create_example(a UInt8) engine = Memory", [], settings: [readonly: 1] ) @@ -241,49 +263,49 @@ defmodule Ch.ConnectionTest do end describe "insert" do - setup %{conn: conn} do + setup ctx do table = "insert_t_#{System.unique_integer([:positive])}" - Ch.query!( - conn, + query!( + ctx, "create table #{table}(a UInt8 default 1, b String) engine = Memory" ) {:ok, table: table} end - test "values", %{conn: conn, table: table} do + test "values", %{table: table} = ctx do assert {:ok, %{num_rows: 3}} = - Ch.query( - conn, + query( + ctx, "insert into {table:Identifier} values (1, 'a'),(2,'b'), (null, null)", %{"table" => table} ) assert {:ok, %{rows: rows}} = - Ch.query(conn, "select * from {table:Identifier}", %{"table" => table}) + query(ctx, "select * from {table:Identifier}", %{"table" => table}) assert rows == [[1, "a"], [2, "b"], [1, ""]] assert {:ok, %{num_rows: 2}} = - Ch.query( - conn, + query( + ctx, "insert into {$0:Identifier}(a, b) values ({$1:UInt8},{$2:String}),({$3:UInt8},{$4:String})", [table, 4, "d", 5, "e"] ) assert {:ok, %{rows: rows}} = - Ch.query(conn, "select * from {table:Identifier} where a >= 4", %{"table" => table}) + query(ctx, "select * from {table:Identifier} where a >= 4", %{"table" => table}) assert rows == [[4, "d"], [5, "e"]] end - test "when readonly", %{conn: conn, table: table} do + test "when readonly", %{table: table} = ctx do settings = [readonly: 1] assert {:error, %Ch.Error{code: 164, message: message}} = - Ch.query( - conn, + query( + ctx, "insert into {table:Identifier} values (1, 'a'), (2, 'b')", %{"table" => table}, settings: settings @@ -292,34 +314,34 @@ defmodule Ch.ConnectionTest do assert message =~ "Cannot execute query in readonly mode." end - test "automatic RowBinary", %{conn: conn, table: table} do + test "automatic RowBinary", %{table: table} = ctx do stmt = "insert into #{table}(a, b) format RowBinary" types = ["UInt8", "String"] rows = [[1, "a"], [2, "b"]] - assert %{num_rows: 2} = Ch.query!(conn, stmt, rows, types: types) + assert %{num_rows: 2} = query!(ctx, stmt, rows, types: types) assert %{rows: rows} = - Ch.query!(conn, "select * from {table:Identifier}", %{"table" => table}) + query!(ctx, "select * from {table:Identifier}", %{"table" => table}) assert rows == [[1, "a"], [2, "b"]] end - test "manual RowBinary", %{conn: conn, table: table} do + test "manual RowBinary", %{table: table} = ctx do stmt = "insert into #{table}(a, b) format RowBinary" types = ["UInt8", "String"] rows = [[1, "a"], [2, "b"]] data = RowBinary.encode_rows(rows, types) - assert %{num_rows: 2} = Ch.query!(conn, stmt, data, encode: false) + assert %{num_rows: 2} = query!(ctx, stmt, data, encode: false) assert %{rows: rows} = - Ch.query!(conn, "select * from {table:Identifier}", %{"table" => table}) + query!(ctx, "select * from {table:Identifier}", %{"table" => table}) assert rows == [[1, "a"], [2, "b"]] end - test "chunked", %{conn: conn, table: table} do + test "chunked", %{table: table} = ctx do types = ["UInt8", "String"] rows = [[1, "a"], [2, "b"], [3, "c"]] @@ -329,122 +351,127 @@ defmodule Ch.ConnectionTest do |> Stream.map(fn chunk -> RowBinary.encode_rows(chunk, types) end) assert {:ok, %{num_rows: 3}} = - Ch.query( - conn, + query( + ctx, "insert into #{table}(a, b) format RowBinary", stream, encode: false ) assert {:ok, %{rows: rows}} = - Ch.query(conn, "select * from {table:Identifier}", %{"table" => table}) + query(ctx, "select * from {table:Identifier}", %{"table" => table}) assert rows == [[1, "a"], [2, "b"], [3, "c"]] end - test "select", %{conn: conn, table: table} do + test "select", %{table: table} = ctx do assert {:ok, %{num_rows: 3}} = - Ch.query( - conn, + query( + ctx, "insert into {table:Identifier} values (1, 'a'), (2, 'b'), (null, null)", %{"table" => table} ) assert {:ok, %{num_rows: 3}} = - Ch.query( - conn, + query( + ctx, "insert into {table:Identifier}(a, b) select a, b from {table:Identifier}", %{"table" => table} ) assert {:ok, %{rows: rows}} = - Ch.query(conn, "select * from {table:Identifier}", %{"table" => table}) + query(ctx, "select * from {table:Identifier}", %{"table" => table}) assert rows == [[1, "a"], [2, "b"], [1, ""], [1, "a"], [2, "b"], [1, ""]] assert {:ok, %{num_rows: 2}} = - Ch.query( - conn, + query( + ctx, "insert into {$0:Identifier}(a, b) select a, b from {$0:Identifier} where a > {$1:UInt8}", [table, 1] ) assert {:ok, %{rows: new_rows}} = - Ch.query(conn, "select * from {table:Identifier}", %{"table" => table}) + query(ctx, "select * from {table:Identifier}", %{"table" => table}) assert new_rows -- rows == [[2, "b"], [2, "b"]] end end - test "delete", %{conn: conn} do - Ch.query!( - conn, - "create table delete_t(a UInt8, b String) engine = MergeTree order by tuple()" - ) + test "delete", ctx do + query!(ctx, "create table delete_t(a UInt8, b String) engine = MergeTree order by tuple()") + + on_exit(fn -> + Ch.Test.query("drop table delete_t", [], database: Ch.Test.database()) + end) - assert {:ok, %{num_rows: 2}} = Ch.query(conn, "insert into delete_t values (1,'a'), (2,'b')") + assert {:ok, %{num_rows: 2}} = query(ctx, "insert into delete_t values (1,'a'), (2,'b')") settings = [allow_experimental_lightweight_delete: 1] assert {:ok, %{rows: [], data: [], command: :delete}} = - Ch.query(conn, "delete from delete_t where 1", [], settings: settings) + query(ctx, "delete from delete_t where 1", [], settings: settings) end - test "query!", %{conn: conn} do - assert %{num_rows: 1, rows: [[1]]} = Ch.query!(conn, "select 1") + test "query!", ctx do + assert %{num_rows: 1, rows: [[1]]} = query!(ctx, "select 1") end describe "types" do - test "multiple types", %{conn: conn} do + test "multiple types", ctx do assert {:ok, %{num_rows: 1, rows: [[1, "a"]]}} = - Ch.query(conn, "select {a:Int8}, {b:String}", %{"a" => 1, "b" => "a"}) + query(ctx, "select {a:Int8}, {b:String}", %{"a" => 1, "b" => "a"}) end - test "ints", %{conn: conn} do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = Ch.query(conn, "select {a:Int8}", %{"a" => 1}) + test "ints", ctx do + assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:Int8}", %{"a" => 1}) assert {:ok, %{num_rows: 1, rows: [[-1000]]}} = - Ch.query(conn, "select {a:Int16}", %{"a" => -1000}) + query(ctx, "select {a:Int16}", %{"a" => -1000}) assert {:ok, %{num_rows: 1, rows: [[100_000]]}} = - Ch.query(conn, "select {a:Int32}", %{"a" => 100_000}) + query(ctx, "select {a:Int32}", %{"a" => 100_000}) - assert {:ok, %{num_rows: 1, rows: [[1]]}} = Ch.query(conn, "select {a:Int64}", %{"a" => 1}) - assert {:ok, %{num_rows: 1, rows: [[1]]}} = Ch.query(conn, "select {a:Int128}", %{"a" => 1}) - assert {:ok, %{num_rows: 1, rows: [[1]]}} = Ch.query(conn, "select {a:Int256}", %{"a" => 1}) + assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:Int64}", %{"a" => 1}) + assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:Int128}", %{"a" => 1}) + assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:Int256}", %{"a" => 1}) end - test "uints", %{conn: conn} do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = Ch.query(conn, "select {a:UInt8}", %{"a" => 1}) - assert {:ok, %{num_rows: 1, rows: [[1]]}} = Ch.query(conn, "select {a:UInt16}", %{"a" => 1}) - assert {:ok, %{num_rows: 1, rows: [[1]]}} = Ch.query(conn, "select {a:UInt32}", %{"a" => 1}) - assert {:ok, %{num_rows: 1, rows: [[1]]}} = Ch.query(conn, "select {a:UInt64}", %{"a" => 1}) + test "uints", ctx do + assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:UInt8}", %{"a" => 1}) + assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:UInt16}", %{"a" => 1}) + assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:UInt32}", %{"a" => 1}) + assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:UInt64}", %{"a" => 1}) assert {:ok, %{num_rows: 1, rows: [[1]]}} = - Ch.query(conn, "select {a:UInt128}", %{"a" => 1}) + query(ctx, "select {a:UInt128}", %{"a" => 1}) assert {:ok, %{num_rows: 1, rows: [[1]]}} = - Ch.query(conn, "select {a:UInt256}", %{"a" => 1}) + query(ctx, "select {a:UInt256}", %{"a" => 1}) end - test "fixed string", %{conn: conn} do + test "fixed string", ctx do assert {:ok, %{num_rows: 1, rows: [[<<0, 0>>]]}} = - Ch.query(conn, "select {a:FixedString(2)}", %{"a" => ""}) + query(ctx, "select {a:FixedString(2)}", %{"a" => ""}) assert {:ok, %{num_rows: 1, rows: [["a" <> <<0>>]]}} = - Ch.query(conn, "select {a:FixedString(2)}", %{"a" => "a"}) + query(ctx, "select {a:FixedString(2)}", %{"a" => "a"}) assert {:ok, %{num_rows: 1, rows: [["aa"]]}} = - Ch.query(conn, "select {a:FixedString(2)}", %{"a" => "aa"}) + query(ctx, "select {a:FixedString(2)}", %{"a" => "aa"}) assert {:ok, %{num_rows: 1, rows: [["aaaaa"]]}} = - Ch.query(conn, "select {a:FixedString(5)}", %{"a" => "aaaaa"}) + query(ctx, "select {a:FixedString(5)}", %{"a" => "aaaaa"}) - Ch.query!(conn, "create table fixed_string_t(a FixedString(3)) engine = Memory") + query!(ctx, "create table fixed_string_t(a FixedString(3)) engine = Memory") + + on_exit(fn -> + Ch.Test.query("drop table fixed_string_t", [], database: Ch.Test.database()) + end) assert {:ok, %{num_rows: 4}} = - Ch.query( - conn, + query( + ctx, "insert into fixed_string_t(a) format RowBinary", [ [""], @@ -455,7 +482,7 @@ defmodule Ch.ConnectionTest do types: ["FixedString(3)"] ) - assert Ch.query!(conn, "select * from fixed_string_t").rows == [ + assert query!(ctx, "select * from fixed_string_t").rows == [ [<<0, 0, 0>>], ["a" <> <<0, 0>>], ["aa" <> <<0>>], @@ -463,32 +490,36 @@ defmodule Ch.ConnectionTest do ] end - test "decimal", %{conn: conn} do + test "decimal", ctx do assert {:ok, %{num_rows: 1, rows: [row]}} = - Ch.query(conn, "SELECT toDecimal32(2, 4) AS x, x / 3, toTypeName(x)") + query(ctx, "SELECT toDecimal32(2, 4) AS x, x / 3, toTypeName(x)") assert row == [Decimal.new("2.0000"), Decimal.new("0.6666"), "Decimal(9, 4)"] assert {:ok, %{num_rows: 1, rows: [row]}} = - Ch.query(conn, "SELECT toDecimal64(2, 4) AS x, x / 3, toTypeName(x)") + query(ctx, "SELECT toDecimal64(2, 4) AS x, x / 3, toTypeName(x)") assert row == [Decimal.new("2.0000"), Decimal.new("0.6666"), "Decimal(18, 4)"] assert {:ok, %{num_rows: 1, rows: [row]}} = - Ch.query(conn, "SELECT toDecimal128(2, 4) AS x, x / 3, toTypeName(x)") + query(ctx, "SELECT toDecimal128(2, 4) AS x, x / 3, toTypeName(x)") assert row == [Decimal.new("2.0000"), Decimal.new("0.6666"), "Decimal(38, 4)"] assert {:ok, %{num_rows: 1, rows: [row]}} = - Ch.query(conn, "SELECT toDecimal256(2, 4) AS x, x / 3, toTypeName(x)") + query(ctx, "SELECT toDecimal256(2, 4) AS x, x / 3, toTypeName(x)") assert row == [Decimal.new("2.0000"), Decimal.new("0.6666"), "Decimal(76, 4)"] - Ch.query!(conn, "create table decimal_t(d Decimal32(4)) engine = Memory") + query!(ctx, "create table decimal_t(d Decimal32(4)) engine = Memory") + + on_exit(fn -> + Ch.Test.query("drop table decimal_t", [], database: Ch.Test.database()) + end) assert %{num_rows: 3} = - Ch.query!( - conn, + query!( + ctx, "insert into decimal_t(d) format RowBinary", _rows = [ [Decimal.new("2.66")], @@ -498,35 +529,39 @@ defmodule Ch.ConnectionTest do types: ["Decimal32(4)"] ) - assert Ch.query!(conn, "select * from decimal_t").rows == [ + assert query!(ctx, "select * from decimal_t").rows == [ [Decimal.new("2.6600")], [Decimal.new("2.6666")], [Decimal.new("2.6667")] ] end - test "boolean", %{conn: conn} do + test "boolean", ctx do assert {:ok, %{num_rows: 1, rows: [[true, "Bool"]]}} = - Ch.query(conn, "select true as col, toTypeName(col)") + query(ctx, "select true as col, toTypeName(col)") assert {:ok, %{num_rows: 1, rows: [[1, "UInt8"]]}} = - Ch.query(conn, "select true == 1 as col, toTypeName(col)") + query(ctx, "select true == 1 as col, toTypeName(col)") - assert {:ok, %{num_rows: 1, rows: [[true, false]]}} = Ch.query(conn, "select true, false") + assert {:ok, %{num_rows: 1, rows: [[true, false]]}} = query(ctx, "select true, false") - Ch.query!(conn, "create table test_bool(A Int64, B Bool) engine = Memory") + query!(ctx, "create table test_bool(A Int64, B Bool) engine = Memory") - Ch.query!(conn, "INSERT INTO test_bool VALUES (1, true),(2,0)") + on_exit(fn -> + Ch.Test.query("drop table test_bool", [], database: Ch.Test.database()) + end) - Ch.query!( - conn, + query!(ctx, "INSERT INTO test_bool VALUES (1, true),(2,0)") + + query!( + ctx, "insert into test_bool(A, B) format RowBinary", _rows = [[3, true], [4, false]], types: ["Int64", "Bool"] ) # anything > 0 is `true`, here `2` is `true` - Ch.query!(conn, "insert into test_bool(A, B) values (5, 2)") + query!(ctx, "insert into test_bool(A, B) values (5, 2)") assert %{ rows: [ @@ -536,15 +571,15 @@ defmodule Ch.ConnectionTest do [4, false, 0], [5, true, 5] ] - } = Ch.query!(conn, "SELECT *, A * B FROM test_bool ORDER BY A") + } = query!(ctx, "SELECT *, A * B FROM test_bool ORDER BY A") end - test "uuid", %{conn: conn} do + test "uuid", ctx do assert {:ok, %{num_rows: 1, rows: [[<<_::16-bytes>>]]}} = - Ch.query(conn, "select generateUUIDv4()") + query(ctx, "select generateUUIDv4()") assert {:ok, %{num_rows: 1, rows: [[uuid, "417ddc5d-e556-4d27-95dd-a34d84e46a50"]]}} = - Ch.query(conn, "select {uuid:UUID} as u, toString(u)", %{ + query(ctx, "select {uuid:UUID} as u, toString(u)", %{ "uuid" => "417ddc5d-e556-4d27-95dd-a34d84e46a50" }) @@ -553,16 +588,21 @@ defmodule Ch.ConnectionTest do |> String.replace("-", "") |> Base.decode16!(case: :lower) - Ch.query!(conn, " CREATE TABLE t_uuid (x UUID, y String) ENGINE Memory") - Ch.query!(conn, "INSERT INTO t_uuid SELECT generateUUIDv4(), 'Example 1'") + query!(ctx, " CREATE TABLE t_uuid (x UUID, y String) ENGINE Memory") + + on_exit(fn -> + Ch.Test.query("drop table t_uuid", [], database: Ch.Test.database()) + end) + + query!(ctx, "INSERT INTO t_uuid SELECT generateUUIDv4(), 'Example 1'") assert {:ok, %{num_rows: 1, rows: [[<<_::16-bytes>>, "Example 1"]]}} = - Ch.query(conn, "SELECT * FROM t_uuid") + query(ctx, "SELECT * FROM t_uuid") - Ch.query!(conn, "INSERT INTO t_uuid (y) VALUES ('Example 2')") + query!(ctx, "INSERT INTO t_uuid (y) VALUES ('Example 2')") - Ch.query!( - conn, + query!( + ctx, "insert into t_uuid(x,y) format RowBinary", _rows = [[uuid, "Example 3"]], types: ["UUID", "String"] @@ -576,27 +616,27 @@ defmodule Ch.ConnectionTest do [<<0::128>>, "Example 2"], [^uuid, "Example 3"] ] - }} = Ch.query(conn, "SELECT * FROM t_uuid ORDER BY y") + }} = query(ctx, "SELECT * FROM t_uuid ORDER BY y") end @tag :skip - test "json", %{conn: conn} do + test "json", ctx do settings = [allow_experimental_object_type: 1] - Ch.query!(conn, "CREATE TABLE json(o JSON) ENGINE = Memory", [], settings: settings) + query!(ctx, "CREATE TABLE json(o JSON) ENGINE = Memory", [], settings: settings) - Ch.query!(conn, ~s|INSERT INTO json VALUES ('{"a": 1, "b": { "c": 2, "d": [1, 2, 3] }}')|) + query!(ctx, ~s|INSERT INTO json VALUES ('{"a": 1, "b": { "c": 2, "d": [1, 2, 3] }}')|) - assert Ch.query!(conn, "SELECT o.a, o.b.c, o.b.d[3] FROM json").rows == [[1, 2, 3]] + assert query!(ctx, "SELECT o.a, o.b.c, o.b.d[3] FROM json").rows == [[1, 2, 3]] # named tuples are not supported yet - assert_raise ArgumentError, fn -> Ch.query!(conn, "SELECT o FROM json") end + assert_raise ArgumentError, fn -> query!(ctx, "SELECT o FROM json") end end @tag :json - test "json as string", %{conn: conn} do + test "json as string", ctx do # after v25 ClickHouse started rendering numbers in JSON as strings - [[version]] = Ch.query!(conn, "select version()").rows + [[version]] = query!(ctx, "select version()").rows parse_version = fn version -> version |> String.split(".") |> Enum.map(&String.to_integer/1) @@ -618,16 +658,20 @@ defmodule Ch.ConnectionTest do ] end - assert Ch.query!(conn, ~s|select '{"answer":42}'::JSON::String|, [], + assert query!(ctx, ~s|select '{"answer":42}'::JSON::String|, [], settings: [enable_json_type: 1] ).rows == expected1 - Ch.query!(conn, "CREATE TABLE test_json_as_string(json JSON) ENGINE = Memory", [], + query!(ctx, "CREATE TABLE test_json_as_string(json JSON) ENGINE = Memory", [], settings: [enable_json_type: 1] ) - Ch.query!( - conn, + on_exit(fn -> + Ch.Test.query("DROP TABLE test_json_as_string", [], database: Ch.Test.database()) + end) + + query!( + ctx, "INSERT INTO test_json_as_string(json) FORMAT RowBinary", _rows = [[Jason.encode_to_iodata!(%{"a" => 42})], [Jason.encode_to_iodata!(%{"b" => 10})]], types: [:string], @@ -637,50 +681,54 @@ defmodule Ch.ConnectionTest do ] ) - assert Ch.query!(conn, "select json::String from test_json_as_string", [], + assert query!(ctx, "select json::String from test_json_as_string", [], settings: [enable_json_type: 1] ).rows == expected2 end # TODO enum16 - test "enum8", %{conn: conn} do + test "enum8", ctx do assert {:ok, %{num_rows: 1, rows: [["Enum8('a' = 1, 'b' = 2)"]]}} = - Ch.query(conn, "SELECT toTypeName(CAST('a', 'Enum(\\'a\\' = 1, \\'b\\' = 2)'))") + query(ctx, "SELECT toTypeName(CAST('a', 'Enum(\\'a\\' = 1, \\'b\\' = 2)'))") assert {:ok, %{num_rows: 1, rows: [["a"]]}} = - Ch.query(conn, "SELECT CAST('a', 'Enum(\\'a\\' = 1, \\'b\\' = 2)')") + query(ctx, "SELECT CAST('a', 'Enum(\\'a\\' = 1, \\'b\\' = 2)')") assert {:ok, %{num_rows: 1, rows: [["b"]]}} = - Ch.query(conn, "select {enum:Enum('a' = 1, 'b' = 2)}", %{"enum" => "b"}) + query(ctx, "select {enum:Enum('a' = 1, 'b' = 2)}", %{"enum" => "b"}) assert {:ok, %{num_rows: 1, rows: [["b"]]}} = - Ch.query(conn, "select {enum:Enum('a' = 1, 'b' = 2)}", %{"enum" => 2}) + query(ctx, "select {enum:Enum('a' = 1, 'b' = 2)}", %{"enum" => 2}) assert {:ok, %{num_rows: 1, rows: [["b"]]}} = - Ch.query(conn, "select {enum:Enum16('a' = 1, 'b' = 2)}", %{"enum" => 2}) + query(ctx, "select {enum:Enum16('a' = 1, 'b' = 2)}", %{"enum" => 2}) - Ch.query!( - conn, + query!( + ctx, "CREATE TABLE t_enum(i UInt8, x Enum('hello' = 1, 'world' = 2)) ENGINE Memory" ) - Ch.query!(conn, "INSERT INTO t_enum VALUES (0, 'hello'), (1, 'world'), (2, 'hello')") + on_exit(fn -> + Ch.Test.query("DROP TABLE t_enum", [], database: Ch.Test.database()) + end) - assert Ch.query!(conn, "SELECT *, CAST(x, 'Int8') FROM t_enum ORDER BY i").rows == [ + query!(ctx, "INSERT INTO t_enum VALUES (0, 'hello'), (1, 'world'), (2, 'hello')") + + assert query!(ctx, "SELECT *, CAST(x, 'Int8') FROM t_enum ORDER BY i").rows == [ [0, "hello", 1], [1, "world", 2], [2, "hello", 1] ] - Ch.query!( - conn, + query!( + ctx, "INSERT INTO t_enum(i, x) FORMAT RowBinary", _rows = [[3, "hello"], [4, "world"], [5, 1], [6, 2]], types: ["UInt8", "Enum8('hello' = 1, 'world' = 2)"] ) - assert Ch.query!(conn, "SELECT *, CAST(x, 'Int8') FROM t_enum ORDER BY i").rows == [ + assert query!(ctx, "SELECT *, CAST(x, 'Int8') FROM t_enum ORDER BY i").rows == [ [0, "hello", 1], [1, "world", 2], [2, "hello", 1], @@ -693,28 +741,32 @@ defmodule Ch.ConnectionTest do # TODO nil enum end - test "map", %{conn: conn} do - assert Ch.query!( - conn, + test "map", ctx do + assert query!( + ctx, "SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map" ).rows == [[%{1 => "Ready", 2 => "Steady", 3 => "Go"}]] - assert Ch.query!(conn, "select {map:Map(String, UInt8)}", %{ + assert query!(ctx, "select {map:Map(String, UInt8)}", %{ "map" => %{"pg" => 13, "hello" => 100} }).rows == [[%{"hello" => 100, "pg" => 13}]] - Ch.query!(conn, "CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory") + query!(ctx, "CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory") + + on_exit(fn -> + Ch.Test.query("DROP TABLE table_map", [], database: Ch.Test.database()) + end) - Ch.query!( - conn, + query!( + ctx, "INSERT INTO table_map VALUES ({'key1':1, 'key2':10}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30})" ) - assert Ch.query!(conn, "SELECT a['key2'] FROM table_map").rows == [[10], [20], [30]] + assert query!(ctx, "SELECT a['key2'] FROM table_map").rows == [[10], [20], [30]] - assert Ch.query!(conn, "INSERT INTO table_map VALUES ({'key3':100}), ({})") + assert query!(ctx, "INSERT INTO table_map VALUES ({'key3':100}), ({})") - assert Ch.query!(conn, "SELECT a['key3'] FROM table_map ORDER BY 1 DESC").rows == [ + assert query!(ctx, "SELECT a['key3'] FROM table_map ORDER BY 1 DESC").rows == [ [100], [0], [0], @@ -722,8 +774,8 @@ defmodule Ch.ConnectionTest do [0] ] - assert Ch.query!( - conn, + assert query!( + ctx, "INSERT INTO table_map FORMAT RowBinary", _rows = [ [%{"key10" => 20, "key20" => 40}], @@ -738,7 +790,7 @@ defmodule Ch.ConnectionTest do types: ["Map(String, UInt64)"] ) - assert Ch.query!(conn, "SELECT * FROM table_map ORDER BY a ASC").rows == [ + assert query!(ctx, "SELECT * FROM table_map ORDER BY a ASC").rows == [ [%{}], [%{}], [%{}], @@ -752,36 +804,40 @@ defmodule Ch.ConnectionTest do ] end - test "tuple", %{conn: conn} do - assert Ch.query!(conn, "SELECT tuple(1,'a') AS x, toTypeName(x)").rows == [ + test "tuple", ctx do + assert query!(ctx, "SELECT tuple(1,'a') AS x, toTypeName(x)").rows == [ [{1, "a"}, "Tuple(UInt8, String)"] ] - assert Ch.query!(conn, "SELECT {$0:Tuple(Int8, String)}", [{-1, "abs"}]).rows == [ + assert query!(ctx, "SELECT {$0:Tuple(Int8, String)}", [{-1, "abs"}]).rows == [ [{-1, "abs"}] ] - assert Ch.query!(conn, "SELECT tuple('a') AS x").rows == [[{"a"}]] + assert query!(ctx, "SELECT tuple('a') AS x").rows == [[{"a"}]] - assert Ch.query!(conn, "SELECT tuple(1, NULL) AS x, toTypeName(x)").rows == [ + assert query!(ctx, "SELECT tuple(1, NULL) AS x, toTypeName(x)").rows == [ [{1, nil}, "Tuple(UInt8, Nullable(Nothing))"] ] # TODO named tuples - Ch.query!(conn, "CREATE TABLE tuples_t (`a` Tuple(String, Int64)) ENGINE = Memory") + query!(ctx, "CREATE TABLE tuples_t (`a` Tuple(String, Int64)) ENGINE = Memory") + + on_exit(fn -> + Ch.Test.query("DROP TABLE tuples_t", [], database: Ch.Test.database()) + end) assert %{num_rows: 2} = - Ch.query!(conn, "INSERT INTO tuples_t VALUES (('y', 10)), (('x',-10))") + query!(ctx, "INSERT INTO tuples_t VALUES (('y', 10)), (('x',-10))") assert %{num_rows: 2} = - Ch.query!( - conn, + query!( + ctx, "INSERT INTO tuples_t FORMAT RowBinary", _rows = [[{"a", 20}], [{"b", 30}]], types: ["Tuple(String, Int64)"] ) - assert Ch.query!(conn, "SELECT a FROM tuples_t ORDER BY a.1 ASC").rows == [ + assert query!(ctx, "SELECT a FROM tuples_t ORDER BY a.1 ASC").rows == [ [{"a", 20}], [{"b", 30}], [{"x", -10}], @@ -789,16 +845,20 @@ defmodule Ch.ConnectionTest do ] end - test "datetime", %{conn: conn} do - Ch.query!( - conn, + test "datetime", ctx do + query!( + ctx, "CREATE TABLE dt(`timestamp` DateTime('Asia/Istanbul'), `event_id` UInt8) ENGINE = Memory" ) - Ch.query!(conn, "INSERT INTO dt Values (1546300800, 1), ('2019-01-01 00:00:00', 2)") + on_exit(fn -> + Ch.Test.query("DROP TABLE dt", [], database: Ch.Test.database()) + end) + + query!(ctx, "INSERT INTO dt Values (1546300800, 1), ('2019-01-01 00:00:00', 2)") assert {:ok, %{num_rows: 2, rows: rows}} = - Ch.query(conn, "SELECT *, toString(timestamp) FROM dt") + query(ctx, "SELECT *, toString(timestamp) FROM dt") assert rows == [ [ @@ -820,7 +880,7 @@ defmodule Ch.ConnectionTest do # https://kb.altinity.com/altinity-kb-queries-and-syntax/time-zones/ assert {:ok, %{num_rows: 1, rows: [[naive_datetime, "2022-12-12 12:00:00"]], headers: headers}} = - Ch.query(conn, "select {$0:DateTime} as d, toString(d)", [naive_noon]) + query(ctx, "select {$0:DateTime} as d, toString(d)", [naive_noon]) # to make this test pass for contributors with non UTC timezone we perform the same steps as ClickHouse # i.e. we give server timezone to the naive datetime and shift it to UTC before comparing with the result @@ -833,10 +893,10 @@ defmodule Ch.ConnectionTest do |> DateTime.to_naive() assert {:ok, %{num_rows: 1, rows: [[~U[2022-12-12 12:00:00Z], "2022-12-12 12:00:00"]]}} = - Ch.query(conn, "select {$0:DateTime('UTC')} as d, toString(d)", [naive_noon]) + query(ctx, "select {$0:DateTime('UTC')} as d, toString(d)", [naive_noon]) assert {:ok, %{num_rows: 1, rows: rows}} = - Ch.query(conn, "select {$0:DateTime('Asia/Bangkok')} as d, toString(d)", [ + query(ctx, "select {$0:DateTime('Asia/Bangkok')} as d, toString(d)", [ naive_noon ]) @@ -853,20 +913,25 @@ defmodule Ch.ConnectionTest do on_exit(fn -> Calendar.put_time_zone_database(prev_tz_db) end) assert_raise ArgumentError, ~r/:utc_only_time_zone_database/, fn -> - Ch.query(conn, "select {$0:DateTime('Asia/Tokyo')}", [naive_noon]) + query(ctx, "select {$0:DateTime('Asia/Tokyo')}", [naive_noon]) end end # TODO are negatives correct? what's the range? - test "date32", %{conn: conn} do - Ch.query!(conn, "CREATE TABLE new(`timestamp` Date32, `event_id` UInt8) ENGINE = Memory;") - Ch.query!(conn, "INSERT INTO new VALUES (4102444800, 1), ('2100-01-01', 2)") + test "date32", ctx do + query!(ctx, "CREATE TABLE new(`timestamp` Date32, `event_id` UInt8) ENGINE = Memory;") + + on_exit(fn -> + Ch.Test.query("DROP TABLE new", [], database: Ch.Test.database()) + end) + + query!(ctx, "INSERT INTO new VALUES (4102444800, 1), ('2100-01-01', 2)") assert {:ok, %{ num_rows: 2, rows: [first_event, [~D[2100-01-01], 2, "2100-01-01"]] - }} = Ch.query(conn, "SELECT *, toString(timestamp) FROM new") + }} = query(ctx, "SELECT *, toString(timestamp) FROM new") # TODO use timezone info to be more exact assert first_event in [ @@ -875,18 +940,18 @@ defmodule Ch.ConnectionTest do ] assert {:ok, %{num_rows: 1, rows: [[~D[1900-01-01], "1900-01-01"]]}} = - Ch.query(conn, "select {$0:Date32} as d, toString(d)", [~D[1900-01-01]]) + query(ctx, "select {$0:Date32} as d, toString(d)", [~D[1900-01-01]]) # max assert {:ok, %{num_rows: 1, rows: [[~D[2299-12-31], "2299-12-31"]]}} = - Ch.query(conn, "select {$0:Date32} as d, toString(d)", [~D[2299-12-31]]) + query(ctx, "select {$0:Date32} as d, toString(d)", [~D[2299-12-31]]) # min assert {:ok, %{num_rows: 1, rows: [[~D[1900-01-01], "1900-01-01"]]}} = - Ch.query(conn, "select {$0:Date32} as d, toString(d)", [~D[1900-01-01]]) + query(ctx, "select {$0:Date32} as d, toString(d)", [~D[1900-01-01]]) - Ch.query!( - conn, + query!( + ctx, "insert into new(timestamp, event_id) format RowBinary", _rows = [[~D[1960-01-01], 3]], types: ["Date32", "UInt8"] @@ -899,7 +964,7 @@ defmodule Ch.ConnectionTest do [~D[2100-01-01], 2, "2100-01-01"], [~D[1960-01-01], 3, "1960-01-01"] ] - } = Ch.query!(conn, "SELECT *, toString(timestamp) FROM new ORDER BY event_id") + } = query!(ctx, "SELECT *, toString(timestamp) FROM new ORDER BY event_id") # TODO use timezone info to be more exact assert first_event in [ @@ -908,19 +973,23 @@ defmodule Ch.ConnectionTest do ] assert %{num_rows: 1, rows: [[3]]} = - Ch.query!(conn, "SELECT event_id FROM new WHERE timestamp = '1960-01-01'") + query!(ctx, "SELECT event_id FROM new WHERE timestamp = '1960-01-01'") end # https://clickhouse.com/docs/sql-reference/data-types/time @tag :time - test "time", %{conn: conn} do + test "time", ctx do settings = [enable_time_time64_type: 1] - Ch.query!(conn, "CREATE TABLE time_t(`time` Time, `event_id` UInt8) ENGINE = Memory", [], + query!(ctx, "CREATE TABLE time_t(`time` Time, `event_id` UInt8) ENGINE = Memory", [], settings: settings ) - Ch.query!(conn, "INSERT INTO time_t VALUES ('100:00:00', 1), (12453, 2)", [], + on_exit(fn -> + Ch.Test.query("DROP TABLE time_t", [], settings: settings, database: Ch.Test.database()) + end) + + query!(ctx, "INSERT INTO time_t VALUES ('100:00:00', 1), (12453, 2)", [], settings: settings ) @@ -930,10 +999,10 @@ defmodule Ch.ConnectionTest do assert_raise ArgumentError, "ClickHouse Time value 3.6e5 (seconds) is out of Elixir's Time range (00:00:00.000000 - 23:59:59.999999)", - fn -> Ch.query!(conn, "select * from time_t", [], settings: settings) end + fn -> query!(ctx, "select * from time_t", [], settings: settings) end - Ch.query!( - conn, + query!( + ctx, "INSERT INTO time_t(time, event_id) FORMAT RowBinary", _rows = [ [~T[00:00:00], 3], @@ -944,7 +1013,7 @@ defmodule Ch.ConnectionTest do types: ["Time", "UInt8"] ) - assert Ch.query!(conn, "select * from time_t where event_id > 1 order by event_id", [], + assert query!(ctx, "select * from time_t where event_id > 1 order by event_id", [], settings: settings ).rows == [[~T[03:27:33], 2], [~T[00:00:00], 3], [~T[12:34:56], 4], [~T[23:59:59], 5]] @@ -952,18 +1021,25 @@ defmodule Ch.ConnectionTest do # https://clickhouse.com/docs/sql-reference/data-types/time64 @tag :time - test "Time64(3)", %{conn: conn} do + test "Time64(3)", ctx do settings = [enable_time_time64_type: 1] - Ch.query!( - conn, + query!( + ctx, "CREATE TABLE time64_3_t(`time` Time64(3), `event_id` UInt8) ENGINE = Memory", [], settings: settings ) - Ch.query!( - conn, + on_exit(fn -> + Ch.Test.query("DROP TABLE time64_3_t", [], + settings: settings, + database: Ch.Test.database() + ) + end) + + query!( + ctx, "INSERT INTO time64_3_t VALUES (15463123, 1), (154600.123, 2), ('100:00:00', 3);", [], settings: settings @@ -975,10 +1051,10 @@ defmodule Ch.ConnectionTest do assert_raise ArgumentError, "ClickHouse Time value 154600.123 (seconds) is out of Elixir's Time range (00:00:00.000000 - 23:59:59.999999)", - fn -> Ch.query!(conn, "select * from time64_3_t", [], settings: settings) end + fn -> query!(ctx, "select * from time64_3_t", [], settings: settings) end - Ch.query!( - conn, + query!( + ctx, "INSERT INTO time64_3_t(time, event_id) FORMAT RowBinary", _rows = [ [~T[00:00:00.000000], 4], @@ -991,8 +1067,8 @@ defmodule Ch.ConnectionTest do types: ["Time64(3)", "UInt8"] ) - assert Ch.query!( - conn, + assert query!( + ctx, "select * from time64_3_t where time < {max_elixir_time:Time64(6)} order by event_id", %{"max_elixir_time" => ~T[23:59:59.999999]}, settings: settings @@ -1008,18 +1084,25 @@ defmodule Ch.ConnectionTest do end @tag :time - test "Time64(6)", %{conn: conn} do + test "Time64(6)", ctx do settings = [enable_time_time64_type: 1] - Ch.query!( - conn, + query!( + ctx, "CREATE TABLE time64_6_t(`time` Time64(6), `event_id` UInt8) ENGINE = Memory", [], settings: settings ) - Ch.query!( - conn, + on_exit(fn -> + Ch.Test.query("DROP TABLE time64_6_t", [], + settings: settings, + database: Ch.Test.database() + ) + end) + + query!( + ctx, "INSERT INTO time64_6_t(time, event_id) FORMAT RowBinary", _rows = [ [~T[00:00:00.000000], 1], @@ -1032,8 +1115,8 @@ defmodule Ch.ConnectionTest do types: ["Time64(6)", "UInt8"] ) - assert Ch.query!( - conn, + assert query!( + ctx, "select * from time64_6_t order by event_id", [], settings: settings @@ -1048,18 +1131,25 @@ defmodule Ch.ConnectionTest do end @tag :time - test "Time64(9)", %{conn: conn} do + test "Time64(9)", ctx do settings = [enable_time_time64_type: 1] - Ch.query!( - conn, + query!( + ctx, "CREATE TABLE time64_9_t(`time` Time64(9), `event_id` UInt8) ENGINE = Memory", [], settings: settings ) - Ch.query!( - conn, + on_exit(fn -> + Ch.Test.query("DROP TABLE time64_9_t", [], + settings: settings, + database: Ch.Test.database() + ) + end) + + query!( + ctx, "INSERT INTO time64_9_t(time, event_id) FORMAT RowBinary", _rows = [ [~T[00:00:00.000000], 1], @@ -1072,8 +1162,8 @@ defmodule Ch.ConnectionTest do types: ["Time64(9)", "UInt8"] ) - assert Ch.query!( - conn, + assert query!( + ctx, "select * from time64_9_t order by event_id", [], settings: settings @@ -1087,19 +1177,23 @@ defmodule Ch.ConnectionTest do ] end - test "datetime64", %{conn: conn} do - Ch.query!( - conn, + test "datetime64", ctx do + query!( + ctx, "CREATE TABLE datetime64_t(`timestamp` DateTime64(3, 'Asia/Istanbul'), `event_id` UInt8) ENGINE = Memory" ) - Ch.query!( - conn, + on_exit(fn -> + Ch.Test.query("DROP TABLE datetime64_t", [], database: Ch.Test.database()) + end) + + query!( + ctx, "INSERT INTO datetime64_t Values (1546300800123, 1), (1546300800.123, 2), ('2019-01-01 00:00:00', 3)" ) assert {:ok, %{num_rows: 3, rows: rows}} = - Ch.query(conn, "SELECT *, toString(timestamp) FROM datetime64_t") + query(ctx, "SELECT *, toString(timestamp) FROM datetime64_t") assert rows == [ [ @@ -1119,8 +1213,8 @@ defmodule Ch.ConnectionTest do ] ] - Ch.query!( - conn, + query!( + ctx, "insert into datetime64_t(event_id, timestamp) format RowBinary", _rows = [ [4, ~N[2021-01-01 12:00:00.123456]], @@ -1130,8 +1224,8 @@ defmodule Ch.ConnectionTest do ) assert {:ok, %{num_rows: 2, rows: rows}} = - Ch.query( - conn, + query( + ctx, "SELECT *, toString(timestamp) FROM datetime64_t WHERE timestamp > '2020-01-01'" ) @@ -1155,7 +1249,7 @@ defmodule Ch.ConnectionTest do # see https://clickhouse.com/docs/en/sql-reference/data-types/datetime # https://kb.altinity.com/altinity-kb-queries-and-syntax/time-zones/ assert {:ok, %{num_rows: 1, rows: [[naive_datetime]], headers: headers}} = - Ch.query(conn, "select {$0:DateTime64(#{precision})}", [naive_noon]) + query(ctx, "select {$0:DateTime64(#{precision})}", [naive_noon]) # to make this test pass for contributors with non UTC timezone we perform the same steps as ClickHouse # i.e. we give server timezone to the naive datetime and shift it to UTC before comparing with the result @@ -1172,18 +1266,18 @@ defmodule Ch.ConnectionTest do assert {:ok, %{num_rows: 1, rows: [[~U[2022-01-01 12:00:00.123Z], "2022-01-01 12:00:00.123"]]}} = - Ch.query(conn, "select {dt:DateTime64(3,'UTC')} as d, toString(d)", %{ + query(ctx, "select {dt:DateTime64(3,'UTC')} as d, toString(d)", %{ "dt" => ~N[2022-01-01 12:00:00.123] }) assert {:ok, %{num_rows: 1, rows: [[~U[1900-01-01 12:00:00.123Z], "1900-01-01 12:00:00.123"]]}} = - Ch.query(conn, "select {dt:DateTime64(3,'UTC')} as d, toString(d)", %{ + query(ctx, "select {dt:DateTime64(3,'UTC')} as d, toString(d)", %{ "dt" => ~N[1900-01-01 12:00:00.123] }) assert {:ok, %{num_rows: 1, rows: [row]}} = - Ch.query(conn, "select {dt:DateTime64(3,'Asia/Bangkok')} as d, toString(d)", %{ + query(ctx, "select {dt:DateTime64(3,'Asia/Bangkok')} as d, toString(d)", %{ "dt" => ~N[2022-01-01 12:00:00.123] }) @@ -1193,37 +1287,41 @@ defmodule Ch.ConnectionTest do ] end - test "nullable", %{conn: conn} do - Ch.query!( - conn, + test "nullable", ctx do + query!( + ctx, "CREATE TABLE nullable (`n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple()" ) - Ch.query!(conn, "INSERT INTO nullable VALUES (1) (NULL) (2) (NULL)") + on_exit(fn -> + Ch.Test.query("DROP TABLE nullable", [], database: Ch.Test.database()) + end) + + query!(ctx, "INSERT INTO nullable VALUES (1) (NULL) (2) (NULL)") assert {:ok, %{num_rows: 4, rows: [[0], [1], [0], [1]]}} = - Ch.query(conn, "SELECT n.null FROM nullable") + query(ctx, "SELECT n.null FROM nullable") assert {:ok, %{num_rows: 4, rows: [[1], [nil], [2], [nil]]}} = - Ch.query(conn, "SELECT n FROM nullable") + query(ctx, "SELECT n FROM nullable") # weird thing about nullables is that, similar to bool, in binary format, any byte larger than 0 is `null` assert {:ok, %{num_rows: 5}} = - Ch.query( - conn, + query( + ctx, "insert into nullable format RowBinary", <<1, 2, 3, 4, 5>>, encode: false ) assert %{num_rows: 1, rows: [[count]]} = - Ch.query!(conn, "select count(*) from nullable where n is null") + query!(ctx, "select count(*) from nullable where n is null") assert count == 2 + 5 end - test "nullable + default", %{conn: conn} do - Ch.query!(conn, """ + test "nullable + default", ctx do + query!(ctx, """ CREATE TABLE ch_nulls ( a UInt8, b UInt8 NULL, @@ -1232,28 +1330,36 @@ defmodule Ch.ConnectionTest do ) ENGINE Memory """) - Ch.query!( - conn, + on_exit(fn -> + Ch.Test.query("DROP TABLE ch_nulls", [], database: Ch.Test.database()) + end) + + query!( + ctx, "INSERT INTO ch_nulls(a, b, c, d) FORMAT RowBinary", [[nil, nil, nil, nil]], types: ["UInt8", "Nullable(UInt8)", "UInt8", "Nullable(UInt8)"] ) # default is ignored... - assert Ch.query!(conn, "SELECT * FROM ch_nulls").rows == [[0, nil, 0, nil]] + assert query!(ctx, "SELECT * FROM ch_nulls").rows == [[0, nil, 0, nil]] end # based on https://github.com/ClickHouse/clickhouse-java/pull/1345/files - test "nullable + input() + default", %{conn: conn} do - Ch.query!(conn, """ + test "nullable + input() + default", ctx do + query!(ctx, """ CREATE TABLE test_insert_default_value( n Int32, s String DEFAULT 'secret' ) ENGINE Memory """) - Ch.query!( - conn, + on_exit(fn -> + Ch.Test.query("DROP TABLE test_insert_default_value", [], database: Ch.Test.database()) + end) + + query!( + ctx, """ INSERT INTO test_insert_default_value SELECT id, name @@ -1264,38 +1370,43 @@ defmodule Ch.ConnectionTest do types: ["UInt32", "Nullable(String)"] ) - assert Ch.query!(conn, "SELECT * FROM test_insert_default_value ORDER BY n").rows == + assert query!(ctx, "SELECT * FROM test_insert_default_value ORDER BY n").rows == [ [-1, "secret"], [1, "secret"] ] end - test "can decode casted Point", %{conn: conn} do - assert Ch.query!(conn, "select cast((0, 1) as Point)").rows == [ + test "can decode casted Point", ctx do + assert query!(ctx, "select cast((0, 1) as Point)").rows == [ _row = [_point = {0.0, 1.0}] ] end - test "can encode and then decode Point in query params", %{conn: conn} do - assert Ch.query!(conn, "select {$0:Point}", [{10, 10}]).rows == [ + test "can encode and then decode Point in query params", ctx do + assert query!(ctx, "select {$0:Point}", [{10, 10}]).rows == [ _row = [_point = {10.0, 10.0}] ] end - test "can insert and select Point", %{conn: conn} do - Ch.query!(conn, "CREATE TABLE geo_point (p Point) ENGINE = Memory()") - Ch.query!(conn, "INSERT INTO geo_point VALUES((10, 10))") - Ch.query!(conn, "INSERT INTO geo_point FORMAT RowBinary", [[{20, 20}]], types: ["Point"]) + test "can insert and select Point", ctx do + query!(ctx, "CREATE TABLE geo_point (p Point) ENGINE = Memory()") + + on_exit(fn -> + Ch.Test.query("DROP TABLE geo_point", [], database: Ch.Test.database()) + end) + + query!(ctx, "INSERT INTO geo_point VALUES((10, 10))") + query!(ctx, "INSERT INTO geo_point FORMAT RowBinary", [[{20, 20}]], types: ["Point"]) - assert Ch.query!(conn, "SELECT p, toTypeName(p) FROM geo_point ORDER BY p ASC").rows == [ + assert query!(ctx, "SELECT p, toTypeName(p) FROM geo_point ORDER BY p ASC").rows == [ [{10.0, 10.0}, "Point"], [{20.0, 20.0}, "Point"] ] # to make our RowBinary is not garbage in garbage out we also test a text format response - assert Ch.query!( - conn, + assert query!( + ctx, "SELECT p, toTypeName(p) FROM geo_point ORDER BY p ASC FORMAT JSONCompact" ).rows |> Jason.decode!() @@ -1305,31 +1416,36 @@ defmodule Ch.ConnectionTest do ] end - test "can decode casted Ring", %{conn: conn} do + test "can decode casted Ring", ctx do ring = [{0.0, 1.0}, {10.0, 3.0}] - assert Ch.query!(conn, "select cast([(0,1),(10,3)] as Ring)").rows == [_row = [ring]] + assert query!(ctx, "select cast([(0,1),(10,3)] as Ring)").rows == [_row = [ring]] end - test "can encode and then decode Ring in query params", %{conn: conn} do + test "can encode and then decode Ring in query params", ctx do ring = [{0.0, 1.0}, {10.0, 3.0}] - assert Ch.query!(conn, "select {$0:Ring}", [ring]).rows == [_row = [ring]] + assert query!(ctx, "select {$0:Ring}", [ring]).rows == [_row = [ring]] end - test "can insert and select Ring", %{conn: conn} do - Ch.query!(conn, "CREATE TABLE geo_ring (r Ring) ENGINE = Memory()") - Ch.query!(conn, "INSERT INTO geo_ring VALUES([(0, 0), (10, 0), (10, 10), (0, 10)])") + test "can insert and select Ring", ctx do + query!(ctx, "CREATE TABLE geo_ring (r Ring) ENGINE = Memory()") + + on_exit(fn -> + Ch.Test.query("DROP TABLE geo_ring", [], database: Ch.Test.database()) + end) + + query!(ctx, "INSERT INTO geo_ring VALUES([(0, 0), (10, 0), (10, 10), (0, 10)])") ring = [{20, 20}, {0, 0}, {0, 20}] - Ch.query!(conn, "INSERT INTO geo_ring FORMAT RowBinary", [[ring]], types: ["Ring"]) + query!(ctx, "INSERT INTO geo_ring FORMAT RowBinary", [[ring]], types: ["Ring"]) - assert Ch.query!(conn, "SELECT r, toTypeName(r) FROM geo_ring ORDER BY r ASC").rows == [ + assert query!(ctx, "SELECT r, toTypeName(r) FROM geo_ring ORDER BY r ASC").rows == [ [[{0.0, 0.0}, {10.0, 0.0}, {10.0, 10.0}, {0.0, 10.0}], "Ring"], [[{20.0, 20.0}, {0.0, 0.0}, {0.0, 20.0}], "Ring"] ] # to make our RowBinary is not garbage in garbage out we also test a text format response - assert Ch.query!( - conn, + assert query!( + ctx, "SELECT r, toTypeName(r) FROM geo_ring ORDER BY r ASC FORMAT JSONCompact" ).rows |> Jason.decode!() @@ -1339,31 +1455,35 @@ defmodule Ch.ConnectionTest do ] end - test "can decode casted Polygon", %{conn: conn} do + test "can decode casted Polygon", ctx do polygon = [[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]] - assert Ch.query!(conn, "select cast([[(0,1),(10,3)],[],[(2,2)]] as Polygon)").rows == [ + assert query!(ctx, "select cast([[(0,1),(10,3)],[],[(2,2)]] as Polygon)").rows == [ _row = [polygon] ] end - test "can encode and then decode Polygon in query params", %{conn: conn} do + test "can encode and then decode Polygon in query params", ctx do polygon = [[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]] - assert Ch.query!(conn, "select {$0:Polygon}", [polygon]).rows == [_row = [polygon]] + assert query!(ctx, "select {$0:Polygon}", [polygon]).rows == [_row = [polygon]] end - test "can insert and select Polygon", %{conn: conn} do - Ch.query!(conn, "CREATE TABLE geo_polygon (pg Polygon) ENGINE = Memory()") + test "can insert and select Polygon", ctx do + query!(ctx, "CREATE TABLE geo_polygon (pg Polygon) ENGINE = Memory()") + + on_exit(fn -> + Ch.Test.query("DROP TABLE geo_polygon", [], database: Ch.Test.database()) + end) - Ch.query!( - conn, + query!( + ctx, "INSERT INTO geo_polygon VALUES([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]])" ) polygon = [[{0, 1.0}, {10, 3.2}], [], [{2, 2}]] - Ch.query!(conn, "INSERT INTO geo_polygon FORMAT RowBinary", [[polygon]], types: ["Polygon"]) + query!(ctx, "INSERT INTO geo_polygon FORMAT RowBinary", [[polygon]], types: ["Polygon"]) - assert Ch.query!(conn, "SELECT pg, toTypeName(pg) FROM geo_polygon ORDER BY pg ASC").rows == + assert query!(ctx, "SELECT pg, toTypeName(pg) FROM geo_polygon ORDER BY pg ASC").rows == [ [[[{0.0, 1.0}, {10.0, 3.2}], [], [{2.0, 2.0}]], "Polygon"], [ @@ -1376,8 +1496,8 @@ defmodule Ch.ConnectionTest do ] # to make our RowBinary is not garbage in garbage out we also test a text format response - assert Ch.query!( - conn, + assert query!( + ctx, "SELECT pg, toTypeName(pg) FROM geo_polygon ORDER BY pg ASC FORMAT JSONCompact" ).rows |> Jason.decode!() @@ -1390,40 +1510,44 @@ defmodule Ch.ConnectionTest do ] end - test "can decode casted MultiPolygon", %{conn: conn} do + test "can decode casted MultiPolygon", ctx do multipolygon = [[[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]], [], [[{3, 3}]]] - assert Ch.query!( - conn, + assert query!( + ctx, "select cast([[[(0,1),(10,3)],[],[(2,2)]],[],[[(3, 3)]]] as MultiPolygon)" ).rows == [ _row = [multipolygon] ] end - test "can encode and then decode MultiPolygon in query params", %{conn: conn} do + test "can encode and then decode MultiPolygon in query params", ctx do multipolygon = [[[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]], [], [[{3, 3}]]] - assert Ch.query!(conn, "select {$0:MultiPolygon}", [multipolygon]).rows == [ + assert query!(ctx, "select {$0:MultiPolygon}", [multipolygon]).rows == [ _row = [multipolygon] ] end - test "can insert and select MultiPolygon", %{conn: conn} do - Ch.query!(conn, "CREATE TABLE geo_multipolygon (mpg MultiPolygon) ENGINE = Memory()") + test "can insert and select MultiPolygon", ctx do + query!(ctx, "CREATE TABLE geo_multipolygon (mpg MultiPolygon) ENGINE = Memory()") + + on_exit(fn -> + Ch.Test.query("DROP TABLE geo_multipolygon", [], database: Ch.Test.database()) + end) - Ch.query!( - conn, + query!( + ctx, "INSERT INTO geo_multipolygon VALUES([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]])" ) multipolygon = [[[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]], [], [[{3, 3}]]] - Ch.query!(conn, "INSERT INTO geo_multipolygon FORMAT RowBinary", [[multipolygon]], + query!(ctx, "INSERT INTO geo_multipolygon FORMAT RowBinary", [[multipolygon]], types: ["MultiPolygon"] ) - assert Ch.query!(conn, "SELECT mpg, toTypeName(mpg) FROM geo_multipolygon ORDER BY mpg ASC").rows == + assert query!(ctx, "SELECT mpg, toTypeName(mpg) FROM geo_multipolygon ORDER BY mpg ASC").rows == [ _row = [ _multipolygon = [ @@ -1454,8 +1578,8 @@ defmodule Ch.ConnectionTest do ] # to make our RowBinary is not garbage in garbage out we also test a text format response - assert Ch.query!( - conn, + assert query!( + ctx, "SELECT mpg, toTypeName(mpg) FROM geo_multipolygon ORDER BY mpg ASC FORMAT JSONCompact" ).rows |> Jason.decode!() @@ -1475,16 +1599,16 @@ defmodule Ch.ConnectionTest do describe "options" do # this test is flaky, sometimes it raises due to ownership timeout @tag capture_log: true, skip: true - test "can provide custom timeout", %{conn: conn} do + test "can provide custom timeout", ctx do assert {:error, %Mint.TransportError{reason: :timeout} = error} = - Ch.query(conn, "select sleep(1)", _params = [], timeout: 100) + query(ctx, "select sleep(1)", _params = [], timeout: 100) assert Exception.message(error) == "timeout" end - test "errors on invalid creds", %{conn: conn} do + test "errors on invalid creds", ctx do assert {:error, %Ch.Error{code: 516} = error} = - Ch.query(conn, "select 1 + 1", _params = [], + query(ctx, "select 1 + 1", _params = [], username: "no-exists", password: "wrong" ) @@ -1493,42 +1617,43 @@ defmodule Ch.ConnectionTest do "Code: 516. DB::Exception: no-exists: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED)" end - test "errors on invalid database", %{conn: conn} do + test "errors on invalid database", ctx do assert {:error, %Ch.Error{code: 81} = error} = - Ch.query(conn, "select 1 + 1", _params = [], database: "no-db") + query(ctx, "select 1 + 1", _params = [], database: "no-db") assert Exception.message(error) =~ "`no-db`" assert Exception.message(error) =~ "UNKNOWN_DATABASE" end - test "can provide custom database", %{conn: conn} do + test "can provide custom database", ctx do assert {:ok, %{num_rows: 1, rows: [[2]]}} = - Ch.query(conn, "select 1 + 1", [], database: "default") + query(ctx, "select 1 + 1", [], database: "default") end end describe "transactions" do - test "commit", %{conn: conn} do - DBConnection.transaction(conn, fn conn -> - Ch.query!(conn, "select 1 + 1") + test "commit", ctx do + DBConnection.transaction(ctx.conn, fn conn -> + ctx = Map.put(ctx, :conn, conn) + query!(ctx, "select 1 + 1") end) end - test "rollback", %{conn: conn} do - DBConnection.transaction(conn, fn conn -> + test "rollback", ctx do + DBConnection.transaction(ctx.conn, fn conn -> DBConnection.rollback(conn, :some_reason) end) end - test "status", %{conn: conn} do - assert DBConnection.status(conn) == :idle + test "status", ctx do + assert DBConnection.status(ctx.conn) == :idle end end describe "stream" do - test "emits result structs containing raw data", %{conn: conn} do + test "emits result structs containing raw data", ctx do results = - DBConnection.run(conn, fn conn -> + DBConnection.run(ctx.conn, fn conn -> conn |> Ch.stream( "select number from system.numbers limit {limit:UInt64}", @@ -1546,14 +1671,14 @@ defmodule Ch.ConnectionTest do |> RowBinary.decode_rows() == Enum.map(0..9999, &[&1]) end - test "disconnects on early halt", %{conn: conn} do + test "disconnects on early halt", ctx do logs = ExUnit.CaptureLog.capture_log(fn -> - Ch.run(conn, fn conn -> + Ch.run(ctx.conn, fn conn -> conn |> Ch.stream("select number from system.numbers") |> Enum.take(1) end) - assert Ch.query!(conn, "select 1 + 1").rows == [[2]] + assert query!(ctx, "select 1 + 1").rows == [[2]] end) assert logs =~ @@ -1562,16 +1687,16 @@ defmodule Ch.ConnectionTest do end describe "prepare" do - test "no-op", %{conn: conn} do + test "no-op", ctx do query = Ch.Query.build("select 1 + 1") assert {:error, %Ch.Error{message: "prepared statements are not supported"}} = - DBConnection.prepare(conn, query) + DBConnection.prepare(ctx.conn, query) end end describe "start_link/1" do - test "can pass options to start_link/1" do + test "can pass options to start_link/1", ctx do db = "#{Ch.Test.database()}_#{System.unique_integer([:positive])}" Ch.Test.query("CREATE DATABASE {db:Identifier}", %{"db" => db}) @@ -1580,19 +1705,21 @@ defmodule Ch.ConnectionTest do end) {:ok, conn} = Ch.start_link(database: db) - Ch.query!(conn, "create table example(a UInt8) engine=Memory") - assert {:ok, %{rows: [["example"]]}} = Ch.query(conn, "show tables") + ctx = Map.put(ctx, :conn, conn) + query!(ctx, "create table example(a UInt8) engine=Memory") + assert {:ok, %{rows: [["example"]]}} = query(ctx, "show tables") end - test "can start without options" do + test "can start without options", ctx do {:ok, conn} = Ch.start_link() - assert {:ok, %{num_rows: 1, rows: [[2]]}} = Ch.query(conn, "select 1 + 1") + ctx = Map.put(ctx, :conn, conn) + assert {:ok, %{num_rows: 1, rows: [[2]]}} = query(ctx, "select 1 + 1") end end describe "RowBinaryWithNamesAndTypes" do - setup %{conn: conn} do - Ch.query!(conn, """ + setup ctx do + query!(ctx, """ create table if not exists row_binary_names_and_types_t ( country_code FixedString(2), rare_string LowCardinality(String), @@ -1605,7 +1732,7 @@ defmodule Ch.ConnectionTest do end) end - test "error on type mismatch", %{conn: conn} do + test "error on type mismatch", ctx do stmt = "insert into row_binary_names_and_types_t format RowBinaryWithNamesAndTypes" rows = [["AB", "rare", -42]] names = ["country_code", "rare_string", "maybe_int32"] @@ -1615,7 +1742,7 @@ defmodule Ch.ConnectionTest do types: [Ch.Types.fixed_string(2), Ch.Types.string(), Ch.Types.nullable(Ch.Types.u32())] ] - assert {:error, %Ch.Error{code: 117, message: message}} = Ch.query(conn, stmt, rows, opts) + assert {:error, %Ch.Error{code: 117, message: message}} = query(ctx, stmt, rows, opts) assert message =~ "Type of 'rare_string' must be LowCardinality(String), not String" opts = [ @@ -1627,11 +1754,11 @@ defmodule Ch.ConnectionTest do ] ] - assert {:error, %Ch.Error{code: 117, message: message}} = Ch.query(conn, stmt, rows, opts) + assert {:error, %Ch.Error{code: 117, message: message}} = query(ctx, stmt, rows, opts) assert message =~ "Type of 'maybe_int32' must be Nullable(Int32), not Nullable(UInt32)" end - test "ok on valid types", %{conn: conn} do + test "ok on valid types", ctx do stmt = "insert into row_binary_names_and_types_t format RowBinaryWithNamesAndTypes" rows = [["AB", "rare", -42]] names = ["country_code", "rare_string", "maybe_int32"] @@ -1645,14 +1772,14 @@ defmodule Ch.ConnectionTest do ] ] - assert {:ok, %{num_rows: 1}} = Ch.query(conn, stmt, rows, opts) + assert {:ok, %{num_rows: 1}} = query(ctx, stmt, rows, opts) end - test "select with lots of columns", %{conn: conn} do + test "select with lots of columns", ctx do select = Enum.map_join(1..1000, ", ", fn i -> "#{i} as col_#{i}" end) stmt = "select #{select} format RowBinaryWithNamesAndTypes" - assert %Ch.Result{columns: columns, rows: [row]} = Ch.query!(conn, stmt) + assert %Ch.Result{columns: columns, rows: [row]} = query!(ctx, stmt) assert length(columns) == 1000 assert List.first(columns) == "col_1" diff --git a/test/ch/multipart_test.exs b/test/ch/multipart_test.exs deleted file mode 100644 index 3d7cd7e..0000000 --- a/test/ch/multipart_test.exs +++ /dev/null @@ -1,23 +0,0 @@ -defmodule Ch.MultipartTest do - use ExUnit.Case, async: true - - setup do - {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} - end - - test "sends multipart", %{conn: conn} do - sql = "SELECT {a:String}, {b:String}" - params = %{"a" => "A", "b" => "B"} - - assert %Ch.Result{rows: [["A", "B"]]} = - Ch.query!(conn, sql, params, multipart: true) - end - - test "sends positional parameters correctly", %{conn: conn} do - sql = "SELECT {$0:String}, {$1:Int32}" - params = ["pos0", 42] - - assert %Ch.Result{rows: [["pos0", 42]]} = - Ch.query!(conn, sql, params, multipart: true) - end -end From aafe0177e06f6dccf9257ae9340fa1bf670b50f2 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Fri, 2 Jan 2026 16:03:08 +0300 Subject: [PATCH 29/34] typos skip --- .typos.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/.typos.toml b/.typos.toml index c98a52a..1b41395 100644 --- a/.typos.toml +++ b/.typos.toml @@ -1,3 +1,4 @@ [default.extend-words] "som" = "som" # ./test/ch/ecto_type_test.exs "ECT" = "ECT" # ./test/ch/query_test.exs +"Evn" = "Evn" # ./CHANGELOG.md From 10f1fe1281521b48c3646e097027619de65ac1eb Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Fri, 2 Jan 2026 16:17:35 +0300 Subject: [PATCH 30/34] eh --- test/ch/connection_test.exs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/ch/connection_test.exs b/test/ch/connection_test.exs index 1e59c6b..bea446a 100644 --- a/test/ch/connection_test.exs +++ b/test/ch/connection_test.exs @@ -6,22 +6,22 @@ defmodule Ch.ConnectionTest do {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} end - defp query( - %{conn: conn, query_options: default_options}, - sql, - params \\ [], - custom_options \\ [] - ) do - Ch.query(conn, sql, params, Keyword.merge(default_options, custom_options)) + defp parameterize_query_options(ctx, custom_options) do + if extra_options = ctx[:query_options] do + Keyword.merge(extra_options, custom_options) + else + custom_options + end + end + + defp query(%{conn: conn} = ctx, sql, params \\ [], custom_options \\ []) do + options = parameterize_query_options(ctx, custom_options) + Ch.query(conn, sql, params, optsions) end - defp query!( - %{conn: conn, query_options: default_options}, - sql, - params \\ [], - custom_options \\ [] - ) do - Ch.query!(conn, sql, params, Keyword.merge(default_options, custom_options)) + defp query!(%{conn: conn} = ctx, sql, params \\ [], custom_options \\ []) do + options = parameterize_query_options(ctx, custom_options) + Ch.query!(conn, sql, params, options) end test "select without params", ctx do From 5574eefd0ce513275dad067129f5ab6a4e40817e Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Fri, 2 Jan 2026 16:19:09 +0300 Subject: [PATCH 31/34] eh! --- test/ch/connection_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ch/connection_test.exs b/test/ch/connection_test.exs index bea446a..35f3836 100644 --- a/test/ch/connection_test.exs +++ b/test/ch/connection_test.exs @@ -16,7 +16,7 @@ defmodule Ch.ConnectionTest do defp query(%{conn: conn} = ctx, sql, params \\ [], custom_options \\ []) do options = parameterize_query_options(ctx, custom_options) - Ch.query(conn, sql, params, optsions) + Ch.query(conn, sql, params, options) end defp query!(%{conn: conn} = ctx, sql, params \\ [], custom_options \\ []) do From 8292590cc785e192e4ee0f00491b7ccfaba05def Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Fri, 2 Jan 2026 16:38:30 +0300 Subject: [PATCH 32/34] cleanup --- test/ch/connection_test.exs | 128 ++++++++---------------------------- test/ch/dynamic_test.exs | 12 ++-- test/ch/json_test.exs | 5 +- test/ch/variant_test.exs | 4 +- test/support/test.ex | 1 + test/test_helper.exs | 13 +++- 6 files changed, 49 insertions(+), 114 deletions(-) diff --git a/test/ch/connection_test.exs b/test/ch/connection_test.exs index 35f3836..36e9452 100644 --- a/test/ch/connection_test.exs +++ b/test/ch/connection_test.exs @@ -170,14 +170,8 @@ defmodule Ch.ConnectionTest do end test "non-utc datetime rowbinary encoding", ctx do - query!( - ctx, - "create table ch_non_utc_datetimes(name String, datetime DateTime) engine Memory" - ) - - on_exit(fn -> - Ch.Test.query("drop table ch_non_utc_datetimes", [], database: Ch.Test.database()) - end) + query!(ctx, "create table ch_non_utc_datetimes(name String, datetime DateTime) engine Memory") + on_exit(fn -> Ch.Test.query("drop table ch_non_utc_datetimes") end) utc = ~U[2024-12-21 05:35:19.886393Z] @@ -248,9 +242,7 @@ defmodule Ch.ConnectionTest do assert {:ok, %{command: :create, num_rows: nil, rows: [], data: []}} = query(ctx, "create table create_example(a UInt8) engine = Memory") - on_exit(fn -> - Ch.Test.query("drop table create_example", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("drop table create_example") end) end test "create with options", ctx do @@ -400,10 +392,7 @@ defmodule Ch.ConnectionTest do test "delete", ctx do query!(ctx, "create table delete_t(a UInt8, b String) engine = MergeTree order by tuple()") - - on_exit(fn -> - Ch.Test.query("drop table delete_t", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("drop table delete_t") end) assert {:ok, %{num_rows: 2}} = query(ctx, "insert into delete_t values (1,'a'), (2,'b')") @@ -464,10 +453,7 @@ defmodule Ch.ConnectionTest do query(ctx, "select {a:FixedString(5)}", %{"a" => "aaaaa"}) query!(ctx, "create table fixed_string_t(a FixedString(3)) engine = Memory") - - on_exit(fn -> - Ch.Test.query("drop table fixed_string_t", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("drop table fixed_string_t") end) assert {:ok, %{num_rows: 4}} = query( @@ -512,10 +498,7 @@ defmodule Ch.ConnectionTest do assert row == [Decimal.new("2.0000"), Decimal.new("0.6666"), "Decimal(76, 4)"] query!(ctx, "create table decimal_t(d Decimal32(4)) engine = Memory") - - on_exit(fn -> - Ch.Test.query("drop table decimal_t", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("drop table decimal_t") end) assert %{num_rows: 3} = query!( @@ -546,10 +529,7 @@ defmodule Ch.ConnectionTest do assert {:ok, %{num_rows: 1, rows: [[true, false]]}} = query(ctx, "select true, false") query!(ctx, "create table test_bool(A Int64, B Bool) engine = Memory") - - on_exit(fn -> - Ch.Test.query("drop table test_bool", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("drop table test_bool") end) query!(ctx, "INSERT INTO test_bool VALUES (1, true),(2,0)") @@ -589,10 +569,7 @@ defmodule Ch.ConnectionTest do |> Base.decode16!(case: :lower) query!(ctx, " CREATE TABLE t_uuid (x UUID, y String) ENGINE Memory") - - on_exit(fn -> - Ch.Test.query("drop table t_uuid", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("drop table t_uuid") end) query!(ctx, "INSERT INTO t_uuid SELECT generateUUIDv4(), 'Example 1'") @@ -666,9 +643,7 @@ defmodule Ch.ConnectionTest do settings: [enable_json_type: 1] ) - on_exit(fn -> - Ch.Test.query("DROP TABLE test_json_as_string", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE test_json_as_string") end) query!( ctx, @@ -709,9 +684,7 @@ defmodule Ch.ConnectionTest do "CREATE TABLE t_enum(i UInt8, x Enum('hello' = 1, 'world' = 2)) ENGINE Memory" ) - on_exit(fn -> - Ch.Test.query("DROP TABLE t_enum", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE t_enum") end) query!(ctx, "INSERT INTO t_enum VALUES (0, 'hello'), (1, 'world'), (2, 'hello')") @@ -752,10 +725,7 @@ defmodule Ch.ConnectionTest do }).rows == [[%{"hello" => 100, "pg" => 13}]] query!(ctx, "CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory") - - on_exit(fn -> - Ch.Test.query("DROP TABLE table_map", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE table_map") end) query!( ctx, @@ -821,10 +791,7 @@ defmodule Ch.ConnectionTest do # TODO named tuples query!(ctx, "CREATE TABLE tuples_t (`a` Tuple(String, Int64)) ENGINE = Memory") - - on_exit(fn -> - Ch.Test.query("DROP TABLE tuples_t", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE tuples_t") end) assert %{num_rows: 2} = query!(ctx, "INSERT INTO tuples_t VALUES (('y', 10)), (('x',-10))") @@ -851,9 +818,7 @@ defmodule Ch.ConnectionTest do "CREATE TABLE dt(`timestamp` DateTime('Asia/Istanbul'), `event_id` UInt8) ENGINE = Memory" ) - on_exit(fn -> - Ch.Test.query("DROP TABLE dt", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE dt") end) query!(ctx, "INSERT INTO dt Values (1546300800, 1), ('2019-01-01 00:00:00', 2)") @@ -920,10 +885,7 @@ defmodule Ch.ConnectionTest do # TODO are negatives correct? what's the range? test "date32", ctx do query!(ctx, "CREATE TABLE new(`timestamp` Date32, `event_id` UInt8) ENGINE = Memory;") - - on_exit(fn -> - Ch.Test.query("DROP TABLE new", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE new") end) query!(ctx, "INSERT INTO new VALUES (4102444800, 1), ('2100-01-01', 2)") @@ -986,7 +948,7 @@ defmodule Ch.ConnectionTest do ) on_exit(fn -> - Ch.Test.query("DROP TABLE time_t", [], settings: settings, database: Ch.Test.database()) + Ch.Test.query("DROP TABLE time_t", [], settings: settings) end) query!(ctx, "INSERT INTO time_t VALUES ('100:00:00', 1), (12453, 2)", [], @@ -1032,10 +994,7 @@ defmodule Ch.ConnectionTest do ) on_exit(fn -> - Ch.Test.query("DROP TABLE time64_3_t", [], - settings: settings, - database: Ch.Test.database() - ) + Ch.Test.query("DROP TABLE time64_3_t", [], settings: settings) end) query!( @@ -1095,10 +1054,7 @@ defmodule Ch.ConnectionTest do ) on_exit(fn -> - Ch.Test.query("DROP TABLE time64_6_t", [], - settings: settings, - database: Ch.Test.database() - ) + Ch.Test.query("DROP TABLE time64_6_t", [], settings: settings) end) query!( @@ -1142,10 +1098,7 @@ defmodule Ch.ConnectionTest do ) on_exit(fn -> - Ch.Test.query("DROP TABLE time64_9_t", [], - settings: settings, - database: Ch.Test.database() - ) + Ch.Test.query("DROP TABLE time64_9_t", [], settings: settings) end) query!( @@ -1183,9 +1136,7 @@ defmodule Ch.ConnectionTest do "CREATE TABLE datetime64_t(`timestamp` DateTime64(3, 'Asia/Istanbul'), `event_id` UInt8) ENGINE = Memory" ) - on_exit(fn -> - Ch.Test.query("DROP TABLE datetime64_t", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE datetime64_t") end) query!( ctx, @@ -1293,9 +1244,7 @@ defmodule Ch.ConnectionTest do "CREATE TABLE nullable (`n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple()" ) - on_exit(fn -> - Ch.Test.query("DROP TABLE nullable", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE nullable") end) query!(ctx, "INSERT INTO nullable VALUES (1) (NULL) (2) (NULL)") @@ -1330,9 +1279,7 @@ defmodule Ch.ConnectionTest do ) ENGINE Memory """) - on_exit(fn -> - Ch.Test.query("DROP TABLE ch_nulls", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE ch_nulls") end) query!( ctx, @@ -1354,9 +1301,7 @@ defmodule Ch.ConnectionTest do ) ENGINE Memory """) - on_exit(fn -> - Ch.Test.query("DROP TABLE test_insert_default_value", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE test_insert_default_value") end) query!( ctx, @@ -1391,10 +1336,7 @@ defmodule Ch.ConnectionTest do test "can insert and select Point", ctx do query!(ctx, "CREATE TABLE geo_point (p Point) ENGINE = Memory()") - - on_exit(fn -> - Ch.Test.query("DROP TABLE geo_point", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE geo_point") end) query!(ctx, "INSERT INTO geo_point VALUES((10, 10))") query!(ctx, "INSERT INTO geo_point FORMAT RowBinary", [[{20, 20}]], types: ["Point"]) @@ -1428,10 +1370,7 @@ defmodule Ch.ConnectionTest do test "can insert and select Ring", ctx do query!(ctx, "CREATE TABLE geo_ring (r Ring) ENGINE = Memory()") - - on_exit(fn -> - Ch.Test.query("DROP TABLE geo_ring", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE geo_ring") end) query!(ctx, "INSERT INTO geo_ring VALUES([(0, 0), (10, 0), (10, 10), (0, 10)])") @@ -1470,10 +1409,7 @@ defmodule Ch.ConnectionTest do test "can insert and select Polygon", ctx do query!(ctx, "CREATE TABLE geo_polygon (pg Polygon) ENGINE = Memory()") - - on_exit(fn -> - Ch.Test.query("DROP TABLE geo_polygon", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE geo_polygon") end) query!( ctx, @@ -1531,10 +1467,7 @@ defmodule Ch.ConnectionTest do test "can insert and select MultiPolygon", ctx do query!(ctx, "CREATE TABLE geo_multipolygon (mpg MultiPolygon) ENGINE = Memory()") - - on_exit(fn -> - Ch.Test.query("DROP TABLE geo_multipolygon", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("DROP TABLE geo_multipolygon") end) query!( ctx, @@ -1699,10 +1632,7 @@ defmodule Ch.ConnectionTest do test "can pass options to start_link/1", ctx do db = "#{Ch.Test.database()}_#{System.unique_integer([:positive])}" Ch.Test.query("CREATE DATABASE {db:Identifier}", %{"db" => db}) - - on_exit(fn -> - Ch.Test.query("DROP DATABASE {db:Identifier}", %{"db" => db}) - end) + on_exit(fn -> Ch.Test.query("DROP DATABASE {db:Identifier}", %{"db" => db}) end) {:ok, conn} = Ch.start_link(database: db) ctx = Map.put(ctx, :conn, conn) @@ -1727,9 +1657,7 @@ defmodule Ch.ConnectionTest do ) engine Memory """) - on_exit(fn -> - Ch.Test.query("truncate row_binary_names_and_types_t", [], database: Ch.Test.database()) - end) + on_exit(fn -> Ch.Test.query("truncate row_binary_names_and_types_t") end) end test "error on type mismatch", ctx do diff --git a/test/ch/dynamic_test.exs b/test/ch/dynamic_test.exs index 256bc52..8f9a693 100644 --- a/test/ch/dynamic_test.exs +++ b/test/ch/dynamic_test.exs @@ -14,7 +14,7 @@ defmodule Ch.DynamicTest do end Ch.query!(conn, "CREATE TABLE test (d Dynamic, id String) ENGINE = Memory;") - on_exit(fn -> Ch.Test.query("DROP TABLE test", [], database: Ch.Test.database()) end) + on_exit(fn -> Ch.Test.query("DROP TABLE test") end) insert = fn value -> id = inspect(value) @@ -269,7 +269,7 @@ defmodule Ch.DynamicTest do test "creating dynamic", %{conn: conn} do # Using Dynamic type in table column definition: Ch.query!(conn, "CREATE TABLE test (d Dynamic) ENGINE = Memory;") - on_exit(fn -> Ch.Test.query("DROP TABLE test", [], database: Ch.Test.database()) end) + on_exit(fn -> Ch.Test.query("DROP TABLE test") end) Ch.query!(conn, "INSERT INTO test VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);") assert Ch.query!(conn, "SELECT d, dynamicType(d) FROM test;").rows == [ @@ -303,7 +303,7 @@ defmodule Ch.DynamicTest do # https://clickhouse.com/docs/sql-reference/data-types/dynamic#reading-dynamic-nested-types-as-subcolumns test "reading dynamic nested types as subcolumns", %{conn: conn} do Ch.query!(conn, "CREATE TABLE test (d Dynamic) ENGINE = Memory;") - on_exit(fn -> Ch.Test.query("DROP TABLE test", [], database: Ch.Test.database()) end) + on_exit(fn -> Ch.Test.query("DROP TABLE test") end) Ch.query!(conn, "INSERT INTO test VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);") assert Ch.query!( @@ -358,7 +358,7 @@ defmodule Ch.DynamicTest do # https://clickhouse.com/docs/sql-reference/data-types/dynamic#converting-a-dynamic-column-to-an-ordinary-column test "converting a dynamic column to an ordinary column", %{conn: conn} do Ch.query!(conn, "CREATE TABLE test (d Dynamic) ENGINE = Memory;") - on_exit(fn -> Ch.Test.query("DROP TABLE test", [], database: Ch.Test.database()) end) + on_exit(fn -> Ch.Test.query("DROP TABLE test") end) Ch.query!(conn, "INSERT INTO test VALUES (NULL), (42), ('42.42'), (true), ('e10');") assert Ch.query!(conn, "SELECT d::Nullable(Float64) FROM test;").rows == [ @@ -377,7 +377,7 @@ defmodule Ch.DynamicTest do "CREATE TABLE test (v Variant(UInt64, String, Array(UInt64))) ENGINE = Memory;" ) - on_exit(fn -> Ch.Test.query("DROP TABLE test", [], database: Ch.Test.database()) end) + on_exit(fn -> Ch.Test.query("DROP TABLE test") end) Ch.query!(conn, "INSERT INTO test VALUES (NULL), (42), ('String'), ([1, 2, 3]);") assert Ch.query!(conn, "SELECT v::Dynamic AS d, dynamicType(d) FROM test;").rows == [ @@ -391,7 +391,7 @@ defmodule Ch.DynamicTest do # https://clickhouse.com/docs/sql-reference/data-types/dynamic#converting-a-dynamicmax_typesn-column-to-another-dynamicmax_typesk test "converting a Dynamic(max_types=N) column to another Dynamic(max_types=K)", %{conn: conn} do Ch.query!(conn, "CREATE TABLE test (d Dynamic(max_types=4)) ENGINE = Memory;") - on_exit(fn -> Ch.Test.query("DROP TABLE test", [], database: Ch.Test.database()) end) + on_exit(fn -> Ch.Test.query("DROP TABLE test") end) Ch.query!(conn, "INSERT INTO test VALUES (NULL), (42), (43), ('42.42'), (true), ([1, 2, 3]);") assert Ch.query!(conn, "SELECT d::Dynamic(max_types=5) as d2, dynamicType(d2) FROM test;").rows == diff --git a/test/ch/json_test.exs b/test/ch/json_test.exs index e402d8d..152c2b4 100644 --- a/test/ch/json_test.exs +++ b/test/ch/json_test.exs @@ -4,10 +4,7 @@ defmodule Ch.JSONTest do @moduletag :json setup do - on_exit(fn -> - Ch.Test.query("DROP TABLE IF EXISTS json_test", [], database: Ch.Test.database()) - end) - + on_exit(fn -> Ch.Test.query("DROP TABLE IF EXISTS json_test") end) {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} end diff --git a/test/ch/variant_test.exs b/test/ch/variant_test.exs index 23a2c57..e8008a5 100644 --- a/test/ch/variant_test.exs +++ b/test/ch/variant_test.exs @@ -43,7 +43,7 @@ defmodule Ch.VariantTest do CREATE TABLE variant_test (v Variant(UInt64, String, Array(UInt64))) ENGINE = Memory; """) - on_exit(fn -> Ch.Test.query("DROP TABLE variant_test", [], database: Ch.Test.database()) end) + on_exit(fn -> Ch.Test.query("DROP TABLE variant_test") end) Ch.query!( conn, @@ -82,7 +82,7 @@ defmodule Ch.VariantTest do CREATE TABLE variant_test (v Variant(UInt64, String, Array(UInt64))) ENGINE = Memory; """) - on_exit(fn -> Ch.Test.query("DROP TABLE variant_test", [], database: Ch.Test.database()) end) + on_exit(fn -> Ch.Test.query("DROP TABLE variant_test") end) Ch.query!( conn, diff --git a/test/support/test.ex b/test/support/test.ex index 4200c01..1a5ac00 100644 --- a/test/support/test.ex +++ b/test/support/test.ex @@ -8,6 +8,7 @@ defmodule Ch.Test do task = Task.async(fn -> {:ok, pid} = Ch.start_link(opts) + opts = Keyword.put_new_lazy(opts, :database, &database/0) Ch.query!(pid, sql, params, opts) end) diff --git a/test/test_helper.exs b/test/test_helper.exs index eef00d8..97caedd 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -21,8 +21,17 @@ Calendar.put_time_zone_database(Tz.TimeZoneDatabase) default_test_db = System.get_env("CH_DATABASE", "ch_elixir_test") Application.put_env(:ch, :database, default_test_db) -Ch.Test.query("DROP DATABASE IF EXISTS {db:Identifier}", %{"db" => default_test_db}) -Ch.Test.query("CREATE DATABASE {db:Identifier}", %{"db" => default_test_db}) +Ch.Test.query( + "DROP DATABASE IF EXISTS {db:Identifier}", + %{"db" => default_test_db}, + database: "default" +) + +Ch.Test.query( + "CREATE DATABASE {db:Identifier}", + %{"db" => default_test_db}, + database: "default" +) %{rows: [[ch_version]]} = Ch.Test.query("SELECT version()") From 99522fc665ad9b573727ab18827bb358da1d70aa Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Sat, 3 Jan 2026 02:23:04 +0300 Subject: [PATCH 33/34] continue --- test/ch/connection_test.exs | 671 +++++++++++++++++++++--------------- test/ch/dynamic_test.exs | 108 +++--- test/support/test.ex | 27 ++ 3 files changed, 484 insertions(+), 322 deletions(-) diff --git a/test/ch/connection_test.exs b/test/ch/connection_test.exs index 36e9452..b7d1b45 100644 --- a/test/ch/connection_test.exs +++ b/test/ch/connection_test.exs @@ -1,81 +1,78 @@ defmodule Ch.ConnectionTest do use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] + + import Ch.Test, + only: [ + parameterize_query: 2, + parameterize_query: 3, + parameterize_query: 4, + parameterize_query!: 2, + parameterize_query!: 3, + parameterize_query!: 4 + ] + alias Ch.RowBinary setup do {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} end - defp parameterize_query_options(ctx, custom_options) do - if extra_options = ctx[:query_options] do - Keyword.merge(extra_options, custom_options) - else - custom_options - end - end - - defp query(%{conn: conn} = ctx, sql, params \\ [], custom_options \\ []) do - options = parameterize_query_options(ctx, custom_options) - Ch.query(conn, sql, params, options) - end - - defp query!(%{conn: conn} = ctx, sql, params \\ [], custom_options \\ []) do - options = parameterize_query_options(ctx, custom_options) - Ch.query!(conn, sql, params, options) - end - test "select without params", ctx do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select 1") + assert {:ok, %{num_rows: 1, rows: [[1]]}} = + parameterize_query(ctx, "select 1") end test "select with types", ctx do assert {:ok, %{num_rows: 1, rows: [[1]]}} = - query(ctx, "select 1", [], types: ["UInt8"]) + parameterize_query(ctx, "select 1", [], types: ["UInt8"]) end test "select with params", ctx do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:UInt8}", %{"a" => 1}) + assert {:ok, %{num_rows: 1, rows: [[1]]}} = + parameterize_query(ctx, "select {a:UInt8}", %{"a" => 1}) assert {:ok, %{num_rows: 1, rows: [[true]]}} = - query(ctx, "select {b:Bool}", %{"b" => true}) + parameterize_query(ctx, "select {b:Bool}", %{"b" => true}) assert {:ok, %{num_rows: 1, rows: [[false]]}} = - query(ctx, "select {b:Bool}", %{"b" => false}) + parameterize_query(ctx, "select {b:Bool}", %{"b" => false}) assert {:ok, %{num_rows: 1, rows: [[nil]]}} = - query(ctx, "select {n:Nullable(Nothing)}", %{"n" => nil}) + parameterize_query(ctx, "select {n:Nullable(Nothing)}", %{"n" => nil}) assert {:ok, %{num_rows: 1, rows: [[1.0]]}} = - query(ctx, "select {a:Float32}", %{"a" => 1.0}) + parameterize_query(ctx, "select {a:Float32}", %{"a" => 1.0}) assert {:ok, %{num_rows: 1, rows: [["a&b=c"]]}} = - query(ctx, "select {a:String}", %{"a" => "a&b=c"}) + parameterize_query(ctx, "select {a:String}", %{"a" => "a&b=c"}) assert {:ok, %{num_rows: 1, rows: [["a\n"]]}} = - query(ctx, "select {a:String}", %{"a" => "a\n"}) + parameterize_query(ctx, "select {a:String}", %{"a" => "a\n"}) assert {:ok, %{num_rows: 1, rows: [["a\t"]]}} = - query(ctx, "select {a:String}", %{"a" => "a\t"}) + parameterize_query(ctx, "select {a:String}", %{"a" => "a\t"}) assert {:ok, %{num_rows: 1, rows: [[["a\tb"]]]}} = - query(ctx, "select {a:Array(String)}", %{"a" => ["a\tb"]}) + parameterize_query(ctx, "select {a:Array(String)}", %{"a" => ["a\tb"]}) assert {:ok, %{num_rows: 1, rows: [[[true, false]]]}} = - query(ctx, "select {a:Array(Bool)}", %{"a" => [true, false]}) + parameterize_query(ctx, "select {a:Array(Bool)}", %{"a" => [true, false]}) assert {:ok, %{num_rows: 1, rows: [[["a", nil, "b"]]]}} = - query(ctx, "select {a:Array(Nullable(String))}", %{"a" => ["a", nil, "b"]}) + parameterize_query(ctx, "select {a:Array(Nullable(String))}", %{ + "a" => ["a", nil, "b"] + }) assert {:ok, %{num_rows: 1, rows: [row]}} = - query(ctx, "select {a:Decimal(9,4)}", %{"a" => Decimal.new("2000.333")}) + parameterize_query(ctx, "select {a:Decimal(9,4)}", %{"a" => Decimal.new("2000.333")}) assert row == [Decimal.new("2000.3330")] assert {:ok, %{num_rows: 1, rows: [[~D[2022-01-01]]]}} = - query(ctx, "select {a:Date}", %{"a" => ~D[2022-01-01]}) + parameterize_query(ctx, "select {a:Date}", %{"a" => ~D[2022-01-01]}) assert {:ok, %{num_rows: 1, rows: [[~D[2022-01-01]]]}} = - query(ctx, "select {a:Date32}", %{"a" => ~D[2022-01-01]}) + parameterize_query(ctx, "select {a:Date32}", %{"a" => ~D[2022-01-01]}) naive_noon = ~N[2022-01-01 12:00:00] @@ -83,7 +80,7 @@ defmodule Ch.ConnectionTest do # see https://clickhouse.com/docs/en/sql-reference/data-types/datetime # https://kb.altinity.com/altinity-kb-queries-and-syntax/time-zones/ assert {:ok, %{num_rows: 1, rows: [[naive_datetime]], headers: headers}} = - query(ctx, "select {naive:DateTime}", %{"naive" => naive_noon}) + parameterize_query(ctx, "select {naive:DateTime}", %{"naive" => naive_noon}) # to make this test pass for contributors with non UTC timezone we perform the same steps as ClickHouse # i.e. we give server timezone to the naive datetime and shift it to UTC before comparing with the result @@ -97,17 +94,17 @@ defmodule Ch.ConnectionTest do # when the timezone information is provided in the type, we don't need to rely on server timezone assert {:ok, %{num_rows: 1, rows: [[bkk_datetime]]}} = - query(ctx, "select {$0:DateTime('Asia/Bangkok')}", [naive_noon]) + parameterize_query(ctx, "select {$0:DateTime('Asia/Bangkok')}", [naive_noon]) assert bkk_datetime == DateTime.from_naive!(naive_noon, "Asia/Bangkok") assert {:ok, %{num_rows: 1, rows: [[~U[2022-01-01 12:00:00Z]]]}} = - query(ctx, "select {$0:DateTime('UTC')}", [naive_noon]) + parameterize_query(ctx, "select {$0:DateTime('UTC')}", [naive_noon]) naive_noon_ms = ~N[2022-01-01 12:00:00.123] assert {:ok, %{num_rows: 1, rows: [[naive_datetime]]}} = - query(ctx, "select {$0:DateTime64(3)}", [naive_noon_ms]) + parameterize_query(ctx, "select {$0:DateTime64(3)}", [naive_noon_ms]) assert NaiveDateTime.compare( naive_datetime, @@ -118,29 +115,29 @@ defmodule Ch.ConnectionTest do ) == :eq assert {:ok, %{num_rows: 1, rows: [[["a", "b'", "\\'c"]]]}} = - query(ctx, "select {a:Array(String)}", %{"a" => ["a", "b'", "\\'c"]}) + parameterize_query(ctx, "select {a:Array(String)}", %{"a" => ["a", "b'", "\\'c"]}) assert {:ok, %{num_rows: 1, rows: [[["a\n", "b\tc"]]]}} = - query(ctx, "select {a:Array(String)}", %{"a" => ["a\n", "b\tc"]}) + parameterize_query(ctx, "select {a:Array(String)}", %{"a" => ["a\n", "b\tc"]}) assert {:ok, %{num_rows: 1, rows: [[[1, 2, 3]]]}} = - query(ctx, "select {a:Array(UInt8)}", %{"a" => [1, 2, 3]}) + parameterize_query(ctx, "select {a:Array(UInt8)}", %{"a" => [1, 2, 3]}) assert {:ok, %{num_rows: 1, rows: [[[[1], [2, 3], []]]]}} = - query(ctx, "select {a:Array(Array(UInt8))}", %{"a" => [[1], [2, 3], []]}) + parameterize_query(ctx, "select {a:Array(Array(UInt8))}", %{"a" => [[1], [2, 3], []]}) uuid = "9B29BD20-924C-4DE5-BDB3-8C2AA1FCE1FC" uuid_bin = uuid |> String.replace("-", "") |> Base.decode16!() assert {:ok, %{num_rows: 1, rows: [[^uuid_bin]]}} = - query(ctx, "select {a:UUID}", %{"a" => uuid}) + parameterize_query(ctx, "select {a:UUID}", %{"a" => uuid}) # TODO # assert {:ok, %{num_rows: 1, rows: [[^uuid_bin]]}} = - # query(ctx, "select {a:UUID}", %{"a" => uuid_bin}) + # parameterize_query(ctx, "select {a:UUID}", %{"a" => uuid_bin}) # pseudo-positional bind - assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {$0:UInt8}", [1]) + assert {:ok, %{num_rows: 1, rows: [[1]]}} = parameterize_query(ctx, "select {$0:UInt8}", [1]) end test "utc datetime query param encoding", ctx do @@ -148,13 +145,15 @@ defmodule Ch.ConnectionTest do msk = DateTime.new!(~D[2021-01-01], ~T[15:00:00], "Europe/Moscow") naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(ctx.conn)) |> DateTime.to_naive() - assert query!(ctx, "select {$0:DateTime} as d, toString(d)", [utc]).rows == + assert parameterize_query!(ctx, "select {$0:DateTime} as d, toString(d)", [utc]).rows == [[~N[2021-01-01 12:00:00], to_string(naive)]] - assert query!(ctx, "select {$0:DateTime('UTC')} as d, toString(d)", [utc]).rows == + assert parameterize_query!(ctx, "select {$0:DateTime('UTC')} as d, toString(d)", [utc]).rows == [[utc, "2021-01-01 12:00:00"]] - assert query!(ctx, "select {$0:DateTime('Europe/Moscow')} as d, toString(d)", [utc]).rows == + assert parameterize_query!(ctx, "select {$0:DateTime('Europe/Moscow')} as d, toString(d)", [ + utc + ]).rows == [[msk, "2021-01-01 15:00:00"]] end @@ -163,14 +162,22 @@ defmodule Ch.ConnectionTest do assert inspect(jp) == "#DateTime<2021-01-01 21:34:56+09:00 JST Asia/Tokyo>" assert [[utc, jp]] = - query!(ctx, "select {$0:DateTime('UTC')}, {$0:DateTime('Asia/Tokyo')}", [jp]).rows + parameterize_query!( + ctx, + "select {$0:DateTime('UTC')}, {$0:DateTime('Asia/Tokyo')}", + [jp] + ).rows assert inspect(utc) == "~U[2021-01-01 12:34:56Z]" assert inspect(jp) == "#DateTime<2021-01-01 21:34:56+09:00 JST Asia/Tokyo>" end test "non-utc datetime rowbinary encoding", ctx do - query!(ctx, "create table ch_non_utc_datetimes(name String, datetime DateTime) engine Memory") + parameterize_query!( + ctx, + "create table ch_non_utc_datetimes(name String, datetime DateTime) engine Memory" + ) + on_exit(fn -> Ch.Test.query("drop table ch_non_utc_datetimes") end) utc = ~U[2024-12-21 05:35:19.886393Z] @@ -181,12 +188,18 @@ defmodule Ch.ConnectionTest do rows = [["taipei", taipei], ["tokyo", tokyo], ["vienna", vienna]] - query!(ctx, "insert into ch_non_utc_datetimes(name, datetime) format RowBinary", rows, + parameterize_query!( + ctx, + "insert into ch_non_utc_datetimes(name, datetime) format RowBinary", + rows, types: ["String", "DateTime"] ) result = - query!(ctx, "select name, cast(datetime as DateTime('UTC')) from ch_non_utc_datetimes") + parameterize_query!( + ctx, + "select name, cast(datetime as DateTime('UTC')) from ch_non_utc_datetimes" + ) |> Map.fetch!(:rows) |> Map.new(fn [name, datetime] -> {name, datetime} end) @@ -200,13 +213,17 @@ defmodule Ch.ConnectionTest do msk = DateTime.new!(~D[2021-01-01], ~T[15:00:00.123456], "Europe/Moscow") naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(ctx.conn)) |> DateTime.to_naive() - assert query!(ctx, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == + assert parameterize_query!(ctx, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == [[~N[2021-01-01 12:00:00.123456], to_string(naive)]] - assert query!(ctx, "select {$0:DateTime64(6, 'UTC')} as d, toString(d)", [utc]).rows == + assert parameterize_query!(ctx, "select {$0:DateTime64(6, 'UTC')} as d, toString(d)", [utc]).rows == [[utc, "2021-01-01 12:00:00.123456"]] - assert query!(ctx, "select {$0:DateTime64(6,'Europe/Moscow')} as d, toString(d)", [utc]).rows == + assert parameterize_query!( + ctx, + "select {$0:DateTime64(6,'Europe/Moscow')} as d, toString(d)", + [utc] + ).rows == [[msk, "2021-01-01 15:00:00.123456"]] end @@ -216,7 +233,7 @@ defmodule Ch.ConnectionTest do utc = ~U[2021-01-01 12:00:00.000000Z] naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(ctx.conn)) |> DateTime.to_naive() - assert query!(ctx, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == + assert parameterize_query!(ctx, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == [[~N[2021-01-01 12:00:00.000000], to_string(naive)]] end @@ -226,28 +243,32 @@ defmodule Ch.ConnectionTest do utc = ~U[2024-05-26 20:00:46.099856Z] naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(ctx.conn)) |> DateTime.to_naive() - assert query!(ctx, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == + assert parameterize_query!(ctx, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == [[~N[2024-05-26 20:00:46.099856Z], to_string(naive)]] end test "select with options", ctx do assert {:ok, %{num_rows: 1, rows: [["async_insert", "Bool", "1"]]}} = - query(ctx, "show settings like 'async_insert'", [], settings: [async_insert: 1]) + parameterize_query(ctx, "show settings like 'async_insert'", [], + settings: [async_insert: 1] + ) assert {:ok, %{num_rows: 1, rows: [["async_insert", "Bool", "0"]]}} = - query(ctx, "show settings like 'async_insert'", [], settings: [async_insert: 0]) + parameterize_query(ctx, "show settings like 'async_insert'", [], + settings: [async_insert: 0] + ) end test "create", ctx do assert {:ok, %{command: :create, num_rows: nil, rows: [], data: []}} = - query(ctx, "create table create_example(a UInt8) engine = Memory") + parameterize_query(ctx, "create table create_example(a UInt8) engine = Memory") on_exit(fn -> Ch.Test.query("drop table create_example") end) end test "create with options", ctx do assert {:error, %Ch.Error{code: 164, message: message}} = - query(ctx, "create table create_example(a UInt8) engine = Memory", [], + parameterize_query(ctx, "create table create_example(a UInt8) engine = Memory", [], settings: [readonly: 1] ) @@ -258,7 +279,7 @@ defmodule Ch.ConnectionTest do setup ctx do table = "insert_t_#{System.unique_integer([:positive])}" - query!( + parameterize_query!( ctx, "create table #{table}(a UInt8 default 1, b String) engine = Memory" ) @@ -268,26 +289,28 @@ defmodule Ch.ConnectionTest do test "values", %{table: table} = ctx do assert {:ok, %{num_rows: 3}} = - query( + parameterize_query( ctx, "insert into {table:Identifier} values (1, 'a'),(2,'b'), (null, null)", %{"table" => table} ) assert {:ok, %{rows: rows}} = - query(ctx, "select * from {table:Identifier}", %{"table" => table}) + parameterize_query(ctx, "select * from {table:Identifier}", %{"table" => table}) assert rows == [[1, "a"], [2, "b"], [1, ""]] assert {:ok, %{num_rows: 2}} = - query( + parameterize_query( ctx, "insert into {$0:Identifier}(a, b) values ({$1:UInt8},{$2:String}),({$3:UInt8},{$4:String})", [table, 4, "d", 5, "e"] ) assert {:ok, %{rows: rows}} = - query(ctx, "select * from {table:Identifier} where a >= 4", %{"table" => table}) + parameterize_query(ctx, "select * from {table:Identifier} where a >= 4", %{ + "table" => table + }) assert rows == [[4, "d"], [5, "e"]] end @@ -296,7 +319,7 @@ defmodule Ch.ConnectionTest do settings = [readonly: 1] assert {:error, %Ch.Error{code: 164, message: message}} = - query( + parameterize_query( ctx, "insert into {table:Identifier} values (1, 'a'), (2, 'b')", %{"table" => table}, @@ -310,10 +333,10 @@ defmodule Ch.ConnectionTest do stmt = "insert into #{table}(a, b) format RowBinary" types = ["UInt8", "String"] rows = [[1, "a"], [2, "b"]] - assert %{num_rows: 2} = query!(ctx, stmt, rows, types: types) + assert %{num_rows: 2} = parameterize_query!(ctx, stmt, rows, types: types) assert %{rows: rows} = - query!(ctx, "select * from {table:Identifier}", %{"table" => table}) + parameterize_query!(ctx, "select * from {table:Identifier}", %{"table" => table}) assert rows == [[1, "a"], [2, "b"]] end @@ -325,10 +348,10 @@ defmodule Ch.ConnectionTest do rows = [[1, "a"], [2, "b"]] data = RowBinary.encode_rows(rows, types) - assert %{num_rows: 2} = query!(ctx, stmt, data, encode: false) + assert %{num_rows: 2} = parameterize_query!(ctx, stmt, data, encode: false) assert %{rows: rows} = - query!(ctx, "select * from {table:Identifier}", %{"table" => table}) + parameterize_query!(ctx, "select * from {table:Identifier}", %{"table" => table}) assert rows == [[1, "a"], [2, "b"]] end @@ -343,7 +366,7 @@ defmodule Ch.ConnectionTest do |> Stream.map(fn chunk -> RowBinary.encode_rows(chunk, types) end) assert {:ok, %{num_rows: 3}} = - query( + parameterize_query( ctx, "insert into #{table}(a, b) format RowBinary", stream, @@ -351,112 +374,130 @@ defmodule Ch.ConnectionTest do ) assert {:ok, %{rows: rows}} = - query(ctx, "select * from {table:Identifier}", %{"table" => table}) + parameterize_query(ctx, "select * from {table:Identifier}", %{"table" => table}) assert rows == [[1, "a"], [2, "b"], [3, "c"]] end test "select", %{table: table} = ctx do assert {:ok, %{num_rows: 3}} = - query( + parameterize_query( ctx, "insert into {table:Identifier} values (1, 'a'), (2, 'b'), (null, null)", %{"table" => table} ) assert {:ok, %{num_rows: 3}} = - query( + parameterize_query( ctx, "insert into {table:Identifier}(a, b) select a, b from {table:Identifier}", %{"table" => table} ) assert {:ok, %{rows: rows}} = - query(ctx, "select * from {table:Identifier}", %{"table" => table}) + parameterize_query(ctx, "select * from {table:Identifier}", %{"table" => table}) assert rows == [[1, "a"], [2, "b"], [1, ""], [1, "a"], [2, "b"], [1, ""]] assert {:ok, %{num_rows: 2}} = - query( + parameterize_query( ctx, "insert into {$0:Identifier}(a, b) select a, b from {$0:Identifier} where a > {$1:UInt8}", [table, 1] ) assert {:ok, %{rows: new_rows}} = - query(ctx, "select * from {table:Identifier}", %{"table" => table}) + parameterize_query(ctx, "select * from {table:Identifier}", %{"table" => table}) assert new_rows -- rows == [[2, "b"], [2, "b"]] end end test "delete", ctx do - query!(ctx, "create table delete_t(a UInt8, b String) engine = MergeTree order by tuple()") + parameterize_query!( + ctx, + "create table delete_t(a UInt8, b String) engine = MergeTree order by tuple()" + ) + on_exit(fn -> Ch.Test.query("drop table delete_t") end) - assert {:ok, %{num_rows: 2}} = query(ctx, "insert into delete_t values (1,'a'), (2,'b')") + assert {:ok, %{num_rows: 2}} = + parameterize_query(ctx, "insert into delete_t values (1,'a'), (2,'b')") settings = [allow_experimental_lightweight_delete: 1] assert {:ok, %{rows: [], data: [], command: :delete}} = - query(ctx, "delete from delete_t where 1", [], settings: settings) + parameterize_query(ctx, "delete from delete_t where 1", [], settings: settings) end test "query!", ctx do - assert %{num_rows: 1, rows: [[1]]} = query!(ctx, "select 1") + assert %{num_rows: 1, rows: [[1]]} = parameterize_query!(ctx, "select 1") end describe "types" do test "multiple types", ctx do assert {:ok, %{num_rows: 1, rows: [[1, "a"]]}} = - query(ctx, "select {a:Int8}, {b:String}", %{"a" => 1, "b" => "a"}) + parameterize_query(ctx, "select {a:Int8}, {b:String}", %{"a" => 1, "b" => "a"}) end test "ints", ctx do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:Int8}", %{"a" => 1}) + assert {:ok, %{num_rows: 1, rows: [[1]]}} = + parameterize_query(ctx, "select {a:Int8}", %{"a" => 1}) assert {:ok, %{num_rows: 1, rows: [[-1000]]}} = - query(ctx, "select {a:Int16}", %{"a" => -1000}) + parameterize_query(ctx, "select {a:Int16}", %{"a" => -1000}) assert {:ok, %{num_rows: 1, rows: [[100_000]]}} = - query(ctx, "select {a:Int32}", %{"a" => 100_000}) + parameterize_query(ctx, "select {a:Int32}", %{"a" => 100_000}) - assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:Int64}", %{"a" => 1}) - assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:Int128}", %{"a" => 1}) - assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:Int256}", %{"a" => 1}) + assert {:ok, %{num_rows: 1, rows: [[1]]}} = + parameterize_query(ctx, "select {a:Int64}", %{"a" => 1}) + + assert {:ok, %{num_rows: 1, rows: [[1]]}} = + parameterize_query(ctx, "select {a:Int128}", %{"a" => 1}) + + assert {:ok, %{num_rows: 1, rows: [[1]]}} = + parameterize_query(ctx, "select {a:Int256}", %{"a" => 1}) end test "uints", ctx do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:UInt8}", %{"a" => 1}) - assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:UInt16}", %{"a" => 1}) - assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:UInt32}", %{"a" => 1}) - assert {:ok, %{num_rows: 1, rows: [[1]]}} = query(ctx, "select {a:UInt64}", %{"a" => 1}) + assert {:ok, %{num_rows: 1, rows: [[1]]}} = + parameterize_query(ctx, "select {a:UInt8}", %{"a" => 1}) + + assert {:ok, %{num_rows: 1, rows: [[1]]}} = + parameterize_query(ctx, "select {a:UInt16}", %{"a" => 1}) + + assert {:ok, %{num_rows: 1, rows: [[1]]}} = + parameterize_query(ctx, "select {a:UInt32}", %{"a" => 1}) assert {:ok, %{num_rows: 1, rows: [[1]]}} = - query(ctx, "select {a:UInt128}", %{"a" => 1}) + parameterize_query(ctx, "select {a:UInt64}", %{"a" => 1}) assert {:ok, %{num_rows: 1, rows: [[1]]}} = - query(ctx, "select {a:UInt256}", %{"a" => 1}) + parameterize_query(ctx, "select {a:UInt128}", %{"a" => 1}) + + assert {:ok, %{num_rows: 1, rows: [[1]]}} = + parameterize_query(ctx, "select {a:UInt256}", %{"a" => 1}) end test "fixed string", ctx do assert {:ok, %{num_rows: 1, rows: [[<<0, 0>>]]}} = - query(ctx, "select {a:FixedString(2)}", %{"a" => ""}) + parameterize_query(ctx, "select {a:FixedString(2)}", %{"a" => ""}) assert {:ok, %{num_rows: 1, rows: [["a" <> <<0>>]]}} = - query(ctx, "select {a:FixedString(2)}", %{"a" => "a"}) + parameterize_query(ctx, "select {a:FixedString(2)}", %{"a" => "a"}) assert {:ok, %{num_rows: 1, rows: [["aa"]]}} = - query(ctx, "select {a:FixedString(2)}", %{"a" => "aa"}) + parameterize_query(ctx, "select {a:FixedString(2)}", %{"a" => "aa"}) assert {:ok, %{num_rows: 1, rows: [["aaaaa"]]}} = - query(ctx, "select {a:FixedString(5)}", %{"a" => "aaaaa"}) + parameterize_query(ctx, "select {a:FixedString(5)}", %{"a" => "aaaaa"}) - query!(ctx, "create table fixed_string_t(a FixedString(3)) engine = Memory") + parameterize_query!(ctx, "create table fixed_string_t(a FixedString(3)) engine = Memory") on_exit(fn -> Ch.Test.query("drop table fixed_string_t") end) assert {:ok, %{num_rows: 4}} = - query( + parameterize_query( ctx, "insert into fixed_string_t(a) format RowBinary", [ @@ -468,7 +509,7 @@ defmodule Ch.ConnectionTest do types: ["FixedString(3)"] ) - assert query!(ctx, "select * from fixed_string_t").rows == [ + assert parameterize_query!(ctx, "select * from fixed_string_t").rows == [ [<<0, 0, 0>>], ["a" <> <<0, 0>>], ["aa" <> <<0>>], @@ -478,30 +519,30 @@ defmodule Ch.ConnectionTest do test "decimal", ctx do assert {:ok, %{num_rows: 1, rows: [row]}} = - query(ctx, "SELECT toDecimal32(2, 4) AS x, x / 3, toTypeName(x)") + parameterize_query(ctx, "SELECT toDecimal32(2, 4) AS x, x / 3, toTypeName(x)") assert row == [Decimal.new("2.0000"), Decimal.new("0.6666"), "Decimal(9, 4)"] assert {:ok, %{num_rows: 1, rows: [row]}} = - query(ctx, "SELECT toDecimal64(2, 4) AS x, x / 3, toTypeName(x)") + parameterize_query(ctx, "SELECT toDecimal64(2, 4) AS x, x / 3, toTypeName(x)") assert row == [Decimal.new("2.0000"), Decimal.new("0.6666"), "Decimal(18, 4)"] assert {:ok, %{num_rows: 1, rows: [row]}} = - query(ctx, "SELECT toDecimal128(2, 4) AS x, x / 3, toTypeName(x)") + parameterize_query(ctx, "SELECT toDecimal128(2, 4) AS x, x / 3, toTypeName(x)") assert row == [Decimal.new("2.0000"), Decimal.new("0.6666"), "Decimal(38, 4)"] assert {:ok, %{num_rows: 1, rows: [row]}} = - query(ctx, "SELECT toDecimal256(2, 4) AS x, x / 3, toTypeName(x)") + parameterize_query(ctx, "SELECT toDecimal256(2, 4) AS x, x / 3, toTypeName(x)") assert row == [Decimal.new("2.0000"), Decimal.new("0.6666"), "Decimal(76, 4)"] - query!(ctx, "create table decimal_t(d Decimal32(4)) engine = Memory") + parameterize_query!(ctx, "create table decimal_t(d Decimal32(4)) engine = Memory") on_exit(fn -> Ch.Test.query("drop table decimal_t") end) assert %{num_rows: 3} = - query!( + parameterize_query!( ctx, "insert into decimal_t(d) format RowBinary", _rows = [ @@ -512,7 +553,7 @@ defmodule Ch.ConnectionTest do types: ["Decimal32(4)"] ) - assert query!(ctx, "select * from decimal_t").rows == [ + assert parameterize_query!(ctx, "select * from decimal_t").rows == [ [Decimal.new("2.6600")], [Decimal.new("2.6666")], [Decimal.new("2.6667")] @@ -521,19 +562,20 @@ defmodule Ch.ConnectionTest do test "boolean", ctx do assert {:ok, %{num_rows: 1, rows: [[true, "Bool"]]}} = - query(ctx, "select true as col, toTypeName(col)") + parameterize_query(ctx, "select true as col, toTypeName(col)") assert {:ok, %{num_rows: 1, rows: [[1, "UInt8"]]}} = - query(ctx, "select true == 1 as col, toTypeName(col)") + parameterize_query(ctx, "select true == 1 as col, toTypeName(col)") - assert {:ok, %{num_rows: 1, rows: [[true, false]]}} = query(ctx, "select true, false") + assert {:ok, %{num_rows: 1, rows: [[true, false]]}} = + parameterize_query(ctx, "select true, false") - query!(ctx, "create table test_bool(A Int64, B Bool) engine = Memory") + parameterize_query!(ctx, "create table test_bool(A Int64, B Bool) engine = Memory") on_exit(fn -> Ch.Test.query("drop table test_bool") end) - query!(ctx, "INSERT INTO test_bool VALUES (1, true),(2,0)") + parameterize_query!(ctx, "INSERT INTO test_bool VALUES (1, true),(2,0)") - query!( + parameterize_query!( ctx, "insert into test_bool(A, B) format RowBinary", _rows = [[3, true], [4, false]], @@ -541,7 +583,7 @@ defmodule Ch.ConnectionTest do ) # anything > 0 is `true`, here `2` is `true` - query!(ctx, "insert into test_bool(A, B) values (5, 2)") + parameterize_query!(ctx, "insert into test_bool(A, B) values (5, 2)") assert %{ rows: [ @@ -551,15 +593,15 @@ defmodule Ch.ConnectionTest do [4, false, 0], [5, true, 5] ] - } = query!(ctx, "SELECT *, A * B FROM test_bool ORDER BY A") + } = parameterize_query!(ctx, "SELECT *, A * B FROM test_bool ORDER BY A") end test "uuid", ctx do assert {:ok, %{num_rows: 1, rows: [[<<_::16-bytes>>]]}} = - query(ctx, "select generateUUIDv4()") + parameterize_query(ctx, "select generateUUIDv4()") assert {:ok, %{num_rows: 1, rows: [[uuid, "417ddc5d-e556-4d27-95dd-a34d84e46a50"]]}} = - query(ctx, "select {uuid:UUID} as u, toString(u)", %{ + parameterize_query(ctx, "select {uuid:UUID} as u, toString(u)", %{ "uuid" => "417ddc5d-e556-4d27-95dd-a34d84e46a50" }) @@ -568,17 +610,17 @@ defmodule Ch.ConnectionTest do |> String.replace("-", "") |> Base.decode16!(case: :lower) - query!(ctx, " CREATE TABLE t_uuid (x UUID, y String) ENGINE Memory") + parameterize_query!(ctx, " CREATE TABLE t_uuid (x UUID, y String) ENGINE Memory") on_exit(fn -> Ch.Test.query("drop table t_uuid") end) - query!(ctx, "INSERT INTO t_uuid SELECT generateUUIDv4(), 'Example 1'") + parameterize_query!(ctx, "INSERT INTO t_uuid SELECT generateUUIDv4(), 'Example 1'") assert {:ok, %{num_rows: 1, rows: [[<<_::16-bytes>>, "Example 1"]]}} = - query(ctx, "SELECT * FROM t_uuid") + parameterize_query(ctx, "SELECT * FROM t_uuid") - query!(ctx, "INSERT INTO t_uuid (y) VALUES ('Example 2')") + parameterize_query!(ctx, "INSERT INTO t_uuid (y) VALUES ('Example 2')") - query!( + parameterize_query!( ctx, "insert into t_uuid(x,y) format RowBinary", _rows = [[uuid, "Example 3"]], @@ -593,27 +635,32 @@ defmodule Ch.ConnectionTest do [<<0::128>>, "Example 2"], [^uuid, "Example 3"] ] - }} = query(ctx, "SELECT * FROM t_uuid ORDER BY y") + }} = parameterize_query(ctx, "SELECT * FROM t_uuid ORDER BY y") end @tag :skip test "json", ctx do settings = [allow_experimental_object_type: 1] - query!(ctx, "CREATE TABLE json(o JSON) ENGINE = Memory", [], settings: settings) + parameterize_query!(ctx, "CREATE TABLE json(o JSON) ENGINE = Memory", [], + settings: settings + ) - query!(ctx, ~s|INSERT INTO json VALUES ('{"a": 1, "b": { "c": 2, "d": [1, 2, 3] }}')|) + parameterize_query!( + ctx, + ~s|INSERT INTO json VALUES ('{"a": 1, "b": { "c": 2, "d": [1, 2, 3] }}')| + ) - assert query!(ctx, "SELECT o.a, o.b.c, o.b.d[3] FROM json").rows == [[1, 2, 3]] + assert parameterize_query!(ctx, "SELECT o.a, o.b.c, o.b.d[3] FROM json").rows == [[1, 2, 3]] # named tuples are not supported yet - assert_raise ArgumentError, fn -> query!(ctx, "SELECT o FROM json") end + assert_raise ArgumentError, fn -> parameterize_query!(ctx, "SELECT o FROM json") end end @tag :json test "json as string", ctx do # after v25 ClickHouse started rendering numbers in JSON as strings - [[version]] = query!(ctx, "select version()").rows + [[version]] = parameterize_query!(ctx, "select version()").rows parse_version = fn version -> version |> String.split(".") |> Enum.map(&String.to_integer/1) @@ -635,17 +682,17 @@ defmodule Ch.ConnectionTest do ] end - assert query!(ctx, ~s|select '{"answer":42}'::JSON::String|, [], + assert parameterize_query!(ctx, ~s|select '{"answer":42}'::JSON::String|, [], settings: [enable_json_type: 1] ).rows == expected1 - query!(ctx, "CREATE TABLE test_json_as_string(json JSON) ENGINE = Memory", [], + parameterize_query!(ctx, "CREATE TABLE test_json_as_string(json JSON) ENGINE = Memory", [], settings: [enable_json_type: 1] ) on_exit(fn -> Ch.Test.query("DROP TABLE test_json_as_string") end) - query!( + parameterize_query!( ctx, "INSERT INTO test_json_as_string(json) FORMAT RowBinary", _rows = [[Jason.encode_to_iodata!(%{"a" => 42})], [Jason.encode_to_iodata!(%{"b" => 10})]], @@ -656,7 +703,7 @@ defmodule Ch.ConnectionTest do ] ) - assert query!(ctx, "select json::String from test_json_as_string", [], + assert parameterize_query!(ctx, "select json::String from test_json_as_string", [], settings: [enable_json_type: 1] ).rows == expected2 end @@ -665,78 +712,90 @@ defmodule Ch.ConnectionTest do test "enum8", ctx do assert {:ok, %{num_rows: 1, rows: [["Enum8('a' = 1, 'b' = 2)"]]}} = - query(ctx, "SELECT toTypeName(CAST('a', 'Enum(\\'a\\' = 1, \\'b\\' = 2)'))") + parameterize_query( + ctx, + "SELECT toTypeName(CAST('a', 'Enum(\\'a\\' = 1, \\'b\\' = 2)'))" + ) assert {:ok, %{num_rows: 1, rows: [["a"]]}} = - query(ctx, "SELECT CAST('a', 'Enum(\\'a\\' = 1, \\'b\\' = 2)')") + parameterize_query(ctx, "SELECT CAST('a', 'Enum(\\'a\\' = 1, \\'b\\' = 2)')") assert {:ok, %{num_rows: 1, rows: [["b"]]}} = - query(ctx, "select {enum:Enum('a' = 1, 'b' = 2)}", %{"enum" => "b"}) + parameterize_query(ctx, "select {enum:Enum('a' = 1, 'b' = 2)}", %{"enum" => "b"}) assert {:ok, %{num_rows: 1, rows: [["b"]]}} = - query(ctx, "select {enum:Enum('a' = 1, 'b' = 2)}", %{"enum" => 2}) + parameterize_query(ctx, "select {enum:Enum('a' = 1, 'b' = 2)}", %{"enum" => 2}) assert {:ok, %{num_rows: 1, rows: [["b"]]}} = - query(ctx, "select {enum:Enum16('a' = 1, 'b' = 2)}", %{"enum" => 2}) + parameterize_query(ctx, "select {enum:Enum16('a' = 1, 'b' = 2)}", %{"enum" => 2}) - query!( + parameterize_query!( ctx, "CREATE TABLE t_enum(i UInt8, x Enum('hello' = 1, 'world' = 2)) ENGINE Memory" ) on_exit(fn -> Ch.Test.query("DROP TABLE t_enum") end) - query!(ctx, "INSERT INTO t_enum VALUES (0, 'hello'), (1, 'world'), (2, 'hello')") + parameterize_query!( + ctx, + "INSERT INTO t_enum VALUES (0, 'hello'), (1, 'world'), (2, 'hello')" + ) - assert query!(ctx, "SELECT *, CAST(x, 'Int8') FROM t_enum ORDER BY i").rows == [ - [0, "hello", 1], - [1, "world", 2], - [2, "hello", 1] - ] + assert parameterize_query!(ctx, "SELECT *, CAST(x, 'Int8') FROM t_enum ORDER BY i").rows == + [ + [0, "hello", 1], + [1, "world", 2], + [2, "hello", 1] + ] - query!( + parameterize_query!( ctx, "INSERT INTO t_enum(i, x) FORMAT RowBinary", _rows = [[3, "hello"], [4, "world"], [5, 1], [6, 2]], types: ["UInt8", "Enum8('hello' = 1, 'world' = 2)"] ) - assert query!(ctx, "SELECT *, CAST(x, 'Int8') FROM t_enum ORDER BY i").rows == [ - [0, "hello", 1], - [1, "world", 2], - [2, "hello", 1], - [3, "hello", 1], - [4, "world", 2], - [5, "hello", 1], - [6, "world", 2] - ] + assert parameterize_query!(ctx, "SELECT *, CAST(x, 'Int8') FROM t_enum ORDER BY i").rows == + [ + [0, "hello", 1], + [1, "world", 2], + [2, "hello", 1], + [3, "hello", 1], + [4, "world", 2], + [5, "hello", 1], + [6, "world", 2] + ] # TODO nil enum end test "map", ctx do - assert query!( + assert parameterize_query!( ctx, "SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map" ).rows == [[%{1 => "Ready", 2 => "Steady", 3 => "Go"}]] - assert query!(ctx, "select {map:Map(String, UInt8)}", %{ + assert parameterize_query!(ctx, "select {map:Map(String, UInt8)}", %{ "map" => %{"pg" => 13, "hello" => 100} }).rows == [[%{"hello" => 100, "pg" => 13}]] - query!(ctx, "CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory") + parameterize_query!(ctx, "CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory") on_exit(fn -> Ch.Test.query("DROP TABLE table_map") end) - query!( + parameterize_query!( ctx, "INSERT INTO table_map VALUES ({'key1':1, 'key2':10}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30})" ) - assert query!(ctx, "SELECT a['key2'] FROM table_map").rows == [[10], [20], [30]] + assert parameterize_query!(ctx, "SELECT a['key2'] FROM table_map").rows == [ + [10], + [20], + [30] + ] - assert query!(ctx, "INSERT INTO table_map VALUES ({'key3':100}), ({})") + assert parameterize_query!(ctx, "INSERT INTO table_map VALUES ({'key3':100}), ({})") - assert query!(ctx, "SELECT a['key3'] FROM table_map ORDER BY 1 DESC").rows == [ + assert parameterize_query!(ctx, "SELECT a['key3'] FROM table_map ORDER BY 1 DESC").rows == [ [100], [0], [0], @@ -744,7 +803,7 @@ defmodule Ch.ConnectionTest do [0] ] - assert query!( + assert parameterize_query!( ctx, "INSERT INTO table_map FORMAT RowBinary", _rows = [ @@ -760,7 +819,7 @@ defmodule Ch.ConnectionTest do types: ["Map(String, UInt64)"] ) - assert query!(ctx, "SELECT * FROM table_map ORDER BY a ASC").rows == [ + assert parameterize_query!(ctx, "SELECT * FROM table_map ORDER BY a ASC").rows == [ [%{}], [%{}], [%{}], @@ -775,36 +834,36 @@ defmodule Ch.ConnectionTest do end test "tuple", ctx do - assert query!(ctx, "SELECT tuple(1,'a') AS x, toTypeName(x)").rows == [ + assert parameterize_query!(ctx, "SELECT tuple(1,'a') AS x, toTypeName(x)").rows == [ [{1, "a"}, "Tuple(UInt8, String)"] ] - assert query!(ctx, "SELECT {$0:Tuple(Int8, String)}", [{-1, "abs"}]).rows == [ + assert parameterize_query!(ctx, "SELECT {$0:Tuple(Int8, String)}", [{-1, "abs"}]).rows == [ [{-1, "abs"}] ] - assert query!(ctx, "SELECT tuple('a') AS x").rows == [[{"a"}]] + assert parameterize_query!(ctx, "SELECT tuple('a') AS x").rows == [[{"a"}]] - assert query!(ctx, "SELECT tuple(1, NULL) AS x, toTypeName(x)").rows == [ + assert parameterize_query!(ctx, "SELECT tuple(1, NULL) AS x, toTypeName(x)").rows == [ [{1, nil}, "Tuple(UInt8, Nullable(Nothing))"] ] # TODO named tuples - query!(ctx, "CREATE TABLE tuples_t (`a` Tuple(String, Int64)) ENGINE = Memory") + parameterize_query!(ctx, "CREATE TABLE tuples_t (`a` Tuple(String, Int64)) ENGINE = Memory") on_exit(fn -> Ch.Test.query("DROP TABLE tuples_t") end) assert %{num_rows: 2} = - query!(ctx, "INSERT INTO tuples_t VALUES (('y', 10)), (('x',-10))") + parameterize_query!(ctx, "INSERT INTO tuples_t VALUES (('y', 10)), (('x',-10))") assert %{num_rows: 2} = - query!( + parameterize_query!( ctx, "INSERT INTO tuples_t FORMAT RowBinary", _rows = [[{"a", 20}], [{"b", 30}]], types: ["Tuple(String, Int64)"] ) - assert query!(ctx, "SELECT a FROM tuples_t ORDER BY a.1 ASC").rows == [ + assert parameterize_query!(ctx, "SELECT a FROM tuples_t ORDER BY a.1 ASC").rows == [ [{"a", 20}], [{"b", 30}], [{"x", -10}], @@ -813,17 +872,20 @@ defmodule Ch.ConnectionTest do end test "datetime", ctx do - query!( + parameterize_query!( ctx, "CREATE TABLE dt(`timestamp` DateTime('Asia/Istanbul'), `event_id` UInt8) ENGINE = Memory" ) on_exit(fn -> Ch.Test.query("DROP TABLE dt") end) - query!(ctx, "INSERT INTO dt Values (1546300800, 1), ('2019-01-01 00:00:00', 2)") + parameterize_query!( + ctx, + "INSERT INTO dt Values (1546300800, 1), ('2019-01-01 00:00:00', 2)" + ) assert {:ok, %{num_rows: 2, rows: rows}} = - query(ctx, "SELECT *, toString(timestamp) FROM dt") + parameterize_query(ctx, "SELECT *, toString(timestamp) FROM dt") assert rows == [ [ @@ -845,7 +907,7 @@ defmodule Ch.ConnectionTest do # https://kb.altinity.com/altinity-kb-queries-and-syntax/time-zones/ assert {:ok, %{num_rows: 1, rows: [[naive_datetime, "2022-12-12 12:00:00"]], headers: headers}} = - query(ctx, "select {$0:DateTime} as d, toString(d)", [naive_noon]) + parameterize_query(ctx, "select {$0:DateTime} as d, toString(d)", [naive_noon]) # to make this test pass for contributors with non UTC timezone we perform the same steps as ClickHouse # i.e. we give server timezone to the naive datetime and shift it to UTC before comparing with the result @@ -858,10 +920,12 @@ defmodule Ch.ConnectionTest do |> DateTime.to_naive() assert {:ok, %{num_rows: 1, rows: [[~U[2022-12-12 12:00:00Z], "2022-12-12 12:00:00"]]}} = - query(ctx, "select {$0:DateTime('UTC')} as d, toString(d)", [naive_noon]) + parameterize_query(ctx, "select {$0:DateTime('UTC')} as d, toString(d)", [ + naive_noon + ]) assert {:ok, %{num_rows: 1, rows: rows}} = - query(ctx, "select {$0:DateTime('Asia/Bangkok')} as d, toString(d)", [ + parameterize_query(ctx, "select {$0:DateTime('Asia/Bangkok')} as d, toString(d)", [ naive_noon ]) @@ -878,22 +942,26 @@ defmodule Ch.ConnectionTest do on_exit(fn -> Calendar.put_time_zone_database(prev_tz_db) end) assert_raise ArgumentError, ~r/:utc_only_time_zone_database/, fn -> - query(ctx, "select {$0:DateTime('Asia/Tokyo')}", [naive_noon]) + parameterize_query(ctx, "select {$0:DateTime('Asia/Tokyo')}", [naive_noon]) end end # TODO are negatives correct? what's the range? test "date32", ctx do - query!(ctx, "CREATE TABLE new(`timestamp` Date32, `event_id` UInt8) ENGINE = Memory;") + parameterize_query!( + ctx, + "CREATE TABLE new(`timestamp` Date32, `event_id` UInt8) ENGINE = Memory;" + ) + on_exit(fn -> Ch.Test.query("DROP TABLE new") end) - query!(ctx, "INSERT INTO new VALUES (4102444800, 1), ('2100-01-01', 2)") + parameterize_query!(ctx, "INSERT INTO new VALUES (4102444800, 1), ('2100-01-01', 2)") assert {:ok, %{ num_rows: 2, rows: [first_event, [~D[2100-01-01], 2, "2100-01-01"]] - }} = query(ctx, "SELECT *, toString(timestamp) FROM new") + }} = parameterize_query(ctx, "SELECT *, toString(timestamp) FROM new") # TODO use timezone info to be more exact assert first_event in [ @@ -902,17 +970,17 @@ defmodule Ch.ConnectionTest do ] assert {:ok, %{num_rows: 1, rows: [[~D[1900-01-01], "1900-01-01"]]}} = - query(ctx, "select {$0:Date32} as d, toString(d)", [~D[1900-01-01]]) + parameterize_query(ctx, "select {$0:Date32} as d, toString(d)", [~D[1900-01-01]]) # max assert {:ok, %{num_rows: 1, rows: [[~D[2299-12-31], "2299-12-31"]]}} = - query(ctx, "select {$0:Date32} as d, toString(d)", [~D[2299-12-31]]) + parameterize_query(ctx, "select {$0:Date32} as d, toString(d)", [~D[2299-12-31]]) # min assert {:ok, %{num_rows: 1, rows: [[~D[1900-01-01], "1900-01-01"]]}} = - query(ctx, "select {$0:Date32} as d, toString(d)", [~D[1900-01-01]]) + parameterize_query(ctx, "select {$0:Date32} as d, toString(d)", [~D[1900-01-01]]) - query!( + parameterize_query!( ctx, "insert into new(timestamp, event_id) format RowBinary", _rows = [[~D[1960-01-01], 3]], @@ -926,7 +994,11 @@ defmodule Ch.ConnectionTest do [~D[2100-01-01], 2, "2100-01-01"], [~D[1960-01-01], 3, "1960-01-01"] ] - } = query!(ctx, "SELECT *, toString(timestamp) FROM new ORDER BY event_id") + } = + parameterize_query!( + ctx, + "SELECT *, toString(timestamp) FROM new ORDER BY event_id" + ) # TODO use timezone info to be more exact assert first_event in [ @@ -935,7 +1007,7 @@ defmodule Ch.ConnectionTest do ] assert %{num_rows: 1, rows: [[3]]} = - query!(ctx, "SELECT event_id FROM new WHERE timestamp = '1960-01-01'") + parameterize_query!(ctx, "SELECT event_id FROM new WHERE timestamp = '1960-01-01'") end # https://clickhouse.com/docs/sql-reference/data-types/time @@ -943,7 +1015,10 @@ defmodule Ch.ConnectionTest do test "time", ctx do settings = [enable_time_time64_type: 1] - query!(ctx, "CREATE TABLE time_t(`time` Time, `event_id` UInt8) ENGINE = Memory", [], + parameterize_query!( + ctx, + "CREATE TABLE time_t(`time` Time, `event_id` UInt8) ENGINE = Memory", + [], settings: settings ) @@ -951,7 +1026,7 @@ defmodule Ch.ConnectionTest do Ch.Test.query("DROP TABLE time_t", [], settings: settings) end) - query!(ctx, "INSERT INTO time_t VALUES ('100:00:00', 1), (12453, 2)", [], + parameterize_query!(ctx, "INSERT INTO time_t VALUES ('100:00:00', 1), (12453, 2)", [], settings: settings ) @@ -961,9 +1036,11 @@ defmodule Ch.ConnectionTest do assert_raise ArgumentError, "ClickHouse Time value 3.6e5 (seconds) is out of Elixir's Time range (00:00:00.000000 - 23:59:59.999999)", - fn -> query!(ctx, "select * from time_t", [], settings: settings) end + fn -> + parameterize_query!(ctx, "select * from time_t", [], settings: settings) + end - query!( + parameterize_query!( ctx, "INSERT INTO time_t(time, event_id) FORMAT RowBinary", _rows = [ @@ -975,7 +1052,10 @@ defmodule Ch.ConnectionTest do types: ["Time", "UInt8"] ) - assert query!(ctx, "select * from time_t where event_id > 1 order by event_id", [], + assert parameterize_query!( + ctx, + "select * from time_t where event_id > 1 order by event_id", + [], settings: settings ).rows == [[~T[03:27:33], 2], [~T[00:00:00], 3], [~T[12:34:56], 4], [~T[23:59:59], 5]] @@ -986,7 +1066,7 @@ defmodule Ch.ConnectionTest do test "Time64(3)", ctx do settings = [enable_time_time64_type: 1] - query!( + parameterize_query!( ctx, "CREATE TABLE time64_3_t(`time` Time64(3), `event_id` UInt8) ENGINE = Memory", [], @@ -997,7 +1077,7 @@ defmodule Ch.ConnectionTest do Ch.Test.query("DROP TABLE time64_3_t", [], settings: settings) end) - query!( + parameterize_query!( ctx, "INSERT INTO time64_3_t VALUES (15463123, 1), (154600.123, 2), ('100:00:00', 3);", [], @@ -1010,9 +1090,11 @@ defmodule Ch.ConnectionTest do assert_raise ArgumentError, "ClickHouse Time value 154600.123 (seconds) is out of Elixir's Time range (00:00:00.000000 - 23:59:59.999999)", - fn -> query!(ctx, "select * from time64_3_t", [], settings: settings) end + fn -> + parameterize_query!(ctx, "select * from time64_3_t", [], settings: settings) + end - query!( + parameterize_query!( ctx, "INSERT INTO time64_3_t(time, event_id) FORMAT RowBinary", _rows = [ @@ -1026,7 +1108,7 @@ defmodule Ch.ConnectionTest do types: ["Time64(3)", "UInt8"] ) - assert query!( + assert parameterize_query!( ctx, "select * from time64_3_t where time < {max_elixir_time:Time64(6)} order by event_id", %{"max_elixir_time" => ~T[23:59:59.999999]}, @@ -1046,7 +1128,7 @@ defmodule Ch.ConnectionTest do test "Time64(6)", ctx do settings = [enable_time_time64_type: 1] - query!( + parameterize_query!( ctx, "CREATE TABLE time64_6_t(`time` Time64(6), `event_id` UInt8) ENGINE = Memory", [], @@ -1057,7 +1139,7 @@ defmodule Ch.ConnectionTest do Ch.Test.query("DROP TABLE time64_6_t", [], settings: settings) end) - query!( + parameterize_query!( ctx, "INSERT INTO time64_6_t(time, event_id) FORMAT RowBinary", _rows = [ @@ -1071,7 +1153,7 @@ defmodule Ch.ConnectionTest do types: ["Time64(6)", "UInt8"] ) - assert query!( + assert parameterize_query!( ctx, "select * from time64_6_t order by event_id", [], @@ -1090,7 +1172,7 @@ defmodule Ch.ConnectionTest do test "Time64(9)", ctx do settings = [enable_time_time64_type: 1] - query!( + parameterize_query!( ctx, "CREATE TABLE time64_9_t(`time` Time64(9), `event_id` UInt8) ENGINE = Memory", [], @@ -1101,7 +1183,7 @@ defmodule Ch.ConnectionTest do Ch.Test.query("DROP TABLE time64_9_t", [], settings: settings) end) - query!( + parameterize_query!( ctx, "INSERT INTO time64_9_t(time, event_id) FORMAT RowBinary", _rows = [ @@ -1115,7 +1197,7 @@ defmodule Ch.ConnectionTest do types: ["Time64(9)", "UInt8"] ) - assert query!( + assert parameterize_query!( ctx, "select * from time64_9_t order by event_id", [], @@ -1131,20 +1213,20 @@ defmodule Ch.ConnectionTest do end test "datetime64", ctx do - query!( + parameterize_query!( ctx, "CREATE TABLE datetime64_t(`timestamp` DateTime64(3, 'Asia/Istanbul'), `event_id` UInt8) ENGINE = Memory" ) on_exit(fn -> Ch.Test.query("DROP TABLE datetime64_t") end) - query!( + parameterize_query!( ctx, "INSERT INTO datetime64_t Values (1546300800123, 1), (1546300800.123, 2), ('2019-01-01 00:00:00', 3)" ) assert {:ok, %{num_rows: 3, rows: rows}} = - query(ctx, "SELECT *, toString(timestamp) FROM datetime64_t") + parameterize_query(ctx, "SELECT *, toString(timestamp) FROM datetime64_t") assert rows == [ [ @@ -1164,7 +1246,7 @@ defmodule Ch.ConnectionTest do ] ] - query!( + parameterize_query!( ctx, "insert into datetime64_t(event_id, timestamp) format RowBinary", _rows = [ @@ -1175,7 +1257,7 @@ defmodule Ch.ConnectionTest do ) assert {:ok, %{num_rows: 2, rows: rows}} = - query( + parameterize_query( ctx, "SELECT *, toString(timestamp) FROM datetime64_t WHERE timestamp > '2020-01-01'" ) @@ -1200,7 +1282,7 @@ defmodule Ch.ConnectionTest do # see https://clickhouse.com/docs/en/sql-reference/data-types/datetime # https://kb.altinity.com/altinity-kb-queries-and-syntax/time-zones/ assert {:ok, %{num_rows: 1, rows: [[naive_datetime]], headers: headers}} = - query(ctx, "select {$0:DateTime64(#{precision})}", [naive_noon]) + parameterize_query(ctx, "select {$0:DateTime64(#{precision})}", [naive_noon]) # to make this test pass for contributors with non UTC timezone we perform the same steps as ClickHouse # i.e. we give server timezone to the naive datetime and shift it to UTC before comparing with the result @@ -1217,20 +1299,24 @@ defmodule Ch.ConnectionTest do assert {:ok, %{num_rows: 1, rows: [[~U[2022-01-01 12:00:00.123Z], "2022-01-01 12:00:00.123"]]}} = - query(ctx, "select {dt:DateTime64(3,'UTC')} as d, toString(d)", %{ + parameterize_query(ctx, "select {dt:DateTime64(3,'UTC')} as d, toString(d)", %{ "dt" => ~N[2022-01-01 12:00:00.123] }) assert {:ok, %{num_rows: 1, rows: [[~U[1900-01-01 12:00:00.123Z], "1900-01-01 12:00:00.123"]]}} = - query(ctx, "select {dt:DateTime64(3,'UTC')} as d, toString(d)", %{ + parameterize_query(ctx, "select {dt:DateTime64(3,'UTC')} as d, toString(d)", %{ "dt" => ~N[1900-01-01 12:00:00.123] }) assert {:ok, %{num_rows: 1, rows: [row]}} = - query(ctx, "select {dt:DateTime64(3,'Asia/Bangkok')} as d, toString(d)", %{ - "dt" => ~N[2022-01-01 12:00:00.123] - }) + parameterize_query( + ctx, + "select {dt:DateTime64(3,'Asia/Bangkok')} as d, toString(d)", + %{ + "dt" => ~N[2022-01-01 12:00:00.123] + } + ) assert row == [ DateTime.new!(~D[2022-01-01], ~T[12:00:00.123], "Asia/Bangkok"), @@ -1239,24 +1325,24 @@ defmodule Ch.ConnectionTest do end test "nullable", ctx do - query!( + parameterize_query!( ctx, "CREATE TABLE nullable (`n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple()" ) on_exit(fn -> Ch.Test.query("DROP TABLE nullable") end) - query!(ctx, "INSERT INTO nullable VALUES (1) (NULL) (2) (NULL)") + parameterize_query!(ctx, "INSERT INTO nullable VALUES (1) (NULL) (2) (NULL)") assert {:ok, %{num_rows: 4, rows: [[0], [1], [0], [1]]}} = - query(ctx, "SELECT n.null FROM nullable") + parameterize_query(ctx, "SELECT n.null FROM nullable") assert {:ok, %{num_rows: 4, rows: [[1], [nil], [2], [nil]]}} = - query(ctx, "SELECT n FROM nullable") + parameterize_query(ctx, "SELECT n FROM nullable") # weird thing about nullables is that, similar to bool, in binary format, any byte larger than 0 is `null` assert {:ok, %{num_rows: 5}} = - query( + parameterize_query( ctx, "insert into nullable format RowBinary", <<1, 2, 3, 4, 5>>, @@ -1264,13 +1350,13 @@ defmodule Ch.ConnectionTest do ) assert %{num_rows: 1, rows: [[count]]} = - query!(ctx, "select count(*) from nullable where n is null") + parameterize_query!(ctx, "select count(*) from nullable where n is null") assert count == 2 + 5 end test "nullable + default", ctx do - query!(ctx, """ + parameterize_query!(ctx, """ CREATE TABLE ch_nulls ( a UInt8, b UInt8 NULL, @@ -1281,7 +1367,7 @@ defmodule Ch.ConnectionTest do on_exit(fn -> Ch.Test.query("DROP TABLE ch_nulls") end) - query!( + parameterize_query!( ctx, "INSERT INTO ch_nulls(a, b, c, d) FORMAT RowBinary", [[nil, nil, nil, nil]], @@ -1289,12 +1375,12 @@ defmodule Ch.ConnectionTest do ) # default is ignored... - assert query!(ctx, "SELECT * FROM ch_nulls").rows == [[0, nil, 0, nil]] + assert parameterize_query!(ctx, "SELECT * FROM ch_nulls").rows == [[0, nil, 0, nil]] end # based on https://github.com/ClickHouse/clickhouse-java/pull/1345/files test "nullable + input() + default", ctx do - query!(ctx, """ + parameterize_query!(ctx, """ CREATE TABLE test_insert_default_value( n Int32, s String DEFAULT 'secret' @@ -1303,7 +1389,7 @@ defmodule Ch.ConnectionTest do on_exit(fn -> Ch.Test.query("DROP TABLE test_insert_default_value") end) - query!( + parameterize_query!( ctx, """ INSERT INTO test_insert_default_value @@ -1315,7 +1401,7 @@ defmodule Ch.ConnectionTest do types: ["UInt32", "Nullable(String)"] ) - assert query!(ctx, "SELECT * FROM test_insert_default_value ORDER BY n").rows == + assert parameterize_query!(ctx, "SELECT * FROM test_insert_default_value ORDER BY n").rows == [ [-1, "secret"], [1, "secret"] @@ -1323,31 +1409,35 @@ defmodule Ch.ConnectionTest do end test "can decode casted Point", ctx do - assert query!(ctx, "select cast((0, 1) as Point)").rows == [ + assert parameterize_query!(ctx, "select cast((0, 1) as Point)").rows == [ _row = [_point = {0.0, 1.0}] ] end test "can encode and then decode Point in query params", ctx do - assert query!(ctx, "select {$0:Point}", [{10, 10}]).rows == [ + assert parameterize_query!(ctx, "select {$0:Point}", [{10, 10}]).rows == [ _row = [_point = {10.0, 10.0}] ] end test "can insert and select Point", ctx do - query!(ctx, "CREATE TABLE geo_point (p Point) ENGINE = Memory()") + parameterize_query!(ctx, "CREATE TABLE geo_point (p Point) ENGINE = Memory()") on_exit(fn -> Ch.Test.query("DROP TABLE geo_point") end) - query!(ctx, "INSERT INTO geo_point VALUES((10, 10))") - query!(ctx, "INSERT INTO geo_point FORMAT RowBinary", [[{20, 20}]], types: ["Point"]) + parameterize_query!(ctx, "INSERT INTO geo_point VALUES((10, 10))") - assert query!(ctx, "SELECT p, toTypeName(p) FROM geo_point ORDER BY p ASC").rows == [ - [{10.0, 10.0}, "Point"], - [{20.0, 20.0}, "Point"] - ] + parameterize_query!(ctx, "INSERT INTO geo_point FORMAT RowBinary", [[{20, 20}]], + types: ["Point"] + ) + + assert parameterize_query!(ctx, "SELECT p, toTypeName(p) FROM geo_point ORDER BY p ASC").rows == + [ + [{10.0, 10.0}, "Point"], + [{20.0, 20.0}, "Point"] + ] # to make our RowBinary is not garbage in garbage out we also test a text format response - assert query!( + assert parameterize_query!( ctx, "SELECT p, toTypeName(p) FROM geo_point ORDER BY p ASC FORMAT JSONCompact" ).rows @@ -1360,30 +1450,37 @@ defmodule Ch.ConnectionTest do test "can decode casted Ring", ctx do ring = [{0.0, 1.0}, {10.0, 3.0}] - assert query!(ctx, "select cast([(0,1),(10,3)] as Ring)").rows == [_row = [ring]] + + assert parameterize_query!(ctx, "select cast([(0,1),(10,3)] as Ring)").rows == [ + _row = [ring] + ] end test "can encode and then decode Ring in query params", ctx do ring = [{0.0, 1.0}, {10.0, 3.0}] - assert query!(ctx, "select {$0:Ring}", [ring]).rows == [_row = [ring]] + assert parameterize_query!(ctx, "select {$0:Ring}", [ring]).rows == [_row = [ring]] end test "can insert and select Ring", ctx do - query!(ctx, "CREATE TABLE geo_ring (r Ring) ENGINE = Memory()") + parameterize_query!(ctx, "CREATE TABLE geo_ring (r Ring) ENGINE = Memory()") on_exit(fn -> Ch.Test.query("DROP TABLE geo_ring") end) - query!(ctx, "INSERT INTO geo_ring VALUES([(0, 0), (10, 0), (10, 10), (0, 10)])") + parameterize_query!( + ctx, + "INSERT INTO geo_ring VALUES([(0, 0), (10, 0), (10, 10), (0, 10)])" + ) ring = [{20, 20}, {0, 0}, {0, 20}] - query!(ctx, "INSERT INTO geo_ring FORMAT RowBinary", [[ring]], types: ["Ring"]) + parameterize_query!(ctx, "INSERT INTO geo_ring FORMAT RowBinary", [[ring]], types: ["Ring"]) - assert query!(ctx, "SELECT r, toTypeName(r) FROM geo_ring ORDER BY r ASC").rows == [ - [[{0.0, 0.0}, {10.0, 0.0}, {10.0, 10.0}, {0.0, 10.0}], "Ring"], - [[{20.0, 20.0}, {0.0, 0.0}, {0.0, 20.0}], "Ring"] - ] + assert parameterize_query!(ctx, "SELECT r, toTypeName(r) FROM geo_ring ORDER BY r ASC").rows == + [ + [[{0.0, 0.0}, {10.0, 0.0}, {10.0, 10.0}, {0.0, 10.0}], "Ring"], + [[{20.0, 20.0}, {0.0, 0.0}, {0.0, 20.0}], "Ring"] + ] # to make our RowBinary is not garbage in garbage out we also test a text format response - assert query!( + assert parameterize_query!( ctx, "SELECT r, toTypeName(r) FROM geo_ring ORDER BY r ASC FORMAT JSONCompact" ).rows @@ -1397,29 +1494,36 @@ defmodule Ch.ConnectionTest do test "can decode casted Polygon", ctx do polygon = [[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]] - assert query!(ctx, "select cast([[(0,1),(10,3)],[],[(2,2)]] as Polygon)").rows == [ - _row = [polygon] - ] + assert parameterize_query!(ctx, "select cast([[(0,1),(10,3)],[],[(2,2)]] as Polygon)").rows == + [ + _row = [polygon] + ] end test "can encode and then decode Polygon in query params", ctx do polygon = [[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]] - assert query!(ctx, "select {$0:Polygon}", [polygon]).rows == [_row = [polygon]] + assert parameterize_query!(ctx, "select {$0:Polygon}", [polygon]).rows == [_row = [polygon]] end test "can insert and select Polygon", ctx do - query!(ctx, "CREATE TABLE geo_polygon (pg Polygon) ENGINE = Memory()") + parameterize_query!(ctx, "CREATE TABLE geo_polygon (pg Polygon) ENGINE = Memory()") on_exit(fn -> Ch.Test.query("DROP TABLE geo_polygon") end) - query!( + parameterize_query!( ctx, "INSERT INTO geo_polygon VALUES([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]])" ) polygon = [[{0, 1.0}, {10, 3.2}], [], [{2, 2}]] - query!(ctx, "INSERT INTO geo_polygon FORMAT RowBinary", [[polygon]], types: ["Polygon"]) - assert query!(ctx, "SELECT pg, toTypeName(pg) FROM geo_polygon ORDER BY pg ASC").rows == + parameterize_query!(ctx, "INSERT INTO geo_polygon FORMAT RowBinary", [[polygon]], + types: ["Polygon"] + ) + + assert parameterize_query!( + ctx, + "SELECT pg, toTypeName(pg) FROM geo_polygon ORDER BY pg ASC" + ).rows == [ [[[{0.0, 1.0}, {10.0, 3.2}], [], [{2.0, 2.0}]], "Polygon"], [ @@ -1432,7 +1536,7 @@ defmodule Ch.ConnectionTest do ] # to make our RowBinary is not garbage in garbage out we also test a text format response - assert query!( + assert parameterize_query!( ctx, "SELECT pg, toTypeName(pg) FROM geo_polygon ORDER BY pg ASC FORMAT JSONCompact" ).rows @@ -1449,7 +1553,7 @@ defmodule Ch.ConnectionTest do test "can decode casted MultiPolygon", ctx do multipolygon = [[[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]], [], [[{3, 3}]]] - assert query!( + assert parameterize_query!( ctx, "select cast([[[(0,1),(10,3)],[],[(2,2)]],[],[[(3, 3)]]] as MultiPolygon)" ).rows == [ @@ -1460,27 +1564,34 @@ defmodule Ch.ConnectionTest do test "can encode and then decode MultiPolygon in query params", ctx do multipolygon = [[[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]], [], [[{3, 3}]]] - assert query!(ctx, "select {$0:MultiPolygon}", [multipolygon]).rows == [ + assert parameterize_query!(ctx, "select {$0:MultiPolygon}", [multipolygon]).rows == [ _row = [multipolygon] ] end test "can insert and select MultiPolygon", ctx do - query!(ctx, "CREATE TABLE geo_multipolygon (mpg MultiPolygon) ENGINE = Memory()") + parameterize_query!( + ctx, + "CREATE TABLE geo_multipolygon (mpg MultiPolygon) ENGINE = Memory()" + ) + on_exit(fn -> Ch.Test.query("DROP TABLE geo_multipolygon") end) - query!( + parameterize_query!( ctx, "INSERT INTO geo_multipolygon VALUES([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]])" ) multipolygon = [[[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]], [], [[{3, 3}]]] - query!(ctx, "INSERT INTO geo_multipolygon FORMAT RowBinary", [[multipolygon]], + parameterize_query!(ctx, "INSERT INTO geo_multipolygon FORMAT RowBinary", [[multipolygon]], types: ["MultiPolygon"] ) - assert query!(ctx, "SELECT mpg, toTypeName(mpg) FROM geo_multipolygon ORDER BY mpg ASC").rows == + assert parameterize_query!( + ctx, + "SELECT mpg, toTypeName(mpg) FROM geo_multipolygon ORDER BY mpg ASC" + ).rows == [ _row = [ _multipolygon = [ @@ -1511,7 +1622,7 @@ defmodule Ch.ConnectionTest do ] # to make our RowBinary is not garbage in garbage out we also test a text format response - assert query!( + assert parameterize_query!( ctx, "SELECT mpg, toTypeName(mpg) FROM geo_multipolygon ORDER BY mpg ASC FORMAT JSONCompact" ).rows @@ -1534,14 +1645,14 @@ defmodule Ch.ConnectionTest do @tag capture_log: true, skip: true test "can provide custom timeout", ctx do assert {:error, %Mint.TransportError{reason: :timeout} = error} = - query(ctx, "select sleep(1)", _params = [], timeout: 100) + parameterize_query(ctx, "select sleep(1)", _params = [], timeout: 100) assert Exception.message(error) == "timeout" end test "errors on invalid creds", ctx do assert {:error, %Ch.Error{code: 516} = error} = - query(ctx, "select 1 + 1", _params = [], + parameterize_query(ctx, "select 1 + 1", _params = [], username: "no-exists", password: "wrong" ) @@ -1552,7 +1663,7 @@ defmodule Ch.ConnectionTest do test "errors on invalid database", ctx do assert {:error, %Ch.Error{code: 81} = error} = - query(ctx, "select 1 + 1", _params = [], database: "no-db") + parameterize_query(ctx, "select 1 + 1", _params = [], database: "no-db") assert Exception.message(error) =~ "`no-db`" assert Exception.message(error) =~ "UNKNOWN_DATABASE" @@ -1560,7 +1671,7 @@ defmodule Ch.ConnectionTest do test "can provide custom database", ctx do assert {:ok, %{num_rows: 1, rows: [[2]]}} = - query(ctx, "select 1 + 1", [], database: "default") + parameterize_query(ctx, "select 1 + 1", [], database: "default") end end @@ -1568,7 +1679,7 @@ defmodule Ch.ConnectionTest do test "commit", ctx do DBConnection.transaction(ctx.conn, fn conn -> ctx = Map.put(ctx, :conn, conn) - query!(ctx, "select 1 + 1") + parameterize_query!(ctx, "select 1 + 1") end) end @@ -1611,7 +1722,7 @@ defmodule Ch.ConnectionTest do conn |> Ch.stream("select number from system.numbers") |> Enum.take(1) end) - assert query!(ctx, "select 1 + 1").rows == [[2]] + assert parameterize_query!(ctx, "select 1 + 1").rows == [[2]] end) assert logs =~ @@ -1636,20 +1747,20 @@ defmodule Ch.ConnectionTest do {:ok, conn} = Ch.start_link(database: db) ctx = Map.put(ctx, :conn, conn) - query!(ctx, "create table example(a UInt8) engine=Memory") - assert {:ok, %{rows: [["example"]]}} = query(ctx, "show tables") + parameterize_query!(ctx, "create table example(a UInt8) engine=Memory") + assert {:ok, %{rows: [["example"]]}} = parameterize_query(ctx, "show tables") end test "can start without options", ctx do {:ok, conn} = Ch.start_link() ctx = Map.put(ctx, :conn, conn) - assert {:ok, %{num_rows: 1, rows: [[2]]}} = query(ctx, "select 1 + 1") + assert {:ok, %{num_rows: 1, rows: [[2]]}} = parameterize_query(ctx, "select 1 + 1") end end describe "RowBinaryWithNamesAndTypes" do setup ctx do - query!(ctx, """ + parameterize_query!(ctx, """ create table if not exists row_binary_names_and_types_t ( country_code FixedString(2), rare_string LowCardinality(String), @@ -1670,7 +1781,9 @@ defmodule Ch.ConnectionTest do types: [Ch.Types.fixed_string(2), Ch.Types.string(), Ch.Types.nullable(Ch.Types.u32())] ] - assert {:error, %Ch.Error{code: 117, message: message}} = query(ctx, stmt, rows, opts) + assert {:error, %Ch.Error{code: 117, message: message}} = + parameterize_query(ctx, stmt, rows, opts) + assert message =~ "Type of 'rare_string' must be LowCardinality(String), not String" opts = [ @@ -1682,7 +1795,9 @@ defmodule Ch.ConnectionTest do ] ] - assert {:error, %Ch.Error{code: 117, message: message}} = query(ctx, stmt, rows, opts) + assert {:error, %Ch.Error{code: 117, message: message}} = + parameterize_query(ctx, stmt, rows, opts) + assert message =~ "Type of 'maybe_int32' must be Nullable(Int32), not Nullable(UInt32)" end @@ -1700,14 +1815,14 @@ defmodule Ch.ConnectionTest do ] ] - assert {:ok, %{num_rows: 1}} = query(ctx, stmt, rows, opts) + assert {:ok, %{num_rows: 1}} = parameterize_query(ctx, stmt, rows, opts) end test "select with lots of columns", ctx do select = Enum.map_join(1..1000, ", ", fn i -> "#{i} as col_#{i}" end) stmt = "select #{select} format RowBinaryWithNamesAndTypes" - assert %Ch.Result{columns: columns, rows: [row]} = query!(ctx, stmt) + assert %Ch.Result{columns: columns, rows: [row]} = parameterize_query!(ctx, stmt) assert length(columns) == 1000 assert List.first(columns) == "col_1" diff --git a/test/ch/dynamic_test.exs b/test/ch/dynamic_test.exs index 8f9a693..7a916d6 100644 --- a/test/ch/dynamic_test.exs +++ b/test/ch/dynamic_test.exs @@ -1,5 +1,6 @@ defmodule Ch.DynamicTest do - use ExUnit.Case + use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] + import Ch.Test, only: [parameterize_query!: 2, parameterize_query!: 3, parameterize_query!: 4] @moduletag :dynamic @@ -7,24 +8,24 @@ defmodule Ch.DynamicTest do {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} end - test "it works", %{conn: conn} do + test "it works", ctx do select = fn literal -> - [row] = Ch.query!(conn, "select #{literal}::Dynamic as d, dynamicType(d)").rows + [row] = parameterize_query!(ctx, "select #{literal}::Dynamic as d, dynamicType(d)").rows row end - Ch.query!(conn, "CREATE TABLE test (d Dynamic, id String) ENGINE = Memory;") + parameterize_query!(ctx, "CREATE TABLE test (d Dynamic, id String) ENGINE = Memory;") on_exit(fn -> Ch.Test.query("DROP TABLE test") end) insert = fn value -> id = inspect(value) - Ch.query!(conn, "insert into test(d, id) format RowBinary", [[value, id]], + parameterize_query!(ctx, "insert into test(d, id) format RowBinary", [[value, id]], types: ["Dynamic", "String"] ).rows [[inserted]] = - Ch.query!(conn, "select d from test where id = {id:String}", %{"id" => id}).rows + parameterize_query!(ctx, "select d from test where id = {id:String}", %{"id" => id}).rows inserted end @@ -96,7 +97,7 @@ defmodule Ch.DynamicTest do # DateTime 0x11 assert select.("'2020-01-01 12:34:56'::DateTime") == [ - Ch.Test.to_clickhouse_naive(conn, ~N[2020-01-01 12:34:56]), + Ch.Test.to_clickhouse_naive(ctx.conn, ~N[2020-01-01 12:34:56]), "DateTime" ] @@ -110,7 +111,10 @@ defmodule Ch.DynamicTest do # DateTime64(P) 0x13 assert select.("'2020-01-01 12:34:56.123456'::DateTime64(6)") == - [Ch.Test.to_clickhouse_naive(conn, ~N[2020-01-01 12:34:56.123456]), "DateTime64(6)"] + [ + Ch.Test.to_clickhouse_naive(ctx.conn, ~N[2020-01-01 12:34:56.123456]), + "DateTime64(6)" + ] # DateTime64(P, time_zone) 0x14 assert [dt64, "DateTime64(6, 'Europe/Prague')"] = @@ -266,13 +270,17 @@ defmodule Ch.DynamicTest do end # https://clickhouse.com/docs/sql-reference/data-types/dynamic#creating-dynamic - test "creating dynamic", %{conn: conn} do + test "creating dynamic", ctx do # Using Dynamic type in table column definition: - Ch.query!(conn, "CREATE TABLE test (d Dynamic) ENGINE = Memory;") + parameterize_query!(ctx, "CREATE TABLE test (d Dynamic) ENGINE = Memory;") on_exit(fn -> Ch.Test.query("DROP TABLE test") end) - Ch.query!(conn, "INSERT INTO test VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);") - assert Ch.query!(conn, "SELECT d, dynamicType(d) FROM test;").rows == [ + parameterize_query!( + ctx, + "INSERT INTO test VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);" + ) + + assert parameterize_query!(ctx, "SELECT d, dynamicType(d) FROM test;").rows == [ [nil, "None"], [42, "Int64"], ["Hello, World!", "String"], @@ -280,13 +288,14 @@ defmodule Ch.DynamicTest do ] # Using CAST from ordinary column: - assert Ch.query!(conn, "SELECT 'Hello, World!'::Dynamic AS d, dynamicType(d);").rows == [ - ["Hello, World!", "String"] - ] + assert parameterize_query!(ctx, "SELECT 'Hello, World!'::Dynamic AS d, dynamicType(d);").rows == + [ + ["Hello, World!", "String"] + ] # Using CAST from Variant column: - assert Ch.query!( - conn, + assert parameterize_query!( + ctx, "SELECT multiIf((number % 3) = 0, number, (number % 3) = 1, range(number + 1), NULL)::Dynamic AS d, dynamicType(d) FROM numbers(3)", [], settings: [ @@ -301,13 +310,17 @@ defmodule Ch.DynamicTest do end # https://clickhouse.com/docs/sql-reference/data-types/dynamic#reading-dynamic-nested-types-as-subcolumns - test "reading dynamic nested types as subcolumns", %{conn: conn} do - Ch.query!(conn, "CREATE TABLE test (d Dynamic) ENGINE = Memory;") + test "reading dynamic nested types as subcolumns", ctx do + parameterize_query!(ctx, "CREATE TABLE test (d Dynamic) ENGINE = Memory;") on_exit(fn -> Ch.Test.query("DROP TABLE test") end) - Ch.query!(conn, "INSERT INTO test VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);") - assert Ch.query!( - conn, + parameterize_query!( + ctx, + "INSERT INTO test VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);" + ) + + assert parameterize_query!( + ctx, "SELECT d, dynamicType(d), d.String, d.Int64, d.`Array(Int64)`, d.Date, d.`Array(String)` FROM test;" ).rows == [ [nil, "None", nil, nil, [], nil, []], @@ -316,8 +329,8 @@ defmodule Ch.DynamicTest do [[1, 2, 3], "Array(Int64)", nil, nil, [1, 2, 3], nil, []] ] - assert Ch.query!( - conn, + assert parameterize_query!( + ctx, "SELECT toTypeName(d.String), toTypeName(d.Int64), toTypeName(d.`Array(Int64)`), toTypeName(d.Date), toTypeName(d.`Array(String)`) FROM test LIMIT 1;" ).rows == [ [ @@ -329,8 +342,8 @@ defmodule Ch.DynamicTest do ] ] - assert Ch.query!( - conn, + assert parameterize_query!( + ctx, "SELECT d, dynamicType(d), dynamicElement(d, 'String'), dynamicElement(d, 'Int64'), dynamicElement(d, 'Array(Int64)'), dynamicElement(d, 'Date'), dynamicElement(d, 'Array(String)') FROM test;" ).rows == [ [nil, "None", nil, nil, [], nil, []], @@ -341,9 +354,9 @@ defmodule Ch.DynamicTest do end # https://clickhouse.com/docs/sql-reference/data-types/dynamic#converting-a-string-column-to-a-dynamic-column-through-parsing - test "converting a string column to a dynamic column through parsing", %{conn: conn} do - assert Ch.query!( - conn, + test "converting a string column to a dynamic column through parsing", ctx do + assert parameterize_query!( + ctx, "SELECT CAST(materialize(map('key1', '42', 'key2', 'true', 'key3', '2020-01-01')), 'Map(String, Dynamic)') as map_of_dynamic, mapApply((k, v) -> (k, dynamicType(v)), map_of_dynamic) as map_of_dynamic_types;", [], settings: [cast_string_to_dynamic_use_inference: 1] @@ -356,12 +369,12 @@ defmodule Ch.DynamicTest do end # https://clickhouse.com/docs/sql-reference/data-types/dynamic#converting-a-dynamic-column-to-an-ordinary-column - test "converting a dynamic column to an ordinary column", %{conn: conn} do - Ch.query!(conn, "CREATE TABLE test (d Dynamic) ENGINE = Memory;") + test "converting a dynamic column to an ordinary column", ctx do + parameterize_query!(ctx, "CREATE TABLE test (d Dynamic) ENGINE = Memory;") on_exit(fn -> Ch.Test.query("DROP TABLE test") end) - Ch.query!(conn, "INSERT INTO test VALUES (NULL), (42), ('42.42'), (true), ('e10');") + parameterize_query!(ctx, "INSERT INTO test VALUES (NULL), (42), ('42.42'), (true), ('e10');") - assert Ch.query!(conn, "SELECT d::Nullable(Float64) FROM test;").rows == [ + assert parameterize_query!(ctx, "SELECT d::Nullable(Float64) FROM test;").rows == [ [nil], [42.0], [42.42], @@ -371,16 +384,16 @@ defmodule Ch.DynamicTest do end # https://clickhouse.com/docs/sql-reference/data-types/dynamic#converting-a-variant-column-to-dynamic-column - test "converting a variant column to dynamic column", %{conn: conn} do - Ch.query!( - conn, + test "converting a variant column to dynamic column", ctx do + parameterize_query!( + ctx, "CREATE TABLE test (v Variant(UInt64, String, Array(UInt64))) ENGINE = Memory;" ) on_exit(fn -> Ch.Test.query("DROP TABLE test") end) - Ch.query!(conn, "INSERT INTO test VALUES (NULL), (42), ('String'), ([1, 2, 3]);") + parameterize_query!(ctx, "INSERT INTO test VALUES (NULL), (42), ('String'), ([1, 2, 3]);") - assert Ch.query!(conn, "SELECT v::Dynamic AS d, dynamicType(d) FROM test;").rows == [ + assert parameterize_query!(ctx, "SELECT v::Dynamic AS d, dynamicType(d) FROM test;").rows == [ [nil, "None"], [42, "UInt64"], ["String", "String"], @@ -389,12 +402,19 @@ defmodule Ch.DynamicTest do end # https://clickhouse.com/docs/sql-reference/data-types/dynamic#converting-a-dynamicmax_typesn-column-to-another-dynamicmax_typesk - test "converting a Dynamic(max_types=N) column to another Dynamic(max_types=K)", %{conn: conn} do - Ch.query!(conn, "CREATE TABLE test (d Dynamic(max_types=4)) ENGINE = Memory;") + test "converting a Dynamic(max_types=N) column to another Dynamic(max_types=K)", ctx do + parameterize_query!(ctx, "CREATE TABLE test (d Dynamic(max_types=4)) ENGINE = Memory;") on_exit(fn -> Ch.Test.query("DROP TABLE test") end) - Ch.query!(conn, "INSERT INTO test VALUES (NULL), (42), (43), ('42.42'), (true), ([1, 2, 3]);") - assert Ch.query!(conn, "SELECT d::Dynamic(max_types=5) as d2, dynamicType(d2) FROM test;").rows == + parameterize_query!( + ctx, + "INSERT INTO test VALUES (NULL), (42), (43), ('42.42'), (true), ([1, 2, 3]);" + ) + + assert parameterize_query!( + ctx, + "SELECT d::Dynamic(max_types=5) as d2, dynamicType(d2) FROM test;" + ).rows == [ [nil, "None"], [42, "Int64"], @@ -404,8 +424,8 @@ defmodule Ch.DynamicTest do [[1, 2, 3], "Array(Int64)"] ] - assert Ch.query!( - conn, + assert parameterize_query!( + ctx, "SELECT d, dynamicType(d), d::Dynamic(max_types=2) as d2, dynamicType(d2), isDynamicElementInSharedData(d2) FROM test;" ).rows == [ [nil, "None", nil, "None", false], diff --git a/test/support/test.ex b/test/support/test.ex index 1a5ac00..301e9ab 100644 --- a/test/support/test.ex +++ b/test/support/test.ex @@ -15,6 +15,33 @@ defmodule Ch.Test do Task.await(task) end + # helper for ExUnit.Case :parameterize + def parameterize_query_options(ctx, options \\ []) do + if default_options = ctx[:query_options] do + Keyword.merge(default_options, options) + else + options + end + end + + def parameterize_query(ctx, sql, params \\ [], options \\ []) do + Ch.query( + ctx.conn, + sql, + params, + parameterize_query_options(ctx, options) + ) + end + + def parameterize_query!(ctx, sql, params \\ [], options \\ []) do + Ch.query!( + ctx.conn, + sql, + params, + parameterize_query_options(ctx, options) + ) + end + # TODO packet: :http? def intercept_packets(socket, buffer \\ <<>>) do receive do From 813e4bc22d88a35a662e7b73aeecb97f62ec7c02 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Sat, 3 Jan 2026 03:08:35 +0300 Subject: [PATCH 34/34] continue --- test/ch/aggregation_test.exs | 2 +- test/ch/faults_test.exs | 59 +++--- test/ch/headers_test.exs | 53 ++++-- test/ch/http_test.exs | 34 +++- test/ch/json_test.exs | 208 ++++++++++++++------- test/ch/query_string_test.exs | 21 ++- test/ch/query_test.exs | 336 ++++++++++++++++++++++------------ test/ch/settings_test.exs | 23 ++- test/ch/stream_test.exs | 35 ++-- test/ch/variant_test.exs | 57 +++--- 10 files changed, 556 insertions(+), 272 deletions(-) diff --git a/test/ch/aggregation_test.exs b/test/ch/aggregation_test.exs index 37bd8a3..651622c 100644 --- a/test/ch/aggregation_test.exs +++ b/test/ch/aggregation_test.exs @@ -1,5 +1,5 @@ defmodule Ch.AggregationTest do - use ExUnit.Case + use ExUnit.Case, async: true setup do conn = start_supervised!({Ch, database: Ch.Test.database()}) diff --git a/test/ch/faults_test.exs b/test/ch/faults_test.exs index 9a41f4e..dae3b7d 100644 --- a/test/ch/faults_test.exs +++ b/test/ch/faults_test.exs @@ -1,6 +1,6 @@ defmodule Ch.FaultsTest do alias Ch.Result - use ExUnit.Case + use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] import Ch.Test, only: [intercept_packets: 1] defp capture_async_log(f) do @@ -18,9 +18,13 @@ defmodule Ch.FaultsTest do {:ok, clickhouse: clickhouse, listen: listen, port: port} end + setup ctx do + {:ok, query_options: ctx[:query_options] || []} + end + describe "connect/1" do test "reconnects to eventually reachable server", ctx do - %{listen: listen, port: port, clickhouse: clickhouse} = ctx + %{listen: listen, port: port, clickhouse: clickhouse, query_options: query_options} = ctx # make the server unreachable :ok = :gen_tcp.close(listen) @@ -31,7 +35,7 @@ defmodule Ch.FaultsTest do log = capture_async_log(fn -> assert {:error, %DBConnection.ConnectionError{reason: :queue_timeout}} = - Ch.query(conn, "select 1 + 1") + Ch.query(conn, "select 1 + 1", [], query_options) # make the server reachable {:ok, listen} = :gen_tcp.listen(port, @socket_opts) @@ -42,7 +46,9 @@ defmodule Ch.FaultsTest do :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) spawn_link(fn -> - assert {:ok, %{num_rows: 1, rows: [[2]]}} = Ch.query(conn, "select 1 + 1") + assert {:ok, %{num_rows: 1, rows: [[2]]}} = + Ch.query(conn, "select 1 + 1", [], query_options) + send(test, :done) end) @@ -252,7 +258,12 @@ defmodule Ch.FaultsTest do end describe "query" do - test "reconnects after timeout", %{port: port, listen: listen, clickhouse: clickhouse} do + test "reconnects after timeout", %{ + port: port, + listen: listen, + clickhouse: clickhouse, + query_options: query_options + } do test = self() log = @@ -268,7 +279,7 @@ defmodule Ch.FaultsTest do spawn_link(fn -> assert {:error, %Mint.TransportError{reason: :timeout}} = - Ch.query(conn, "select 1 + 1") + Ch.query(conn, "select 1 + 1", [], query_options) end) # failed select 1 + 1 @@ -283,7 +294,9 @@ defmodule Ch.FaultsTest do :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) spawn_link(fn -> - assert {:ok, %{num_rows: 1, rows: [[2]]}} = Ch.query(conn, "select 1 + 1") + assert {:ok, %{num_rows: 1, rows: [[2]]}} = + Ch.query(conn, "select 1 + 1", [], query_options) + send(test, :done) end) @@ -298,7 +311,7 @@ defmodule Ch.FaultsTest do end test "reconnects after closed on response", ctx do - %{port: port, listen: listen, clickhouse: clickhouse} = ctx + %{port: port, listen: listen, clickhouse: clickhouse, query_options: query_options} = ctx test = self() log = @@ -314,7 +327,7 @@ defmodule Ch.FaultsTest do spawn_link(fn -> assert {:error, %Mint.TransportError{reason: :closed}} = - Ch.query(conn, "select 1 + 1") + Ch.query(conn, "select 1 + 1", [], query_options) end) # failed select 1 + 1 @@ -330,7 +343,9 @@ defmodule Ch.FaultsTest do :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) spawn_link(fn -> - assert {:ok, %{num_rows: 1, rows: [[2]]}} = Ch.query(conn, "select 1 + 1") + assert {:ok, %{num_rows: 1, rows: [[2]]}} = + Ch.query(conn, "select 1 + 1", [], query_options) + send(test, :done) end) @@ -344,7 +359,7 @@ defmodule Ch.FaultsTest do end test "reconnects after Connection: close response from server", ctx do - %{port: port, listen: listen, clickhouse: clickhouse} = ctx + %{port: port, listen: listen, clickhouse: clickhouse, query_options: query_options} = ctx test = self() log = @@ -359,7 +374,9 @@ defmodule Ch.FaultsTest do :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) spawn_link(fn -> - assert {:ok, %{num_rows: 1, rows: [[2]]}} = Ch.query(conn, "select 1 + 1") + assert {:ok, %{num_rows: 1, rows: [[2]]}} = + Ch.query(conn, "select 1 + 1", [], query_options) + send(test, :done) end) @@ -386,7 +403,7 @@ defmodule Ch.FaultsTest do spawn_link(fn -> assert {:ok, %{num_rows: 1, rows: [[2]]}} = - Ch.query(conn, "select 1 + 1") + Ch.query(conn, "select 1 + 1", [], query_options) send(test, :done) end) @@ -405,7 +422,7 @@ defmodule Ch.FaultsTest do # TODO non-chunked request test "reconnects after closed before streaming request", ctx do - %{port: port, listen: listen, clickhouse: clickhouse} = ctx + %{port: port, listen: listen, clickhouse: clickhouse, query_options: query_options} = ctx test = self() rows = [[1, 2], [3, 4]] @@ -431,7 +448,7 @@ defmodule Ch.FaultsTest do conn, "insert into unknown_table(a,b) format RowBinary", stream, - encode: false + Keyword.merge(query_options, encode: false) ) end) @@ -448,7 +465,7 @@ defmodule Ch.FaultsTest do conn, "insert into unknown_table(a,b) format RowBinary", stream, - encode: false + Keyword.merge(query_options, encode: false) ) assert message =~ ~r/UNKNOWN_TABLE/ @@ -467,7 +484,7 @@ defmodule Ch.FaultsTest do end test "reconnects after closed while streaming request", ctx do - %{port: port, listen: listen, clickhouse: clickhouse} = ctx + %{port: port, listen: listen, clickhouse: clickhouse, query_options: query_options} = ctx test = self() rows = [[1, 2], [3, 4]] @@ -490,7 +507,7 @@ defmodule Ch.FaultsTest do conn, "insert into unknown_table(a,b) format RowBinary", stream, - encode: false + Keyword.merge(query_options, encode: false) ) end) @@ -511,7 +528,7 @@ defmodule Ch.FaultsTest do conn, "insert into unknown_table(a,b) format RowBinary", stream, - encode: false + Keyword.merge(query_options, encode: false) ) assert message =~ ~r/UNKNOWN_TABLE/ @@ -530,7 +547,7 @@ defmodule Ch.FaultsTest do end test "warns on different server name", ctx do - %{port: port, listen: listen, clickhouse: clickhouse} = ctx + %{port: port, listen: listen, clickhouse: clickhouse, query_options: query_options} = ctx test = self() header = "X-ClickHouse-Server-Display-Name" @@ -549,7 +566,7 @@ defmodule Ch.FaultsTest do :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) spawn_link(fn -> - assert {:ok, %Result{rows: [[1]]}} = Ch.query(conn, "select 1") + assert {:ok, %Result{rows: [[1]]}} = Ch.query(conn, "select 1", [], query_options) send(test, :done) end) diff --git a/test/ch/headers_test.exs b/test/ch/headers_test.exs index 458e3a3..2d3da43 100644 --- a/test/ch/headers_test.exs +++ b/test/ch/headers_test.exs @@ -1,17 +1,28 @@ defmodule Ch.HeadersTest do - use ExUnit.Case, async: true + use ExUnit.Case, + async: true, + parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] setup do {:ok, conn} = Ch.start_link() {:ok, conn: conn} end - test "can request gzipped response through headers", %{conn: conn} do + setup ctx do + {:ok, query_options: ctx[:query_options] || []} + end + + test "can request gzipped response through headers", %{conn: conn, query_options: query_options} do assert {:ok, %{rows: data, data: data, headers: headers}} = - Ch.query(conn, "select number from system.numbers limit 100", [], - decode: false, - settings: [enable_http_compression: 1], - headers: [{"accept-encoding", "gzip"}] + Ch.query( + conn, + "select number from system.numbers limit 100", + [], + Keyword.merge(query_options, + decode: false, + settings: [enable_http_compression: 1], + headers: [{"accept-encoding", "gzip"}] + ) ) assert :proplists.get_value("content-type", headers) == "application/octet-stream" @@ -22,12 +33,17 @@ defmodule Ch.HeadersTest do assert <<0x1F, 0x8B, _rest::bytes>> = IO.iodata_to_binary(data) end - test "can request lz4 response through headers", %{conn: conn} do + test "can request lz4 response through headers", %{conn: conn, query_options: query_options} do assert {:ok, %{rows: data, data: data, headers: headers}} = - Ch.query(conn, "select number from system.numbers limit 100", [], - decode: false, - settings: [enable_http_compression: 1], - headers: [{"accept-encoding", "lz4"}] + Ch.query( + conn, + "select number from system.numbers limit 100", + [], + Keyword.merge(query_options, + decode: false, + settings: [enable_http_compression: 1], + headers: [{"accept-encoding", "lz4"}] + ) ) assert :proplists.get_value("content-type", headers) == "application/octet-stream" @@ -38,12 +54,17 @@ defmodule Ch.HeadersTest do assert <<0x04, 0x22, 0x4D, 0x18, _rest::bytes>> = IO.iodata_to_binary(data) end - test "can request zstd response through headers", %{conn: conn} do + test "can request zstd response through headers", %{conn: conn, query_options: query_options} do assert {:ok, %{rows: data, data: data, headers: headers}} = - Ch.query(conn, "select number from system.numbers limit 100", [], - decode: false, - settings: [enable_http_compression: 1], - headers: [{"accept-encoding", "zstd"}] + Ch.query( + conn, + "select number from system.numbers limit 100", + [], + Keyword.merge(query_options, + decode: false, + settings: [enable_http_compression: 1], + headers: [{"accept-encoding", "zstd"}] + ) ) assert :proplists.get_value("content-type", headers) == "application/octet-stream" diff --git a/test/ch/http_test.exs b/test/ch/http_test.exs index f9517ba..c0f802b 100644 --- a/test/ch/http_test.exs +++ b/test/ch/http_test.exs @@ -1,37 +1,53 @@ defmodule Ch.HTTPTest do - use ExUnit.Case, async: true + use ExUnit.Case, + async: true, + parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] @moduletag :slow + setup ctx do + {:ok, query_options: ctx[:query_options] || []} + end + describe "user-agent" do setup do {:ok, ch: start_supervised!(Ch)} end - test "sets user-agent to ch/ by default", %{ch: ch} do - %Ch.Result{rows: [[123]], headers: resp_header} = Ch.query!(ch, "select 123") + test "sets user-agent to ch/ by default", %{ch: ch, query_options: query_options} do + %Ch.Result{rows: [[123]], headers: resp_header} = + Ch.query!(ch, "select 123", [], query_options) + {"x-clickhouse-query-id", query_id} = List.keyfind!(resp_header, "x-clickhouse-query-id", 0) - assert query_http_user_agent(ch, query_id) == "ch/" <> Mix.Project.config()[:version] + + assert query_http_user_agent(ch, query_id, query_options) == + "ch/" <> Mix.Project.config()[:version] end - test "uses the provided user-agent", %{ch: ch} do + test "uses the provided user-agent", %{ch: ch, query_options: query_options} do req_headers = [{"user-agent", "plausible/0.1.0"}] %Ch.Result{rows: [[123]], headers: resp_header} = - Ch.query!(ch, "select 123", _params = [], headers: req_headers) + Ch.query!( + ch, + "select 123", + _params = [], + Keyword.merge(query_options, headers: req_headers) + ) {"x-clickhouse-query-id", query_id} = List.keyfind!(resp_header, "x-clickhouse-query-id", 0) - assert query_http_user_agent(ch, query_id) == "plausible/0.1.0" + assert query_http_user_agent(ch, query_id, query_options) == "plausible/0.1.0" end end - defp query_http_user_agent(ch, query_id) do + defp query_http_user_agent(ch, query_id, query_options) do retry(fn -> %Ch.Result{rows: [[user_agent]]} = Ch.query!( ch, "select http_user_agent from system.query_log where query_id = {query_id:String} limit 1", - %{"query_id" => query_id} + %{"query_id" => query_id}, + query_options ) user_agent diff --git a/test/ch/json_test.exs b/test/ch/json_test.exs index 152c2b4..b41a82d 100644 --- a/test/ch/json_test.exs +++ b/test/ch/json_test.exs @@ -1,16 +1,20 @@ defmodule Ch.JSONTest do - use ExUnit.Case + use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] @moduletag :json + setup ctx do + {:ok, query_options: ctx[:query_options] || []} + end + setup do on_exit(fn -> Ch.Test.query("DROP TABLE IF EXISTS json_test") end) {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} end - test "simple json", %{conn: conn} do + test "simple json", %{conn: conn, query_options: query_options} do select = fn literal -> - [[value]] = Ch.query!(conn, "select '#{literal}'::json").rows + [[value]] = Ch.query!(conn, "select '#{literal}'::json", [], query_options).rows value end @@ -51,19 +55,31 @@ defmodule Ch.JSONTest do end # https://clickhouse.com/docs/sql-reference/data-types/newjson#using-json-in-a-table-column-definition - test "basic", %{conn: conn} do - Ch.query!(conn, "CREATE TABLE json_test (json JSON, id UInt8) ENGINE = Memory") + test "basic", %{conn: conn, query_options: query_options} do + Ch.query!( + conn, + "CREATE TABLE json_test (json JSON, id UInt8) ENGINE = Memory", + [], + query_options + ) - Ch.query!(conn, """ - INSERT INTO json_test VALUES - ('{"a" : {"b" : 42}, "c" : [1, 2, 3]}', 0), - ('{"f" : "Hello, World!"}', 1), - ('{"a" : {"b" : 43, "e" : 10}, "c" : [4, 5, 6]}', 2) - """) + Ch.query!( + conn, + """ + INSERT INTO json_test VALUES + ('{"a" : {"b" : 42}, "c" : [1, 2, 3]}', 0), + ('{"f" : "Hello, World!"}', 1), + ('{"a" : {"b" : 43, "e" : 10}, "c" : [4, 5, 6]}', 2) + """, + [], + query_options + ) assert Ch.query!( conn, - "SELECT json FROM json_test ORDER BY id" + "SELECT json FROM json_test ORDER BY id", + [], + query_options ).rows == [ [%{"a" => %{"b" => 42}, "c" => [1, 2, 3]}], [%{"f" => "Hello, World!"}], @@ -74,17 +90,24 @@ defmodule Ch.JSONTest do conn, "INSERT INTO json_test(json, id) FORMAT RowBinary", [[%{"a" => %{"b" => 999}, "some other" => "json value", "from" => "rowbinary"}, 3]], - types: ["JSON", "UInt8"] + Keyword.merge(query_options, types: ["JSON", "UInt8"]) ) assert Ch.query!( conn, - "SELECT json FROM json_test where json.from = 'rowbinary'" + "SELECT json FROM json_test where json.from = 'rowbinary'", + [], + query_options ).rows == [ [%{"from" => "rowbinary", "some other" => "json value", "a" => %{"b" => 999}}] ] - assert Ch.query!(conn, "select json.a.b, json.a.g, json.c, json.d from json_test order by id").rows == + assert Ch.query!( + conn, + "select json.a.b, json.a.g, json.c, json.d from json_test order by id", + [], + query_options + ).rows == [ [42, nil, [1, 2, 3], nil], [nil, nil, nil, nil], @@ -94,19 +117,31 @@ defmodule Ch.JSONTest do end # https://clickhouse.com/docs/sql-reference/data-types/newjson#using-json-in-a-table-column-definition - test "with skip (i.e. extra type options)", %{conn: conn} do - Ch.query!(conn, "CREATE TABLE json_test (json JSON(a.b UInt32, SKIP a.e)) ENGINE = Memory;") + test "with skip (i.e. extra type options)", %{conn: conn, query_options: query_options} do + Ch.query!( + conn, + "CREATE TABLE json_test (json JSON(a.b UInt32, SKIP a.e)) ENGINE = Memory;", + [], + query_options + ) - Ch.query!(conn, """ - INSERT INTO json_test VALUES - ('{"a" : {"b" : 42}, "c" : [1, 2, 3]}'), - ('{"f" : "Hello, World!"}'), - ('{"a" : {"b" : 43, "e" : 10}, "c" : [4, 5, 6]}'); - """) + Ch.query!( + conn, + """ + INSERT INTO json_test VALUES + ('{"a" : {"b" : 42}, "c" : [1, 2, 3]}'), + ('{"f" : "Hello, World!"}'), + ('{"a" : {"b" : 43, "e" : 10}, "c" : [4, 5, 6]}'); + """, + [], + query_options + ) assert Ch.query!( conn, - "SELECT json FROM json_test" + "SELECT json FROM json_test", + [], + query_options ).rows == [ [%{"a" => %{"b" => 42}, "c" => [1, 2, 3]}], [%{"a" => %{"b" => 0}, "f" => "Hello, World!"}], @@ -115,40 +150,60 @@ defmodule Ch.JSONTest do end # https://clickhouse.com/docs/sql-reference/data-types/newjson#reading-json-paths-as-sub-columns - test "reading json paths as subcolumns", %{conn: conn} do - Ch.query!(conn, "CREATE TABLE json_test (json JSON(a.b UInt32, SKIP a.e)) ENGINE = Memory") + test "reading json paths as subcolumns", %{conn: conn, query_options: query_options} do + Ch.query!( + conn, + "CREATE TABLE json_test (json JSON(a.b UInt32, SKIP a.e)) ENGINE = Memory", + [], + query_options + ) - Ch.query!(conn, """ - INSERT INTO json_test VALUES - ('{"a" : {"b" : 42, "g" : 42.42}, "c" : [1, 2, 3], "d" : "2020-01-01"}'), - ('{"f" : "Hello, World!", "d" : "2020-01-02"}'), - ('{"a" : {"b" : 43, "e" : 10, "g" : 43.43}, "c" : [4, 5, 6]}'); - """) + Ch.query!( + conn, + """ + INSERT INTO json_test VALUES + ('{"a" : {"b" : 42, "g" : 42.42}, "c" : [1, 2, 3], "d" : "2020-01-01"}'), + ('{"f" : "Hello, World!", "d" : "2020-01-02"}'), + ('{"a" : {"b" : 43, "e" : 10, "g" : 43.43}, "c" : [4, 5, 6]}'); + """, + [], + query_options + ) assert Ch.query!( conn, - "SELECT json FROM json_test" + "SELECT json FROM json_test", + [], + query_options ).rows == [ [%{"a" => %{"b" => 42, "g" => 42.42}, "c" => [1, 2, 3], "d" => "2020-01-01"}], [%{"a" => %{"b" => 0}, "d" => "2020-01-02", "f" => "Hello, World!"}], [%{"a" => %{"b" => 43, "g" => 43.43}, "c" => [4, 5, 6]}] ] - assert Ch.query!(conn, "SELECT json.a.b, json.a.g, json.c, json.d FROM json_test").rows == [ + assert Ch.query!( + conn, + "SELECT json.a.b, json.a.g, json.c, json.d FROM json_test", + [], + query_options + ).rows == [ [42, 42.42, [1, 2, 3], ~D[2020-01-01]], [0, nil, nil, ~D[2020-01-02]], [43, 43.43, [4, 5, 6], nil] ] - assert Ch.query!(conn, "SELECT json.non.existing.path FROM json_test").rows == [ - [nil], - [nil], - [nil] - ] + assert Ch.query!(conn, "SELECT json.non.existing.path FROM json_test", [], query_options).rows == + [ + [nil], + [nil], + [nil] + ] assert Ch.query!( conn, - "SELECT toTypeName(json.a.b), toTypeName(json.a.g), toTypeName(json.c), toTypeName(json.d) FROM json_test;" + "SELECT toTypeName(json.a.b), toTypeName(json.a.g), toTypeName(json.c), toTypeName(json.d) FROM json_test;", + [], + query_options ).rows == [ ["UInt32", "Dynamic", "Dynamic", "Dynamic"], ["UInt32", "Dynamic", "Dynamic", "Dynamic"], @@ -164,7 +219,9 @@ defmodule Ch.JSONTest do json.d.:Date, dynamicType(json.d) FROM json_test - """ + """, + [], + query_options ).rows == [ [42.42, "Float64", ~D[2020-01-01], "Date"], [nil, "None", ~D[2020-01-02], "Date"], @@ -176,7 +233,9 @@ defmodule Ch.JSONTest do """ SELECT json.a.g::UInt64 AS uint FROM json_test; - """ + """, + [], + query_options ).rows == [ [42], [0], @@ -184,22 +243,27 @@ defmodule Ch.JSONTest do ] assert_raise Ch.Error, ~r/Conversion between numeric types and UUID is not supported/, fn -> - Ch.query!(conn, "SELECT json.a.g::UUID AS float FROM json_test;") + Ch.query!(conn, "SELECT json.a.g::UUID AS float FROM json_test;", [], query_options) end end # https://clickhouse.com/docs/sql-reference/data-types/newjson#reading-json-sub-objects-as-sub-columns - test "reading json subobjects as subcolumns", %{conn: conn} do - Ch.query!(conn, "CREATE TABLE json_test (json JSON) ENGINE = Memory;") + test "reading json subobjects as subcolumns", %{conn: conn, query_options: query_options} do + Ch.query!(conn, "CREATE TABLE json_test (json JSON) ENGINE = Memory;", [], query_options) - Ch.query!(conn, """ - INSERT INTO json_test VALUES - ('{"a" : {"b" : {"c" : 42, "g" : 42.42}}, "c" : [1, 2, 3], "d" : {"e" : {"f" : {"g" : "Hello, World", "h" : [1, 2, 3]}}}}'), - ('{"f" : "Hello, World!", "d" : {"e" : {"f" : {"h" : [4, 5, 6]}}}}'), - ('{"a" : {"b" : {"c" : 43, "e" : 10, "g" : 43.43}}, "c" : [4, 5, 6]}'); - """) + Ch.query!( + conn, + """ + INSERT INTO json_test VALUES + ('{"a" : {"b" : {"c" : 42, "g" : 42.42}}, "c" : [1, 2, 3], "d" : {"e" : {"f" : {"g" : "Hello, World", "h" : [1, 2, 3]}}}}'), + ('{"f" : "Hello, World!", "d" : {"e" : {"f" : {"h" : [4, 5, 6]}}}}'), + ('{"a" : {"b" : {"c" : 43, "e" : 10, "g" : 43.43}}, "c" : [4, 5, 6]}'); + """, + [], + query_options + ) - assert Ch.query!(conn, "SELECT json FROM json_test;").rows == [ + assert Ch.query!(conn, "SELECT json FROM json_test;", [], query_options).rows == [ [ %{ "a" => %{"b" => %{"c" => 42, "g" => 42.42}}, @@ -216,26 +280,32 @@ defmodule Ch.JSONTest do ] ] - assert Ch.query!(conn, "SELECT json.^a.b, json.^d.e.f FROM json_test;").rows == [ - [%{"c" => 42, "g" => 42.42}, %{"g" => "Hello, World", "h" => [1, 2, 3]}], - [%{}, %{"h" => [4, 5, 6]}], - [%{"c" => 43, "e" => 10, "g" => 43.43}, %{}] - ] + assert Ch.query!(conn, "SELECT json.^a.b, json.^d.e.f FROM json_test;", [], query_options).rows == + [ + [%{"c" => 42, "g" => 42.42}, %{"g" => "Hello, World", "h" => [1, 2, 3]}], + [%{}, %{"h" => [4, 5, 6]}], + [%{"c" => 43, "e" => 10, "g" => 43.43}, %{}] + ] end # TODO # https://clickhouse.com/docs/sql-reference/data-types/newjson#handling-arrays-of-json-objects - test "handling arrays of json objects", %{conn: conn} do - Ch.query!(conn, "CREATE TABLE json_test (json JSON) ENGINE = Memory;") + test "handling arrays of json objects", %{conn: conn, query_options: query_options} do + Ch.query!(conn, "CREATE TABLE json_test (json JSON) ENGINE = Memory;", [], query_options) - Ch.query!(conn, """ - INSERT INTO json_test VALUES - ('{"a" : {"b" : [{"c" : 42, "d" : "Hello", "f" : [[{"g" : 42.42}]], "k" : {"j" : 1000}}, {"c" : 43}, {"e" : [1, 2, 3], "d" : "My", "f" : [[{"g" : 43.43, "h" : "2020-01-01"}]], "k" : {"j" : 2000}}]}}'), - ('{"a" : {"b" : [1, 2, 3]}}'), - ('{"a" : {"b" : [{"c" : 44, "f" : [[{"h" : "2020-01-02"}]]}, {"e" : [4, 5, 6], "d" : "World", "f" : [[{"g" : 44.44}]], "k" : {"j" : 3000}}]}}'); - """) + Ch.query!( + conn, + """ + INSERT INTO json_test VALUES + ('{"a" : {"b" : [{"c" : 42, "d" : "Hello", "f" : [[{"g" : 42.42}]], "k" : {"j" : 1000}}, {"c" : 43}, {"e" : [1, 2, 3], "d" : "My", "f" : [[{"g" : 43.43, "h" : "2020-01-01"}]], "k" : {"j" : 2000}}]}}'), + ('{"a" : {"b" : [1, 2, 3]}}'), + ('{"a" : {"b" : [{"c" : 44, "f" : [[{"h" : "2020-01-02"}]]}, {"e" : [4, 5, 6], "d" : "World", "f" : [[{"g" : 44.44}]], "k" : {"j" : 3000}}]}}'); + """, + [], + query_options + ) - assert Ch.query!(conn, "SELECT json FROM json_test;").rows == [ + assert Ch.query!(conn, "SELECT json FROM json_test;", [], query_options).rows == [ [ %{ "a" => %{ @@ -277,13 +347,15 @@ defmodule Ch.JSONTest do # TODO assert_raise ArgumentError, "unsupported dynamic type JSON", fn -> - Ch.query!(conn, "SELECT json.a.b, dynamicType(json.a.b) FROM json_test;") + Ch.query!(conn, "SELECT json.a.b, dynamicType(json.a.b) FROM json_test;", [], query_options) end assert_raise ArgumentError, "unsupported dynamic type JSON", fn -> Ch.query!( conn, - "SELECT json.a.b.:`Array(JSON)`.c, json.a.b.:`Array(JSON)`.f, json.a.b.:`Array(JSON)`.d FROM json_test;" + "SELECT json.a.b.:`Array(JSON)`.c, json.a.b.:`Array(JSON)`.f, json.a.b.:`Array(JSON)`.d FROM json_test;", + [], + query_options ) end end diff --git a/test/ch/query_string_test.exs b/test/ch/query_string_test.exs index 6dcd829..398ab3e 100644 --- a/test/ch/query_string_test.exs +++ b/test/ch/query_string_test.exs @@ -1,5 +1,11 @@ defmodule Ch.QueryStringTest do - use ExUnit.Case, async: true + use ExUnit.Case, + async: true, + parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] + + setup ctx do + {:ok, query_options: ctx[:query_options] || []} + end setup do {:ok, conn: start_supervised!(Ch)} @@ -8,16 +14,21 @@ defmodule Ch.QueryStringTest do # For more info see # https://clickhouse.com/docs/en/interfaces/http#tabs-in-url-parameters # "escaped" format is the same as https://clickhouse.com/docs/en/interfaces/formats#tabseparated-data-formatting - test "binaries are escaped properly", %{conn: conn} do + test "binaries are escaped properly", %{conn: conn, query_options: query_options} do for s <- ["\t", "\n", "\\", "'", "\b", "\f", "\r", "\0"] do - assert Ch.query!(conn, "select {s:String}", %{"s" => s}).rows == [[s]] + assert Ch.query!(conn, "select {s:String}", %{"s" => s}, query_options).rows == [[s]] end # example from https://clickhouse.com/docs/en/interfaces/http#tabs-in-url-parameters - assert Ch.query!(conn, "select splitByChar('\t', 'abc\t123')").rows == + assert Ch.query!(conn, "select splitByChar('\t', 'abc\t123')", [], query_options).rows == [[["abc", "123"]]] - assert Ch.query!(conn, "select splitByChar('\t', {arg1:String})", %{"arg1" => "abc\t123"}).rows == + assert Ch.query!( + conn, + "select splitByChar('\t', {arg1:String})", + %{"arg1" => "abc\t123"}, + query_options + ).rows == [[["abc", "123"]]] end end diff --git a/test/ch/query_test.exs b/test/ch/query_test.exs index 56fef45..056cb0c 100644 --- a/test/ch/query_test.exs +++ b/test/ch/query_test.exs @@ -1,7 +1,14 @@ defmodule Ch.QueryTest do - use ExUnit.Case, async: true + use ExUnit.Case, + async: true, + parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] + alias Ch.Query + setup ctx do + {:ok, query_options: ctx[:query_options] || []} + end + test "to_string" do query = Query.build(["select ", 1 + ?0, ?+, 2 + ?0]) assert to_string(query) == "select 1+2" @@ -49,51 +56,59 @@ defmodule Ch.QueryTest do {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} end - test "iodata", %{conn: conn} do - assert [[123]] = Ch.query!(conn, ["S", ?E, ["LEC" | "T"], " ", ~c"123"]).rows + test "iodata", %{conn: conn, query_options: query_options} do + assert [[123]] = + Ch.query!(conn, ["S", ?E, ["LEC" | "T"], " ", ~c"123"], [], query_options).rows end - test "decode basic types", %{conn: conn} do - assert [[nil]] = Ch.query!(conn, "SELECT NULL").rows - assert [[true, false]] = Ch.query!(conn, "SELECT true, false").rows - assert [["e"]] = Ch.query!(conn, "SELECT 'e'::char").rows - assert [["ẽ"]] = Ch.query!(conn, "SELECT 'ẽ'::char").rows - assert [[42]] = Ch.query!(conn, "SELECT 42").rows - assert [[42.0]] = Ch.query!(conn, "SELECT 42::float").rows - assert [[42.0]] = Ch.query!(conn, "SELECT 42.0").rows + test "decode basic types", %{conn: conn, query_options: query_options} do + assert [[nil]] = Ch.query!(conn, "SELECT NULL", [], query_options).rows + assert [[true, false]] = Ch.query!(conn, "SELECT true, false", [], query_options).rows + assert [["e"]] = Ch.query!(conn, "SELECT 'e'::char", [], query_options).rows + assert [["ẽ"]] = Ch.query!(conn, "SELECT 'ẽ'::char", [], query_options).rows + assert [[42]] = Ch.query!(conn, "SELECT 42", [], query_options).rows + assert [[42.0]] = Ch.query!(conn, "SELECT 42::float", [], query_options).rows + assert [[42.0]] = Ch.query!(conn, "SELECT 42.0", [], query_options).rows # TODO [[:NaN]] ? - assert [[nil]] = Ch.query!(conn, "SELECT 'NaN'::float").rows + assert [[nil]] = Ch.query!(conn, "SELECT 'NaN'::float", [], query_options).rows # TODO [[:int]] ? - assert [[nil]] = Ch.query!(conn, "SELECT 'inf'::float").rows + assert [[nil]] = Ch.query!(conn, "SELECT 'inf'::float", [], query_options).rows # TODO [[:"-inf"]] ? - assert [[nil]] = Ch.query!(conn, "SELECT '-inf'::float").rows - assert [["ẽric"]] = Ch.query!(conn, "SELECT 'ẽric'").rows - assert [["ẽric"]] = Ch.query!(conn, "SELECT 'ẽric'::varchar").rows + assert [[nil]] = Ch.query!(conn, "SELECT '-inf'::float", [], query_options).rows + assert [["ẽric"]] = Ch.query!(conn, "SELECT 'ẽric'", [], query_options).rows + assert [["ẽric"]] = Ch.query!(conn, "SELECT 'ẽric'::varchar", [], query_options).rows # TODO # assert [[<<1, 2, 3>>]] = Ch.query!(conn, "SELECT '\\001\\002\\003'::bytea").rows end - test "decode numeric", %{conn: conn} do - assert [[Decimal.new("42.0000000000")]] == Ch.query!(conn, "SELECT 42::numeric(10,10)").rows + test "decode numeric", %{conn: conn, query_options: query_options} do + assert [[Decimal.new("42.0000000000")]] == + Ch.query!(conn, "SELECT 42::numeric(10,10)", [], query_options).rows end @tag skip: true - test "decode json/jsonb", %{conn: conn} do + test "decode json/jsonb", %{conn: conn, query_options: query_options} do assert_raise ArgumentError, "Object('json') type is not supported", fn -> - assert [[%{"foo" => 42}]] == Ch.query!(conn, "SELECT '{\"foo\": 42}'::json").rows + assert [[%{"foo" => 42}]] == + Ch.query!(conn, "SELECT '{\"foo\": 42}'::json", [], query_options).rows end end - test "decode uuid", %{conn: conn} do + test "decode uuid", %{conn: conn, query_options: query_options} do uuid = <<160, 238, 188, 153, 156, 11, 78, 248, 187, 109, 107, 185, 189, 56, 10, 17>> assert [[^uuid]] = - Ch.query!(conn, "SELECT 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'::UUID").rows + Ch.query!( + conn, + "SELECT 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'::UUID", + [], + query_options + ).rows end # https://clickhouse.com/docs/sql-reference/data-types/time @tag :time - test "decode time", %{conn: conn} do + test "decode time", %{conn: conn, query_options: query_options} do settings = [enable_time_time64_type: 1] times = [ @@ -105,10 +120,20 @@ defmodule Ch.QueryTest do for time <- times do %{value: value, expected: expected} = time - assert Ch.query!(conn, "SELECT '#{value}'::time", [], settings: settings).rows == + assert Ch.query!( + conn, + "SELECT '#{value}'::time", + [], + Keyword.merge(query_options, settings: settings) + ).rows == [[expected]] - assert Ch.query!(conn, "SELECT {time:Time}", %{"time" => expected}, settings: settings).rows == + assert Ch.query!( + conn, + "SELECT {time:Time}", + %{"time" => expected}, + Keyword.merge(query_options, settings: settings) + ).rows == [[expected]] end @@ -118,29 +143,53 @@ defmodule Ch.QueryTest do assert_raise ArgumentError, "ClickHouse Time value -1.0 (seconds) is out of Elixir's Time range (00:00:00.000000 - 23:59:59.999999)", - fn -> Ch.query!(conn, "SELECT '-00:00:01'::time", [], settings: settings) end + fn -> + Ch.query!( + conn, + "SELECT '-00:00:01'::time", + [], + Keyword.merge(query_options, settings: settings) + ) + end assert_raise ArgumentError, "ClickHouse Time value 3599999.0 (seconds) is out of Elixir's Time range (00:00:00.000000 - 23:59:59.999999)", - fn -> Ch.query!(conn, "SELECT '999:59:59'::time", [], settings: settings) end + fn -> + Ch.query!( + conn, + "SELECT '999:59:59'::time", + [], + Keyword.merge(query_options, settings: settings) + ) + end assert_raise ArgumentError, "ClickHouse Time value -3599999.0 (seconds) is out of Elixir's Time range (00:00:00.000000 - 23:59:59.999999)", - fn -> Ch.query!(conn, "SELECT '-999:59:59'::time", [], settings: settings) end + fn -> + Ch.query!( + conn, + "SELECT '-999:59:59'::time", + [], + Keyword.merge(query_options, settings: settings) + ) + end # ** (Ch.Error) Code: 457. DB::Exception: Value 12:34:56.123456 cannot be parsed as Time for query parameter 'time' # because it isn't parsed completely: only 8 of 15 bytes was parsed: 12:34:56. (BAD_QUERY_PARAMETER) # (version 25.6.3.116 (official build)) assert_raise Ch.Error, ~r/only 8 of 15 bytes was parsed/, fn -> - Ch.query!(conn, "SELECT {time:Time}", %{"time" => ~T[12:34:56.123456]}, - settings: settings + Ch.query!( + conn, + "SELECT {time:Time}", + %{"time" => ~T[12:34:56.123456]}, + Keyword.merge(query_options, settings: settings) ) end end # https://clickhouse.com/docs/sql-reference/data-types/time64 @tag :time - test "decode time64", %{conn: conn} do + test "decode time64", %{conn: conn, query_options: query_options} do settings = [enable_time_time64_type: 1] times = [ @@ -175,14 +224,19 @@ defmodule Ch.QueryTest do for time <- times do %{value: value, precision: precision, expected: expected} = time - assert Ch.query!(conn, "SELECT '#{value}'::time64(#{precision})", [], settings: settings).rows == + assert Ch.query!( + conn, + "SELECT '#{value}'::time64(#{precision})", + [], + Keyword.merge(query_options, settings: settings) + ).rows == [[expected]] assert Ch.query!( conn, "SELECT {time:time64(#{precision})}", %{"time" => expected}, - settings: settings + Keyword.merge(query_options, settings: settings) ).rows == [[expected]] end @@ -194,79 +248,110 @@ defmodule Ch.QueryTest do assert_raise ArgumentError, "ClickHouse Time value -1.0 (seconds) is out of Elixir's Time range (00:00:00.000000 - 23:59:59.999999)", fn -> - Ch.query!(conn, "SELECT '-00:00:01.000'::time64(6)", [], settings: settings) + Ch.query!( + conn, + "SELECT '-00:00:01.000'::time64(6)", + [], + Keyword.merge(query_options, settings: settings) + ) end assert_raise ArgumentError, "ClickHouse Time value 3599999.999999 (seconds) is out of Elixir's Time range (00:00:00.000000 - 23:59:59.999999)", fn -> - Ch.query!(conn, "SELECT '999:59:59.999999999'::time64(6)", [], - settings: settings + Ch.query!( + conn, + "SELECT '999:59:59.999999999'::time64(6)", + [], + Keyword.merge(query_options, settings: settings) ) end assert_raise ArgumentError, "ClickHouse Time value -3599999.999999 (seconds) is out of Elixir's Time range (00:00:00.000000 - 23:59:59.999999)", fn -> - Ch.query!(conn, "SELECT '-999:59:59.999999999'::time64(6)", [], - settings: settings + Ch.query!( + conn, + "SELECT '-999:59:59.999999999'::time64(6)", + [], + Keyword.merge(query_options, settings: settings) ) end end - test "decode arrays", %{conn: conn} do - assert [[[]]] = Ch.query!(conn, "SELECT []").rows - assert [[[1]]] = Ch.query!(conn, "SELECT [1]").rows - assert [[[1, 2]]] = Ch.query!(conn, "SELECT [1,2]").rows - assert [[[[0], [1]]]] = Ch.query!(conn, "SELECT [[0],[1]]").rows - assert [[[[0]]]] = Ch.query!(conn, "SELECT [[0]]").rows + test "decode arrays", %{conn: conn, query_options: query_options} do + assert [[[]]] = Ch.query!(conn, "SELECT []", [], query_options).rows + assert [[[1]]] = Ch.query!(conn, "SELECT [1]", [], query_options).rows + assert [[[1, 2]]] = Ch.query!(conn, "SELECT [1,2]", [], query_options).rows + assert [[[[0], [1]]]] = Ch.query!(conn, "SELECT [[0],[1]]", [], query_options).rows + assert [[[[0]]]] = Ch.query!(conn, "SELECT [[0]]", [], query_options).rows end - test "decode tuples", %{conn: conn} do - assert [[{"Hello", 123}]] = Ch.query!(conn, "select ('Hello', 123)").rows - assert [[{"Hello", 123}]] = Ch.query!(conn, "select ('Hello' as a, 123 as b)").rows - assert [[{"Hello", 123}]] = Ch.query!(conn, "select ('Hello' as a_, 123 as b)").rows + test "decode tuples", %{conn: conn, query_options: query_options} do + assert [[{"Hello", 123}]] = Ch.query!(conn, "select ('Hello', 123)", [], query_options).rows + + assert [[{"Hello", 123}]] = + Ch.query!(conn, "select ('Hello' as a, 123 as b)", [], query_options).rows + + assert [[{"Hello", 123}]] = + Ch.query!(conn, "select ('Hello' as a_, 123 as b)", [], query_options).rows + # TODO - # assert [[{"Hello", 123}]] = Ch.query!(conn, "select ('Hello' as a$, 123 as b)").rows + # assert [[{"Hello", 123}]] = Ch.query!(conn, "select ('Hello' as a$, 123 as b)", [], query_options).rows end - test "decode network types", %{conn: conn} do - assert [[{127, 0, 0, 1} = ipv4]] = Ch.query!(conn, "SELECT '127.0.0.1'::inet4").rows + test "decode network types", %{conn: conn, query_options: query_options} do + assert [[{127, 0, 0, 1} = ipv4]] = + Ch.query!(conn, "SELECT '127.0.0.1'::inet4", [], query_options).rows + assert :inet.ntoa(ipv4) == ~c"127.0.0.1" - assert [[{0, 0, 0, 0, 0, 0, 0, 1} = ipv6]] = Ch.query!(conn, "SELECT '::1'::inet6").rows + assert [[{0, 0, 0, 0, 0, 0, 0, 1} = ipv6]] = + Ch.query!(conn, "SELECT '::1'::inet6", [], query_options).rows + assert :inet.ntoa(ipv6) == ~c"::1" - assert [[ipv6]] = Ch.query!(conn, "SELECT '2001:44c8:129:2632:33:0:252:2'::inet6").rows + assert [[ipv6]] = + Ch.query!(conn, "SELECT '2001:44c8:129:2632:33:0:252:2'::inet6", [], query_options).rows + assert :inet.ntoa(ipv6) == ~c"2001:44c8:129:2632:33:0:252:2" end - test "decoded binaries copy behaviour", %{conn: conn} do + test "decoded binaries copy behaviour", %{conn: conn, query_options: query_options} do text = "hello world" - assert [[bin]] = Ch.query!(conn, "SELECT {$0:String}", [text]).rows + assert [[bin]] = Ch.query!(conn, "SELECT {$0:String}", [text], query_options).rows assert :binary.referenced_byte_size(bin) == :binary.referenced_byte_size("hello world") # For OTP 20+ refc binaries up to 64 bytes might be copied during a GC text = String.duplicate("hello world", 6) - assert [[bin]] = Ch.query!(conn, "SELECT {$0:String}", [text]).rows + assert [[bin]] = Ch.query!(conn, "SELECT {$0:String}", [text], query_options).rows assert :binary.referenced_byte_size(bin) == byte_size(text) end - test "encode basic types", %{conn: conn} do + test "encode basic types", %{conn: conn, query_options: query_options} do # TODO # assert [[nil, nil]] = query("SELECT $1::text, $2::int", [nil, nil]) - assert [[true, false]] = Ch.query!(conn, "SELECT {$0:bool}, {$1:Bool}", [true, false]).rows - assert [["ẽ"]] = Ch.query!(conn, "SELECT {$0:char}", ["ẽ"]).rows - assert [[42]] = Ch.query!(conn, "SELECT {$0:int}", [42]).rows - assert [[42.0, 43.0]] = Ch.query!(conn, "SELECT {$0:float}, {$1:float}", [42, 43.0]).rows - assert [[nil, nil]] = Ch.query!(conn, "SELECT {$0:float}, {$1:float}", ["NaN", "nan"]).rows - assert [[nil]] = Ch.query!(conn, "SELECT {$0:float}", ["inf"]).rows - assert [[nil]] = Ch.query!(conn, "SELECT {$0:float}", ["-inf"]).rows - assert [["ẽric"]] = Ch.query!(conn, "SELECT {$0:varchar}", ["ẽric"]).rows - assert [[<<1, 2, 3>>]] = Ch.query!(conn, "SELECT {$0:bytea}", [<<1, 2, 3>>]).rows + assert [[true, false]] = + Ch.query!(conn, "SELECT {$0:bool}, {$1:Bool}", [true, false], query_options).rows + + assert [["ẽ"]] = Ch.query!(conn, "SELECT {$0:char}", ["ẽ"], query_options).rows + assert [[42]] = Ch.query!(conn, "SELECT {$0:int}", [42], query_options).rows + + assert [[42.0, 43.0]] = + Ch.query!(conn, "SELECT {$0:float}, {$1:float}", [42, 43.0], query_options).rows + + assert [[nil, nil]] = + Ch.query!(conn, "SELECT {$0:float}, {$1:float}", ["NaN", "nan"], query_options).rows + + assert [[nil]] = Ch.query!(conn, "SELECT {$0:float}", ["inf"], query_options).rows + assert [[nil]] = Ch.query!(conn, "SELECT {$0:float}", ["-inf"], query_options).rows + assert [["ẽric"]] = Ch.query!(conn, "SELECT {$0:varchar}", ["ẽric"], query_options).rows + + assert [[<<1, 2, 3>>]] = + Ch.query!(conn, "SELECT {$0:bytea}", [<<1, 2, 3>>], query_options).rows end - test "encode numeric", %{conn: conn} do + test "encode numeric", %{conn: conn, query_options: query_options} do nums = [ {"42", "numeric(2,0)"}, {"0.4242", "numeric(4,4)"}, @@ -290,114 +375,141 @@ defmodule Ch.QueryTest do Enum.each(nums, fn {num, type} -> dec = Decimal.new(num) - assert [[dec]] == Ch.query!(conn, "SELECT {$0:#{type}}", [dec]).rows + assert [[dec]] == Ch.query!(conn, "SELECT {$0:#{type}}", [dec], query_options).rows end) end - test "encode integers and floats as numeric", %{conn: conn} do + test "encode integers and floats as numeric", %{conn: conn, query_options: query_options} do dec = Decimal.new(1) - assert [[dec]] == Ch.query!(conn, "SELECT {$0:numeric(1,0)}", [1]).rows + assert [[dec]] == Ch.query!(conn, "SELECT {$0:numeric(1,0)}", [1], query_options).rows dec = Decimal.from_float(1.0) - assert [[dec]] == Ch.query!(conn, "SELECT {$0:numeric(2,1)}", [1.0]).rows + assert [[dec]] == Ch.query!(conn, "SELECT {$0:numeric(2,1)}", [1.0], query_options).rows end @tag skip: true - test "encode json/jsonb", %{conn: conn} do + test "encode json/jsonb", %{conn: conn, query_options: query_options} do json = %{"foo" => 42} - assert [[json]] == Ch.query!(conn, "SELECT {$0::json}", [json]).rows + assert [[json]] == Ch.query!(conn, "SELECT {$0::json}", [json], query_options).rows end - test "encode uuid", %{conn: conn} do + test "encode uuid", %{conn: conn, query_options: query_options} do # TODO uuid = <<0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15>> uuid_hex = "00010203-0405-0607-0809-0a0b0c0d0e0f" - assert [[^uuid]] = Ch.query!(conn, "SELECT {$0:UUID}", [uuid_hex]).rows + assert [[^uuid]] = Ch.query!(conn, "SELECT {$0:UUID}", [uuid_hex], query_options).rows end - test "encode arrays", %{conn: conn} do - assert [[[]]] = Ch.query!(conn, "SELECT {$0:Array(integer)}", [[]]).rows - assert [[[1]]] = Ch.query!(conn, "SELECT {$0:Array(integer)}", [[1]]).rows - assert [[[1, 2]]] = Ch.query!(conn, "SELECT {$0:Array(integer)}", [[1, 2]]).rows + test "encode arrays", %{conn: conn, query_options: query_options} do + assert [[[]]] = Ch.query!(conn, "SELECT {$0:Array(integer)}", [[]], query_options).rows + assert [[[1]]] = Ch.query!(conn, "SELECT {$0:Array(integer)}", [[1]], query_options).rows - assert [[["1"]]] = Ch.query!(conn, "SELECT {$0:Array(String)}", [["1"]]).rows - assert [[[true]]] = Ch.query!(conn, "SELECT {$0:Array(Bool)}", [[true]]).rows + assert [[[1, 2]]] = + Ch.query!(conn, "SELECT {$0:Array(integer)}", [[1, 2]], query_options).rows + + assert [[["1"]]] = Ch.query!(conn, "SELECT {$0:Array(String)}", [["1"]], query_options).rows + assert [[[true]]] = Ch.query!(conn, "SELECT {$0:Array(Bool)}", [[true]], query_options).rows assert [[[~D[2023-01-01]]]] = - Ch.query!(conn, "SELECT {$0:Array(Date)}", [[~D[2023-01-01]]]).rows + Ch.query!(conn, "SELECT {$0:Array(Date)}", [[~D[2023-01-01]]], query_options).rows assert [[[Ch.Test.to_clickhouse_naive(conn, ~N[2023-01-01 12:00:00])]]] == - Ch.query!(conn, "SELECT {$0:Array(DateTime)}", [[~N[2023-01-01 12:00:00]]]).rows + Ch.query!( + conn, + "SELECT {$0:Array(DateTime)}", + [[~N[2023-01-01 12:00:00]]], + query_options + ).rows assert [[[~U[2023-01-01 12:00:00Z]]]] == - Ch.query!(conn, "SELECT {$0:Array(DateTime('UTC'))}", [[~N[2023-01-01 12:00:00]]]).rows + Ch.query!( + conn, + "SELECT {$0:Array(DateTime('UTC'))}", + [[~N[2023-01-01 12:00:00]]], + query_options + ).rows assert [[[~N[2023-01-01 12:00:00]]]] == - Ch.query!(conn, "SELECT {$0:Array(DateTime)}", [[~U[2023-01-01 12:00:00Z]]]).rows + Ch.query!( + conn, + "SELECT {$0:Array(DateTime)}", + [[~U[2023-01-01 12:00:00Z]]], + query_options + ).rows assert [[[~U[2023-01-01 12:00:00Z]]]] == - Ch.query!(conn, "SELECT {$0:Array(DateTime('UTC'))}", [[~U[2023-01-01 12:00:00Z]]]).rows + Ch.query!( + conn, + "SELECT {$0:Array(DateTime('UTC'))}", + [[~U[2023-01-01 12:00:00Z]]], + query_options + ).rows assert [[[[0], [1]]]] = - Ch.query!(conn, "SELECT {$0:Array(Array(integer))}", [[[0], [1]]]).rows + Ch.query!(conn, "SELECT {$0:Array(Array(integer))}", [[[0], [1]]], query_options).rows - assert [[[[0]]]] = Ch.query!(conn, "SELECT {$0:Array(Array(integer))}", [[[0]]]).rows - # assert [[[1, nil, 3]]] = Ch.query!(conn, "SELECT {$0:Array(integer)}", [[1, nil, 3]]).rows + assert [[[[0]]]] = + Ch.query!(conn, "SELECT {$0:Array(Array(integer))}", [[[0]]], query_options).rows + + # assert [[[1, nil, 3]]] = Ch.query!(conn, "SELECT {$0:Array(integer)}", [[1, nil, 3]], query_options).rows end - test "encode network types", %{conn: conn} do + test "encode network types", %{conn: conn, query_options: query_options} do # TODO, or wrap in custom struct like in postgrex # assert [["127.0.0.1/32"]] = - # Ch.query!(conn, "SELECT {$0:inet4}::text", [{127, 0, 0, 1}]).rows + # Ch.query!(conn, "SELECT {$0:inet4}::text", [{127, 0, 0, 1}], query_options).rows - assert [[{127, 0, 0, 1}]] = Ch.query!(conn, "SELECT {$0:text}::inet4", ["127.0.0.1"]).rows + assert [[{127, 0, 0, 1}]] = + Ch.query!(conn, "SELECT {$0:text}::inet4", ["127.0.0.1"], query_options).rows assert [[{0, 0, 0, 0, 0, 0, 0, 1}]] = - Ch.query!(conn, "SELECT {$0:text}::inet6", ["::1"]).rows + Ch.query!(conn, "SELECT {$0:text}::inet6", ["::1"], query_options).rows end - test "result struct", %{conn: conn} do - assert {:ok, res} = Ch.query(conn, "SELECT 123 AS a, 456 AS b") + test "result struct", %{conn: conn, query_options: query_options} do + assert {:ok, res} = Ch.query(conn, "SELECT 123 AS a, 456 AS b", [], query_options) assert %Ch.Result{} = res assert res.command == :select assert res.columns == ["a", "b"] assert res.num_rows == 1 end - test "empty result struct", %{conn: conn} do - assert %Ch.Result{} = res = Ch.query!(conn, "select number, 'a' as b from numbers(0)") + test "empty result struct", %{conn: conn, query_options: query_options} do + assert %Ch.Result{} = + res = Ch.query!(conn, "select number, 'a' as b from numbers(0)", [], query_options) + assert res.command == :select assert res.columns == ["number", "b"] assert res.rows == [] assert res.num_rows == 0 end - test "error struct", %{conn: conn} do - assert {:error, %Ch.Error{}} = Ch.query(conn, "SELECT 123 + 'a'") + test "error struct", %{conn: conn, query_options: query_options} do + assert {:error, %Ch.Error{}} = Ch.query(conn, "SELECT 123 + 'a'", [], query_options) end - test "error code", %{conn: conn} do - assert {:error, %Ch.Error{code: 62}} = Ch.query(conn, "wat") + test "error code", %{conn: conn, query_options: query_options} do + assert {:error, %Ch.Error{code: 62}} = Ch.query(conn, "wat", [], query_options) end - test "connection works after failure in execute", %{conn: conn} do - assert {:error, %Ch.Error{}} = Ch.query(conn, "wat") - assert [[42]] = Ch.query!(conn, "SELECT 42").rows + test "connection works after failure in execute", %{conn: conn, query_options: query_options} do + assert {:error, %Ch.Error{}} = Ch.query(conn, "wat", [], query_options) + assert [[42]] = Ch.query!(conn, "SELECT 42", [], query_options).rows end - test "async test", %{conn: conn} do + test "async test", %{conn: conn, query_options: query_options} do self_pid = self() Enum.each(1..10, fn _ -> spawn_link(fn -> - send(self_pid, Ch.query!(conn, "SELECT sleep(0.05)").rows) + send(self_pid, Ch.query!(conn, "SELECT sleep(0.05)", [], query_options).rows) end) end) - assert [[42]] = Ch.query!(conn, "SELECT 42").rows + assert [[42]] = Ch.query!(conn, "SELECT 42", [], query_options).rows Enum.each(1..10, fn _ -> - assert_receive [[0]] + assert_receive [[0]], :timer.seconds(1) end) end @@ -406,13 +518,13 @@ defmodule Ch.QueryTest do end end - test "query before and after idle ping" do + test "query before and after idle ping", %{query_options: query_options} do opts = [backoff_type: :stop, idle_interval: 1] {:ok, pid} = Ch.start_link(opts) - assert {:ok, _} = Ch.query(pid, "SELECT 42") + assert {:ok, _} = Ch.query(pid, "SELECT 42", [], query_options) :timer.sleep(20) - assert {:ok, _} = Ch.query(pid, "SELECT 42") + assert {:ok, _} = Ch.query(pid, "SELECT 42", [], query_options) :timer.sleep(20) - assert {:ok, _} = Ch.query(pid, "SELECT 42") + assert {:ok, _} = Ch.query(pid, "SELECT 42", [], query_options) end end diff --git a/test/ch/settings_test.exs b/test/ch/settings_test.exs index eb28489..ea28cf3 100644 --- a/test/ch/settings_test.exs +++ b/test/ch/settings_test.exs @@ -1,24 +1,33 @@ defmodule Ch.SettingsTest do - use ExUnit.Case + use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] - test "can start without settings" do + setup ctx do + {:ok, query_options: ctx[:query_options] || []} + end + + test "can start without settings", %{query_options: query_options} do assert {:ok, conn} = Ch.start_link() assert {:ok, %{num_rows: 1, rows: [["async_insert", "Bool", "0"]]}} = - Ch.query(conn, "show settings like 'async_insert'") + Ch.query(conn, "show settings like 'async_insert'", [], query_options) end - test "can pass default settings" do + test "can pass default settings", %{query_options: query_options} do assert {:ok, conn} = Ch.start_link(settings: [async_insert: 1]) assert {:ok, %{num_rows: 1, rows: [["async_insert", "Bool", "1"]]}} = - Ch.query(conn, "show settings like 'async_insert'") + Ch.query(conn, "show settings like 'async_insert'", [], query_options) end - test "can overwrite default settings with options" do + test "can overwrite default settings with options", %{query_options: query_options} do assert {:ok, conn} = Ch.start_link(settings: [async_insert: 1]) assert {:ok, %{num_rows: 1, rows: [["async_insert", "Bool", "0"]]}} = - Ch.query(conn, "show settings like 'async_insert'", [], settings: [async_insert: 0]) + Ch.query( + conn, + "show settings like 'async_insert'", + [], + Keyword.merge(query_options, settings: [async_insert: 0]) + ) end end diff --git a/test/ch/stream_test.exs b/test/ch/stream_test.exs index 2d6e4c6..62be761 100644 --- a/test/ch/stream_test.exs +++ b/test/ch/stream_test.exs @@ -1,17 +1,25 @@ defmodule Ch.StreamTest do - use ExUnit.Case + use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] alias Ch.{Result, RowBinary} + setup ctx do + {:ok, query_options: ctx[:query_options] || []} + end + setup do {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} end describe "enumerable Ch.stream/4" do - test "emits %Ch.Result{}", %{conn: conn} do + test "emits %Ch.Result{}", %{conn: conn, query_options: query_options} do results = DBConnection.run(conn, fn conn -> conn - |> Ch.stream("select * from numbers({count:UInt64})", %{"count" => 1_000_000}) + |> Ch.stream( + "select * from numbers({count:UInt64})", + %{"count" => 1_000_000}, + query_options + ) |> Enum.into([]) end) @@ -19,23 +27,27 @@ defmodule Ch.StreamTest do Enum.to_list(0..999_999) end - test "raises on error", %{conn: conn} do + test "raises on error", %{conn: conn, query_options: query_options} do assert_raise Ch.Error, ~r/Code: 62. DB::Exception: Syntax error: failed at position 8/, fn -> DBConnection.run(conn, fn conn -> - conn |> Ch.stream("select ", %{"count" => 1_000_000}) |> Enum.into([]) + conn + |> Ch.stream("select ", %{"count" => 1_000_000}, query_options) + |> Enum.into([]) end) end end - test "large strings", %{conn: conn} do + test "large strings", %{conn: conn, query_options: query_options} do results = DBConnection.run(conn, fn conn -> conn - |> Ch.stream("select repeat('abc', 500000) from numbers({count:UInt64})", %{ - "count" => 10 - }) + |> Ch.stream( + "select repeat('abc', 500000) from numbers({count:UInt64})", + %{"count" => 10}, + query_options + ) |> Enum.into([]) end) @@ -47,8 +59,9 @@ defmodule Ch.StreamTest do end describe "collectable Ch.stream/4" do - test "inserts chunks", %{conn: conn} do + test "inserts chunks", %{conn: conn, query_options: query_options} do Ch.query!(conn, "create table collect_stream(i UInt64) engine Memory") + on_exit(fn -> Ch.Test.query("DROP TABLE collect_stream") end) assert %Ch.Result{command: :insert, num_rows: 1_000_000} = DBConnection.run(conn, fn conn -> @@ -61,7 +74,7 @@ defmodule Ch.StreamTest do conn, "insert into collect_stream(i) format RowBinary", _params = [], - encode: false + Keyword.merge(query_options, encode: false) ) ) end) diff --git a/test/ch/variant_test.exs b/test/ch/variant_test.exs index e8008a5..fbd7144 100644 --- a/test/ch/variant_test.exs +++ b/test/ch/variant_test.exs @@ -1,5 +1,6 @@ defmodule Ch.VariantTest do - use ExUnit.Case + use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] + import Ch.Test, only: [parameterize_query!: 2, parameterize_query!: 4] # https://clickhouse.com/docs/sql-reference/data-types/variant @@ -10,17 +11,26 @@ defmodule Ch.VariantTest do {:ok, conn: conn} end - test "basic", %{conn: conn} do - assert Ch.query!(conn, "select null::Variant(UInt64, String, Array(UInt64))").rows == [[nil]] - assert Ch.query!(conn, "select [1]::Variant(UInt64, String, Array(UInt64))").rows == [[[1]]] - assert Ch.query!(conn, "select 0::Variant(UInt64, String, Array(UInt64))").rows == [[0]] + test "basic", ctx do + assert parameterize_query!(ctx, "select null::Variant(UInt64, String, Array(UInt64))").rows == + [[nil]] - assert Ch.query!(conn, "select 'Hello, World!'::Variant(UInt64, String, Array(UInt64))").rows == + assert parameterize_query!(ctx, "select [1]::Variant(UInt64, String, Array(UInt64))").rows == + [[[1]]] + + assert parameterize_query!(ctx, "select 0::Variant(UInt64, String, Array(UInt64))").rows == [ + [0] + ] + + assert parameterize_query!( + ctx, + "select 'Hello, World!'::Variant(UInt64, String, Array(UInt64))" + ).rows == [["Hello, World!"]] end # https://github.com/plausible/ch/issues/272 - test "ordering internal types", %{conn: conn} do + test "ordering internal types", ctx do test = %{ "'hello'" => "hello", "-10" => -10, @@ -30,27 +40,27 @@ defmodule Ch.VariantTest do } for {value, expected} <- test do - assert Ch.query!( - conn, + assert parameterize_query!( + ctx, "select #{value}::Variant(String, Int32, Bool, Map(String, Nullable(String)))" ).rows == [[expected]] end end - test "with a table", %{conn: conn} do + test "with a table", ctx do # https://clickhouse.com/docs/sql-reference/data-types/variant#creating-variant - Ch.query!(conn, """ + parameterize_query!(ctx, """ CREATE TABLE variant_test (v Variant(UInt64, String, Array(UInt64))) ENGINE = Memory; """) on_exit(fn -> Ch.Test.query("DROP TABLE variant_test") end) - Ch.query!( - conn, + parameterize_query!( + ctx, "INSERT INTO variant_test VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);" ) - assert Ch.query!(conn, "SELECT v FROM variant_test").rows == [ + assert parameterize_query!(ctx, "SELECT v FROM variant_test").rows == [ [nil], [42], ["Hello, World!"], @@ -58,7 +68,10 @@ defmodule Ch.VariantTest do ] # https://clickhouse.com/docs/sql-reference/data-types/variant#reading-variant-nested-types-as-subcolumns - assert Ch.query!(conn, "SELECT v, v.String, v.UInt64, v.`Array(UInt64)` FROM variant_test;").rows == + assert parameterize_query!( + ctx, + "SELECT v, v.String, v.UInt64, v.`Array(UInt64)` FROM variant_test;" + ).rows == [ [nil, nil, nil, []], [42, nil, 42, []], @@ -66,8 +79,8 @@ defmodule Ch.VariantTest do [[1, 2, 3], nil, nil, [1, 2, 3]] ] - assert Ch.query!( - conn, + assert parameterize_query!( + ctx, "SELECT v, variantElement(v, 'String'), variantElement(v, 'UInt64'), variantElement(v, 'Array(UInt64)') FROM variant_test;" ).rows == [ [nil, nil, nil, []], @@ -77,21 +90,21 @@ defmodule Ch.VariantTest do ] end - test "rowbinary", %{conn: conn} do - Ch.query!(conn, """ + test "rowbinary", ctx do + parameterize_query!(ctx, """ CREATE TABLE variant_test (v Variant(UInt64, String, Array(UInt64))) ENGINE = Memory; """) on_exit(fn -> Ch.Test.query("DROP TABLE variant_test") end) - Ch.query!( - conn, + parameterize_query!( + ctx, "INSERT INTO variant_test FORMAT RowBinary", [[nil], [42], ["Hello, World!"], [[1, 2, 3]]], types: ["Variant(UInt64, String, Array(UInt64))"] ) - assert Ch.query!(conn, "SELECT v FROM variant_test").rows == [ + assert parameterize_query!(ctx, "SELECT v FROM variant_test").rows == [ [nil], [42], ["Hello, World!"],