diff --git a/bin/mzbench b/bin/mzbench deleted file mode 100755 index 87627085a9c52..0000000000000 --- a/bin/mzbench +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. -# -# mzcompose — runs Docker Compose with Materialize customizations. - -exec "$(dirname "$0")"/pyactivate --dev -m materialize.cli.mzbench "$@" diff --git a/ci/test/pipeline.template.yml b/ci/test/pipeline.template.yml index f5605a3f16822..b759fba261d5a 100644 --- a/ci/test/pipeline.template.yml +++ b/ci/test/pipeline.template.yml @@ -178,33 +178,6 @@ steps: run: ci timeout_in_minutes: 30 - - id: aggregations-benchmark - label: "aggregations benchmark sanity check" - depends_on: build-x86_64 - plugins: - - ./ci/plugins/mzcompose: - composition: aggregations - run: ci - timeout_in_minutes: 10 - - - id: avro-upsert-benchmark - label: "avro upsert benchmark sanity check" - depends_on: build-x86_64 - plugins: - - ./ci/plugins/mzcompose: - composition: avro-upsert - run: ci - timeout_in_minutes: 10 - - - id: kafka-sink-avro-debezium-benchmark - label: "kafka sink (avro debezium) benchmark sanity check" - depends_on: build-x86_64 - plugins: - - ./ci/plugins/mzcompose: - composition: kafka-sink-avro-debezium - run: ci - timeout_in_minutes: 10 - - id: catalog-compat label: ":book: catalog compatibility check" depends_on: build-x86_64 diff --git a/doc/developer/mzbench.md b/doc/developer/mzbench.md deleted file mode 100644 index 1c16f1512a382..0000000000000 --- a/doc/developer/mzbench.md +++ /dev/null @@ -1,126 +0,0 @@ -# mzbench - -mzbench is a tool that wraps mzcompose in order to repeatedly run benchmarks against one or more -versions of Materialize, with a matrix of variables to test. Currently, mzbench explicitly varies -the value for `MZ_WORKERS` and allows for choosing which materialize versions to test. Users can -also select the number of times to test each version, as a means of improving confidence in the -results. - -## Running a Benchmark - -Benchmarks are identified by the name of their composition and compositions are expected to -expose workflows with specific names. To run a benchmark, run `mzbench` from the root of your -materialize git checkout: - - ./bin/mzbench - -This script defaults to choosing the benchmark most suitable for developer machines -(`benchmark-medium` for fast laptops) and will run through a permutation of benchmark variables. -It will print a CSV that looks something like the following to your console: - - git_revision,num_workers,iteration,seconds_taken,rows_per_second,grafana_url - NONE,1,0,3,333333,http://localhost:3000/d/materialize-overview/materialize-overview?from=1612572459000&to=1612573285000&tz=UTC - -### Comparing Results Between Materialize Versions - -Benchmark results can be considered either absolute (X operations per second) or relative -(10% faster than the previous release). Our current crop of benchmarks are written to test -relative performance. To help engineers better understand materialized performance, mzbench -supports running the benchmark over multiple versions of materialized: - - ./bin/mzbench ... - -For example, here's how to compare the performance of materialized, built from your local working -directory, against `v0.6.1` and `v0.7.0`: - - ./bin/mzbench v0.7.0 v0.6.1 - -If you don't want to benchmark the version in your local working directory, such as when -benchmarking in the cloud, you can supply the `--no-benchmark-this-checkout` flag: - - ./bin/mzbench --no-benchmark-this-checkout origin/main v0.7.0 v0.6.1 - -### Controlling the Number of Iterations - -Our benchmarks are not sufficiently advanced to provide an accurate picture after only a single -run. By default, mzbench will run each measurement 6 times. This can be controlled via the -`--num-measurements` flag: - - ./bin/mzbench --num-measurements 3 - -## Visualizing Results - -If the benchmark is configured to start `perf-dash` services during the setup step, then you can -visualize results in your browser. These results are update in real-time, as the results are -written to the persistent datastore for perf-dash. To view the benchmark results in your browser, -you can run `mzbench` with the `--web` flag: - - ./bin/mzbench --web ... - -This will open a new browser window, pointed at the web interface for perf-dash, once the services -have started up. - -## Adding New Benchmarks (Compositions) - -`mzbench` expects that benchmarks expose 3 different sizes of benchmarks: - -- `benchmark-ci` - for tests that are intended to verify correctness, not performance. -- `benchmark-medium` - for tests that are intended for developers to understand the performance - changes between two versions of the code. -- `benchmark-large` - for tests that are intended to verify performance in absolute terms. These - are the tests that we run in our cloud based benchmarks. - -Users of `mzbench` can choose which variant to run by using the `--size` flag. - -To run a given benchmark, `mzbench` assumes that there are 2 corresponding workflows for each -benchmark: - -- `setup-` - this job is run once at mzbench startup to initialize any supporting - resources, such as a Kafka cluster or other source of data. -- `run-` - this job is run once for iteration of the benchmark. This workflow is - responsible for clearing any materialized state and making sure that materialized is restarted - with the new parameters set by the benchmark iteration. - -This means that, to write a new benchmark that supports running in CI, on laptops and in the -cloud, 6 workflows must exist: - -- `setup-benchmark-ci` -- `run-benchmark-ci` -- `setup-benchmark-medium` -- `run-benchmark-medium` -- `setup-benchmark-large` -- `run-benchmark-large` - -## Worker Balance Visualization - -If run with the `--web` flag, `mzbench` will open a browser to the results visualizations. In -addition to the results visualization, there is a demonstration visualization calling "Elapsed -Time Per Worker" (you may need to scroll down to see it). This visualization is a "technology -demostration" for replicating system tables to another materialized instance to power a real-time -visualization. - -The code to make this all work lives under `misc/perf-dash` and consists of 3 major components: - -1. `perf-dash-scraper`. A utility that periodically queries system tables, encodes them as Avro and - writes them to a Kafka topic. -2. `perf-dash-metrics`: A materialized instanced configured to read metrics from Kafka and - maintain a materialized view called `time_per_worker`. -3. `perf-dash-web`: A web server configured serve the visualization, pushing updates to the - browser directly from tail. - -Similar metrics can be inferred by using Prometheus / Grafana, as many of the system table -aggregations are also exported as prometheus metrics. So why go through the work of scraping -system tables, writing avro schemas, writing materialized views and custom visualizations? There -are a few motivations: - -1. Demonstrate materialize feature parity with an existing metrics / reporting stack. -2. Demonstrate the value in replicating system tables as SQL to another materialized instance for - telemetry purposes. -3. Demonstrate how to write an event-sourced application using materialized / using materialized - as a denormalizatin engine. -4. Demonstrate the value of real-time visualizations over 5-second refresh, even for internal - engineering projects. -5. Provide real data that can be used as a test-bed for features, particularly around temporal - features (aggregations, sliding / tumbling windows, etc). -6. Lossless metrics collection. Instead of relying on Prometheus scrape intervals, metrics are - persistently recorded in Kafka. No more gaps in our metrics. diff --git a/misc/perf-dash/app/Dockerfile b/misc/perf-dash/app/Dockerfile deleted file mode 100644 index 87bf89b1505b8..0000000000000 --- a/misc/perf-dash/app/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -MZFROM wikirecent-server - -# Override the wikirecent HTML / JS template files with our own -COPY templates /app/templates - -# Override the views that we expose via TAIL -CMD ["--dbhost=perf-dash-metrics", \ - "--dbport=8675", \ - "benchmark_results", \ - "benchmark_progress", \ - "time_per_worker"] diff --git a/misc/perf-dash/app/mzbuild.yml b/misc/perf-dash/app/mzbuild.yml deleted file mode 100644 index faf0e91a11178..0000000000000 --- a/misc/perf-dash/app/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: perf-dash-web -publish: false diff --git a/misc/perf-dash/app/templates/index.html b/misc/perf-dash/app/templates/index.html deleted file mode 100644 index 707c89555b559..0000000000000 --- a/misc/perf-dash/app/templates/index.html +++ /dev/null @@ -1,203 +0,0 @@ - - - {% block title %}Materialize Performance Dashboard{% end %} - - - - - - - -
-
-
- - - - - - diff --git a/misc/perf-dash/create_views/Dockerfile b/misc/perf-dash/create_views/Dockerfile deleted file mode 100644 index 440e69e9ff3a2..0000000000000 --- a/misc/perf-dash/create_views/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -FROM postgres - -RUN mkdir /data -WORKDIR /data -COPY views . - -COPY create_views /usr/local/bin -ENTRYPOINT ["/usr/local/bin/create_views"] diff --git a/misc/perf-dash/create_views/create_views b/misc/perf-dash/create_views/create_views deleted file mode 100755 index 0072452cf0466..0000000000000 --- a/misc/perf-dash/create_views/create_views +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -set -euo pipefail - -for view_file in *.sql; do - psql -v ON_ERROR_STOP=1 -h perf-dash-metrics -p 8675 -U materialize -f "${view_file}" materialize -done diff --git a/misc/perf-dash/create_views/mzbuild.yml b/misc/perf-dash/create_views/mzbuild.yml deleted file mode 100644 index fac0df167db99..0000000000000 --- a/misc/perf-dash/create_views/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: perf-dash-create-views -publish: false diff --git a/misc/perf-dash/create_views/views/benchmark_views.sql b/misc/perf-dash/create_views/views/benchmark_views.sql deleted file mode 100644 index dea3e910fe35e..0000000000000 --- a/misc/perf-dash/create_views/views/benchmark_views.sql +++ /dev/null @@ -1,53 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE benchmark_run_params_v0 -FROM KAFKA BROKER 'kafka:9093' -TOPIC 'dev.mtrlz.benchmarks.runs.params.v0' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE UPSERT -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081'; - -CREATE SOURCE benchmark_run_begin_v0 -FROM KAFKA BROKER 'kafka:9093' -TOPIC 'dev.mtrlz.benchmarks.runs.begin.v0' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE UPSERT -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081'; - -CREATE SOURCE benchmark_run_results_v0 -FROM KAFKA BROKER 'kafka:9093' -TOPIC 'dev.mtrlz.benchmarks.runs.results.v0' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE UPSERT -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081'; - -CREATE OR REPLACE MATERIALIZED VIEW benchmark_progress AS - SELECT bp.benchmark_id, - bb.run_id, - bb.start_ms, - CASE WHEN br.end_ms IS NOT NULL THEN br.end_ms - bb.start_ms ELSE null END AS runtime, - bp.git_ref, - bp.mz_workers, - br.end_ms, - CASE WHEN br.result IS NOT NULL THEN br.result ELSE 'running' END AS result - FROM benchmark_run_begin_v0 AS bb - INNER JOIN benchmark_run_params_v0 AS bp ON bb.run_id = bp.run_id - LEFT JOIN benchmark_run_results_v0 AS br ON bb.run_id = br.run_id; - -CREATE OR REPLACE MATERIALIZED VIEW benchmark_results AS - SELECT bp.benchmark_id, - bb.run_id, - bp.git_ref, - bp.mz_workers, - br.result, - br.rows_per_second - FROM benchmark_run_begin_v0 AS bb - INNER JOIN benchmark_run_params_v0 AS bp ON bb.run_id = bp.run_id - INNER JOIN benchmark_run_results_v0 AS br ON bb.run_id = br.run_id; diff --git a/misc/perf-dash/create_views/views/system_views.sql b/misc/perf-dash/create_views/views/system_views.sql deleted file mode 100644 index f3f02277921f2..0000000000000 --- a/misc/perf-dash/create_views/views/system_views.sql +++ /dev/null @@ -1,36 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE time_per_operator_v0 -FROM KAFKA BROKER 'kafka:9093' -TOPIC 'dev.mtrlz.time_per_operator.v0' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE SOURCE time_per_operator_per_worker_v0 -FROM KAFKA BROKER 'kafka:9093' -TOPIC 'dev.mtrlz.time_per_operator_per_worker.v0' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE SOURCE time_per_worker_v0 -FROM KAFKA BROKER 'kafka:9093' -TOPIC 'dev.mtrlz.time_per_worker.v0' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE MATERIALIZED VIEW IF NOT EXISTS time_per_operator_per_worker AS - SELECT * FROM time_per_operator_per_worker_v0; - -CREATE MATERIALIZED VIEW IF NOT EXISTS time_per_operator AS - SELECT * FROM time_per_operator_v0; - -CREATE MATERIALIZED VIEW IF NOT EXISTS time_per_worker AS - SELECT mz_cluster_id, mz_scheduling_worker_id, mz_scheduling_elapsed_ns FROM time_per_worker_v0 - WHERE mz_logical_timestamp = (SELECT max(mz_logical_timestamp) FROM time_per_worker_v0); diff --git a/misc/perf-dash/scraper/Dockerfile b/misc/perf-dash/scraper/Dockerfile deleted file mode 100644 index 8bf1457393c59..0000000000000 --- a/misc/perf-dash/scraper/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -MZFROM ubuntu-base - -ARG DEBIAN_FRONTEND="noninteractive" - -RUN apt-get update && apt-get install -y librdkafka-dev python3-dev python3 python3-pip tzdata && apt-get clean - -COPY requirements.txt /tmp -RUN pip3 install -r /tmp/requirements.txt - -RUN mkdir -p /usr/share/perf-scraper/metrics -COPY metrics /usr/share/perf-scraper/metrics - -COPY scrape_system_metrics /usr/local/bin -ENTRYPOINT ["scrape_system_metrics"] -CMD ["scrape"] diff --git a/misc/perf-dash/scraper/README.md b/misc/perf-dash/scraper/README.md deleted file mode 100644 index ed9192464bd65..0000000000000 --- a/misc/perf-dash/scraper/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# scraper - -Captures metrics from Materialize and dumps them to a per-metric Kafka topic. diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/key-schema.avsc b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/key-schema.avsc deleted file mode 100644 index 1a6289d935987..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/key-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "key", - "type" : "record", - "namespace": "dev.mtrlz.time_per_operator.v0", - "fields": [ - { - "name": "mz_cluster_id", - "type": "string" - } - ] -} diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/query.sql b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/query.sql deleted file mode 100644 index aba45e81a6c46..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/query.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - --- Get time spent in each Operator -SELECT mz_cluster_id() AS mz_cluster_id, - mz_logical_timestamp()::float AS mz_logical_timestamp, - mz_dataflow_operators.id AS mz_dataflow_operator_id, - mz_dataflow_operators.name AS mz_dataflow_operator_name, - sum(mz_scheduling_elapsed.elapsed_ns)::bigint AS mz_scheduling_elapsed_ns -FROM mz_scheduling_elapsed, - mz_dataflow_operators -WHERE mz_scheduling_elapsed.id = mz_dataflow_operators.id - AND mz_scheduling_elapsed.worker = mz_dataflow_operators.worker -GROUP BY mz_dataflow_operators.id, - mz_dataflow_operators.name -ORDER BY mz_dataflow_operators.id ASC; diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/value-schema.avsc b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/value-schema.avsc deleted file mode 100644 index 20882dba9bafe..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/value-schema.avsc +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "value", - "type" : "record", - "namespace": "dev.mtrlz.time_per_operator.v0", - "fields": [ - {"name": "mz_cluster_id", "type": "string"}, - {"name": "mz_logical_timestamp", "type": "double"}, - {"name": "mz_dataflow_operator_id", "type": "int"}, - {"name": "mz_dataflow_operator_name", "type": "string"}, - {"name": "mz_scheduling_elapsed_ns", "type": "long"} - ] -} diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/key-schema.avsc b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/key-schema.avsc deleted file mode 100644 index 6016e2ddeb697..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/key-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "key", - "type" : "record", - "namespace": "dev.mtrlz.time_per_operator_per_worker.v0", - "fields": [ - { - "name": "mz_cluster_id", - "type": "string" - } - ] -} diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/query.sql b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/query.sql deleted file mode 100644 index c40d7fc14cc06..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/query.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - --- Get time spent in each Operator per Worker -SELECT mz_cluster_id() AS mz_cluster_id, - mz_logical_timestamp()::float AS mz_logical_timestamp, - mz_dataflow_operators.id AS mz_dataflow_operator_id, - mz_dataflow_operators.worker AS mz_dataflow_operator_worker, - mz_dataflow_operators.name AS mz_dataflow_operator_name, - mz_scheduling_elapsed.elapsed_ns mz_scheduling_elapsed_ns -FROM mz_scheduling_elapsed, - mz_dataflow_operators -WHERE mz_scheduling_elapsed.id = mz_dataflow_operators.id - AND mz_scheduling_elapsed.worker = mz_dataflow_operators.worker -ORDER BY mz_dataflow_operators.id ASC; diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/value-schema.avsc b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/value-schema.avsc deleted file mode 100644 index f2ccbc7d5a8d0..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/value-schema.avsc +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "value", - "type" : "record", - "namespace": "dev.mtrlz.time_per_operator_per_worker.v0", - "fields": [ - {"name": "mz_cluster_id", "type": "string"}, - {"name": "mz_logical_timestamp", "type": "double"}, - {"name": "mz_dataflow_operator_id", "type": "int"}, - {"name": "mz_dataflow_operator_worker", "type": "int"}, - {"name": "mz_dataflow_operator_name", "type": "string"}, - {"name": "mz_scheduling_elapsed_ns", "type": "long"} - ] -} diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/key-schema.avsc b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/key-schema.avsc deleted file mode 100644 index efd7c2e8599df..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/key-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "key", - "type" : "record", - "namespace": "dev.mtrlz.time_per_worker.v0", - "fields": [ - { - "name": "mz_cluster_id", - "type": "string" - } - ] -} diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/query.sql b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/query.sql deleted file mode 100644 index 57a8aa2a1bf7e..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/query.sql +++ /dev/null @@ -1,17 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - --- Get time spent in each Worker -SELECT mz_cluster_id() AS mz_cluster_id, - mz_logical_timestamp()::float AS mz_logical_timestamp, - mz_scheduling_elapsed.worker AS mz_scheduling_worker_id, - sum(mz_scheduling_elapsed.elapsed_ns)::bigint AS mz_scheduling_elapsed_ns -FROM mz_scheduling_elapsed -GROUP BY mz_scheduling_elapsed.worker -ORDER BY mz_scheduling_elapsed.worker ASC; diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/value-schema.avsc b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/value-schema.avsc deleted file mode 100644 index 10a8c6ec1175a..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/value-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "value", - "type" : "record", - "namespace": "dev.mtrlz.time_per_worker.v0", - "fields": [ - {"name": "mz_cluster_id", "type": "string"}, - {"name": "mz_logical_timestamp", "type": "double"}, - {"name": "mz_scheduling_worker_id", "type": "int"}, - {"name": "mz_scheduling_elapsed_ns", "type": "long"} - ] -} diff --git a/misc/perf-dash/scraper/mzbuild.yml b/misc/perf-dash/scraper/mzbuild.yml deleted file mode 100644 index 49fda457ea979..0000000000000 --- a/misc/perf-dash/scraper/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: perf-dash-scraper -publish: false diff --git a/misc/perf-dash/scraper/requirements.txt b/misc/perf-dash/scraper/requirements.txt deleted file mode 100644 index d7f2fb2ecebb1..0000000000000 --- a/misc/perf-dash/scraper/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -confluent-kafka[avro]==1.5.0 -dateparser==1.0.0 -psycopg2-binary==2.8.6 diff --git a/misc/perf-dash/scraper/scrape_system_metrics b/misc/perf-dash/scraper/scrape_system_metrics deleted file mode 100755 index 9c368e9cde040..0000000000000 --- a/misc/perf-dash/scraper/scrape_system_metrics +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -import argparse -import datetime -import io -import os -import pathlib -import sys -import time -import typing -import urllib - -import confluent_kafka.admin -import confluent_kafka.avro -import confluent_kafka.schema_registry -import dateparser -import psycopg2 -import psycopg2.extensions -import psycopg2.extras - -METRICS_DIRECTORY = "/usr/share/perf-scraper/metrics" - - -class Scraper: - - def __init__(self, metric_dir: pathlib.Path, kafka_config: typing.Dict[str, str]): - """Construct a producer for writing to the given topic, using the given config.""" - self.topic_name = metric_dir.name - - key_schema = confluent_kafka.avro.load(pathlib.Path(metric_dir, 'key-schema.avsc')) - value_schema = confluent_kafka.avro.load(pathlib.Path(metric_dir, 'value-schema.avsc')) - - self.query = pathlib.Path(metric_dir, 'query.sql').read_text().strip() - - # Namespace for both schemas should be the name of the topic - assert key_schema.namespace == value_schema.namespace - assert self.topic_name == key_schema.namespace - - self.producer = confluent_kafka.avro.AvroProducer(kafka_config, - default_key_schema=key_schema, - default_value_schema=value_schema) - - def scrape(self, cursor: psycopg2.extensions.cursor) -> None: - """Issue a query against the system tables and write the data to Kafka.""" - cursor.execute(self.query) - for row in cursor: - key = {"mz_cluster_id": row["mz_cluster_id"]} - self.producer.produce(topic=self.topic_name, key=key, value=row) - self.producer.flush() - - -class MetricsScrapers: - - def __init__(self, args: argparse.Namespace): - """Create an instance responsible for recording benchmark metrics.""" - self.scrapers: typing.Dict[str, Scraper] = {} - - # See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - self.kafka_config = {'bootstrap.servers': f"{args.kafka_host}:{args.kafka_port}", - 'schema.registry.url': f'http://{args.csr_host}:{args.csr_port}'} - for metric_dir in pathlib.Path(args.metrics_directory).iterdir(): - self.scrapers[metric_dir.name] = Scraper(metric_dir, self.kafka_config) - - def scrape(self, dsn: str) -> None: - """Encode key and value using Avro and send event to Kafka.""" - with psycopg2.connect(dsn) as conn: - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cursor: - for scraper in self.scrapers.values(): - scraper.scrape(cursor) - - -def create_topics(args: argparse.Namespace) -> None: - """Create the topics for each metric and exit.""" - - csr_config = {'url': f'http://{args.csr_host}:{args.csr_port}'} - csr = confluent_kafka.schema_registry.SchemaRegistryClient(csr_config) - - kafka_config = {'bootstrap.servers': f"{args.kafka_host}:{args.kafka_port}"} - admin_client = confluent_kafka.admin.AdminClient(kafka_config) - - for metric_dir in pathlib.Path(args.metrics_directory).iterdir(): - - key_schema_str = pathlib.Path(metric_dir, 'key-schema.avsc').read_text().strip() - key_schema = confluent_kafka.schema_registry.Schema(key_schema_str, 'AVRO') - value_schema_str = pathlib.Path(metric_dir, 'value-schema.avsc').read_text().strip() - value_schema = confluent_kafka.schema_registry.Schema(value_schema_str, 'AVRO') - - csr.register_schema(f"{metric_dir.name}-key", key_schema) - csr.register_schema(f"{metric_dir.name}-value", value_schema) - - # Create topics takes and returns a list of futures but we're going to call create topic - # while iterating on each directory, so these are just lists of 1 - topics = [confluent_kafka.admin.NewTopic(metric_dir.name, num_partitions=10, replication_factor=1)] - # Don't bother trying to catch the error, let's just fail startup and raise the error - [future.result() for future in admin_client.create_topics(topics).values()] - - -def scrape(args: argparse.Namespace) -> None: - """Grab system metrics from the configured materialized instance, replicating them to Kafka""" - dsn = f'postgresql://{args.dbuser}@{args.dbhost}:{args.dbport}/{args.dbname}' - scrapers = MetricsScrapers(args) - while True: - scrapers.scrape(dsn) - time.sleep(args.scrape_interval) - - -def run(args: argparse.Namespace) -> None: - """Wait for the query to settle or timeout and then dump ingest metrics.""" - args.action(args) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - - parser.add_argument( - "-v", - "--verbose", - action="store_true", - help="Enable verbose logging to print the results of each prometheus query", - ) - - parser.add_argument( - "--metrics-directory", - type=str, - default=METRICS_DIRECTORY, - help="Directory containing metrics definitions and source queries", - ) - - parser.add_argument( - "--kafka-host", - help="Name of the kafka broker", - type=str, - default="kafka", - ) - - parser.add_argument( - "--kafka-port", help="Port the connect to the broker over", type=int, default=9092 - ) - - parser.add_argument( - "--csr-host", help="Hostname of the schema registry", type=str, - default="schema-registry" - ) - parser.add_argument( - "--csr-port", help="Port that schema registry is listening on", type=int, - default=8081 - ) - - sub_parsers = parser.add_subparsers() - - # Create Topics Subcommand and Flags - create_topic_parser = sub_parsers.add_parser("create-topics") - create_topic_parser.set_defaults(action=create_topics) - - # Scrape Subcommand and Flags - scrape_parser = sub_parsers.add_parser("scrape") - scrape_parser.set_defaults(action=scrape) - - scrape_parser.add_argument( - "--dbhost", help="materialized hostname", default="materialized", type=str - ) - scrape_parser.add_argument( - "--dbname", help="materialized database name", default="materialize", type=str - ) - scrape_parser.add_argument( - "--dbport", help="materialized port number", default=6875, type=int - ) - scrape_parser.add_argument( - "--dbuser", help="materialized username", default="materialize", type=str - ) - - scrape_parser.add_argument( - "--scrape-interval", help="How often, in seconds, to scrape metrics", type=int, - default=1 - ) - - return parser.parse_args() - - -if __name__ == '__main__': - run(parse_args()) diff --git a/test/bench/aggregations/README.md b/test/bench/aggregations/README.md deleted file mode 100644 index 3516c43c37136..0000000000000 --- a/test/bench/aggregations/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Aggregations Benchmark - -This benchmark tests the performance for aggregations using the Decimal datatype. - -## Running the Aggregations Benchmark - -To run this benchmark, use [mzbench](../../../doc/developer/mzbench.md). diff --git a/test/bench/aggregations/create_views/Dockerfile b/test/bench/aggregations/create_views/Dockerfile deleted file mode 100644 index 41007ae2b1144..0000000000000 --- a/test/bench/aggregations/create_views/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -FROM postgres - -RUN mkdir /data -COPY views /data - -COPY create_views /usr/local/bin -ENTRYPOINT ["/usr/local/bin/create_views"] diff --git a/test/bench/aggregations/create_views/create_views b/test/bench/aggregations/create_views/create_views deleted file mode 100755 index be4129677ade2..0000000000000 --- a/test/bench/aggregations/create_views/create_views +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -set -euo pipefail - -view_file="/data/${1:-aggregation_views.sql}" - -psql -v ON_ERROR_STOP=1 -h materialized -p 6875 -U materialize -f "${view_file}" materialize diff --git a/test/bench/aggregations/create_views/mzbuild.yml b/test/bench/aggregations/create_views/mzbuild.yml deleted file mode 100644 index dede67cb19883..0000000000000 --- a/test/bench/aggregations/create_views/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: aggregations-bench-create-views -publish: false diff --git a/test/bench/aggregations/create_views/views/aggregation_views.sql b/test/bench/aggregations/create_views/views/aggregation_views.sql deleted file mode 100644 index ceb8cd1e45ee2..0000000000000 --- a/test/bench/aggregations/create_views/views/aggregation_views.sql +++ /dev/null @@ -1,118 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE source_aggregationtest -FROM KAFKA BROKER 'kafka:9092' -TOPIC 'aggregationtest' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE MATERIALIZED VIEW aggregationtest AS SELECT - sum(("OuterRecord")."DecValue00") AS dec00_sum, - sum(("OuterRecord")."DecValue01") AS dec01_sum, - sum(("OuterRecord")."DecValue02") AS dec02_sum, - sum(("OuterRecord")."DecValue03") AS dec03_sum, - sum(("OuterRecord")."DecValue04") AS dec04_sum, - sum(("OuterRecord")."DecValue05") AS dec05_sum, - sum(("OuterRecord")."DecValue06") AS dec06_sum, - sum(("OuterRecord")."DecValue07") AS dec07_sum, - sum(("OuterRecord")."DecValue08") AS dec08_sum, - sum(("OuterRecord")."DecValue09") AS dec09_sum, - sum(("OuterRecord")."DecValue10") AS dec10_sum, - sum(("OuterRecord")."DecValue11") AS dec11_sum, - sum(("OuterRecord")."DecValue12") AS dec12_sum, - sum(("OuterRecord")."DecValue13") AS dec13_sum, - sum(("OuterRecord")."DecValue14") AS dec14_sum, - sum(("OuterRecord")."DecValue15") AS dec15_sum, - sum(("OuterRecord")."DecValue16") AS dec16_sum, - sum(("OuterRecord")."DecValue17") AS dec17_sum, - sum(("OuterRecord")."DecValue18") AS dec18_sum, - sum(("OuterRecord")."DecValue19") AS dec19_sum, - sum(("OuterRecord")."DecValue20") AS dec20_sum, - sum(("OuterRecord")."DecValue21") AS dec21_sum, - sum(("OuterRecord")."DecValue22") AS dec22_sum, - sum(("OuterRecord")."DecValue23") AS dec23_sum, - sum(("OuterRecord")."DecValue24") AS dec24_sum, - sum(("OuterRecord")."DecValue25") AS dec25_sum, - sum(("OuterRecord")."DecValue26") AS dec26_sum, - sum(("OuterRecord")."DecValue27") AS dec27_sum, - sum(("OuterRecord")."DecValue28") AS dec28_sum, - sum(("OuterRecord")."DecValue29") AS dec29_sum, - sum(("OuterRecord")."DecValue30") AS dec30_sum, - sum(("OuterRecord")."DecValue31") AS dec31_sum, - sum(("OuterRecord")."DecValue32") AS dec32_sum, - sum(("OuterRecord")."DecValue33") AS dec33_sum, - sum(("OuterRecord")."DecValue34") AS dec34_sum, - sum(("OuterRecord")."DecValue35") AS dec35_sum, - sum(("OuterRecord")."DecValue36") AS dec36_sum, - sum(("OuterRecord")."DecValue37") AS dec37_sum, - sum(("OuterRecord")."DecValue38") AS dec38_sum, - sum(("OuterRecord")."DecValue39") AS dec39_sum, - sum(("OuterRecord")."DecValue40") AS dec40_sum, - sum(("OuterRecord")."DecValue41") AS dec41_sum, - sum(("OuterRecord")."DecValue42") AS dec42_sum, - sum(("OuterRecord")."DecValue43") AS dec43_sum, - sum(("OuterRecord")."DecValue44") AS dec44_sum, - sum(("OuterRecord")."DecValue45") AS dec45_sum, - sum(("OuterRecord")."DecValue46") AS dec46_sum, - sum(("OuterRecord")."DecValue47") AS dec47_sum, - sum(("OuterRecord")."DecValue48") AS dec48_sum, - sum(("OuterRecord")."DecValue49") AS dec49_sum, - sum(("OuterRecord")."DecValue50") AS dec50_sum, - sum(("OuterRecord")."DecValue51") AS dec51_sum, - sum(("OuterRecord")."DecValue52") AS dec52_sum, - sum(("OuterRecord")."DecValue53") AS dec53_sum, - sum(("OuterRecord")."DecValue54") AS dec54_sum, - sum(("OuterRecord")."DecValue55") AS dec55_sum, - sum(("OuterRecord")."DecValue56") AS dec56_sum, - sum(("OuterRecord")."DecValue57") AS dec57_sum, - sum(("OuterRecord")."DecValue58") AS dec58_sum, - sum(("OuterRecord")."DecValue59") AS dec59_sum, - sum(("OuterRecord")."DecValue60") AS dec60_sum, - sum(("OuterRecord")."DecValue61") AS dec61_sum, - sum(("OuterRecord")."DecValue62") AS dec62_sum, - sum(("OuterRecord")."DecValue63") AS dec63_sum, - sum(("OuterRecord")."DecValue64") AS dec64_sum, - sum(("OuterRecord")."DecValue65") AS dec65_sum, - sum(("OuterRecord")."DecValue66") AS dec66_sum, - sum(("OuterRecord")."DecValue67") AS dec67_sum, - sum(("OuterRecord")."DecValue68") AS dec68_sum, - sum(("OuterRecord")."DecValue69") AS dec69_sum, - sum(("OuterRecord")."DecValue70") AS dec70_sum, - sum(("OuterRecord")."DecValue71") AS dec71_sum, - sum(("OuterRecord")."DecValue72") AS dec72_sum, - sum(("OuterRecord")."DecValue73") AS dec73_sum, - sum(("OuterRecord")."DecValue74") AS dec74_sum, - sum(("OuterRecord")."DecValue75") AS dec75_sum, - sum(("OuterRecord")."DecValue76") AS dec76_sum, - sum(("OuterRecord")."DecValue77") AS dec77_sum, - sum(("OuterRecord")."DecValue78") AS dec78_sum, - sum(("OuterRecord")."DecValue79") AS dec79_sum, - sum(("OuterRecord")."DecValue80") AS dec80_sum, - sum(("OuterRecord")."DecValue81") AS dec81_sum, - sum(("OuterRecord")."DecValue82") AS dec82_sum, - sum(("OuterRecord")."DecValue83") AS dec83_sum, - sum(("OuterRecord")."DecValue84") AS dec84_sum, - sum(("OuterRecord")."DecValue85") AS dec85_sum, - sum(("OuterRecord")."DecValue86") AS dec86_sum, - sum(("OuterRecord")."DecValue87") AS dec87_sum, - sum(("OuterRecord")."DecValue88") AS dec88_sum, - sum(("OuterRecord")."DecValue89") AS dec89_sum, - sum(("OuterRecord")."DecValue90") AS dec90_sum, - sum(("OuterRecord")."DecValue91") AS dec91_sum, - sum(("OuterRecord")."DecValue92") AS dec92_sum, - sum(("OuterRecord")."DecValue93") AS dec93_sum, - sum(("OuterRecord")."DecValue94") AS dec94_sum, - sum(("OuterRecord")."DecValue95") AS dec95_sum, - sum(("OuterRecord")."DecValue96") AS dec96_sum, - sum(("OuterRecord")."DecValue97") AS dec97_sum, - sum(("OuterRecord")."DecValue98") AS dec98_sum, - sum(("OuterRecord")."DecValue99") AS dec99_sum -FROM source_aggregationtest -GROUP BY ("Key1Unused"); diff --git a/test/bench/aggregations/generator/Dockerfile b/test/bench/aggregations/generator/Dockerfile deleted file mode 100644 index bee7c48185e93..0000000000000 --- a/test/bench/aggregations/generator/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -MZFROM kgen - -FROM python:3.9-slim - -COPY --from=0 /usr/local/bin/kgen /usr/local/bin - -RUN mkdir -p /usr/share/generator/distributions -COPY distributions /usr/share/generator/distributions - -COPY generate_data /usr/local/bin -ENTRYPOINT ["generate_data"] diff --git a/test/bench/aggregations/generator/distributions/benchmark/key-distribution.json b/test/bench/aggregations/generator/distributions/benchmark/key-distribution.json deleted file mode 100644 index 42d53cb7aeb7c..0000000000000 --- a/test/bench/aggregations/generator/distributions/benchmark/key-distribution.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "com.acme.avro.aggregationtestkey::Key1": [ - 0, - 9999 - ] -} diff --git a/test/bench/aggregations/generator/distributions/benchmark/value-distribution.json b/test/bench/aggregations/generator/distributions/benchmark/value-distribution.json deleted file mode 100644 index 8a0af05281aec..0000000000000 --- a/test/bench/aggregations/generator/distributions/benchmark/value-distribution.json +++ /dev/null @@ -1,406 +0,0 @@ -{ - "com.acme.avro.aggregationtest::Key1Unused": [ - 0, - 9999 - ], - "com.acme.avro.OuterRecord::DecValue00": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue01": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue02": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue03": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue04": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue05": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue06": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue07": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue08": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue09": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue10": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue11": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue12": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue13": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue14": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue15": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue16": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue17": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue18": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue19": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue20": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue21": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue22": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue23": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue24": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue25": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue26": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue27": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue28": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue29": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue30": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue31": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue32": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue33": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue34": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue35": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue36": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue37": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue38": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue39": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue40": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue41": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue42": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue43": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue44": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue45": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue46": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue47": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue48": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue49": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue50": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue51": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue52": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue53": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue54": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue55": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue56": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue57": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue58": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue59": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue60": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue61": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue62": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue63": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue64": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue65": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue66": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue67": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue68": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue69": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue70": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue71": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue72": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue73": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue74": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue75": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue76": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue77": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue78": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue79": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue80": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue81": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue82": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue83": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue84": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue85": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue86": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue87": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue88": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue89": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue90": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue91": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue92": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue93": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue94": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue95": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue96": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue97": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue98": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue99": [ - 10000, - 1000000000 - ] -} diff --git a/test/bench/aggregations/generator/distributions/key-schema.json b/test/bench/aggregations/generator/distributions/key-schema.json deleted file mode 100644 index 2fac5d8195cbe..0000000000000 --- a/test/bench/aggregations/generator/distributions/key-schema.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "aggregationtestkey", - "type": "record", - "namespace": "com.acme.avro", - "fields": [ - { - "name": "Key1", - "type": "long" - } - ] -} diff --git a/test/bench/aggregations/generator/distributions/medium/key-distribution.json b/test/bench/aggregations/generator/distributions/medium/key-distribution.json deleted file mode 100644 index 2dbf5c851e833..0000000000000 --- a/test/bench/aggregations/generator/distributions/medium/key-distribution.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "com.acme.avro.aggregationtestkey::Key1": [ - 0, - 999 - ] -} diff --git a/test/bench/aggregations/generator/distributions/medium/value-distribution.json b/test/bench/aggregations/generator/distributions/medium/value-distribution.json deleted file mode 100644 index 8a0af05281aec..0000000000000 --- a/test/bench/aggregations/generator/distributions/medium/value-distribution.json +++ /dev/null @@ -1,406 +0,0 @@ -{ - "com.acme.avro.aggregationtest::Key1Unused": [ - 0, - 9999 - ], - "com.acme.avro.OuterRecord::DecValue00": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue01": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue02": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue03": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue04": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue05": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue06": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue07": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue08": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue09": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue10": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue11": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue12": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue13": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue14": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue15": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue16": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue17": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue18": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue19": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue20": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue21": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue22": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue23": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue24": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue25": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue26": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue27": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue28": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue29": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue30": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue31": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue32": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue33": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue34": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue35": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue36": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue37": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue38": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue39": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue40": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue41": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue42": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue43": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue44": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue45": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue46": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue47": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue48": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue49": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue50": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue51": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue52": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue53": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue54": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue55": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue56": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue57": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue58": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue59": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue60": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue61": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue62": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue63": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue64": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue65": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue66": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue67": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue68": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue69": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue70": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue71": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue72": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue73": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue74": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue75": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue76": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue77": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue78": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue79": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue80": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue81": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue82": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue83": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue84": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue85": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue86": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue87": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue88": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue89": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue90": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue91": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue92": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue93": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue94": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue95": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue96": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue97": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue98": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue99": [ - 10000, - 1000000000 - ] -} diff --git a/test/bench/aggregations/generator/distributions/small/key-distribution.json b/test/bench/aggregations/generator/distributions/small/key-distribution.json deleted file mode 100644 index 0cac4b6dc9d5e..0000000000000 --- a/test/bench/aggregations/generator/distributions/small/key-distribution.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "com.acme.avro.aggregationtestkey::Key1": [ - 0, - 9 - ] -} diff --git a/test/bench/aggregations/generator/distributions/small/value-distribution.json b/test/bench/aggregations/generator/distributions/small/value-distribution.json deleted file mode 100644 index 8a0af05281aec..0000000000000 --- a/test/bench/aggregations/generator/distributions/small/value-distribution.json +++ /dev/null @@ -1,406 +0,0 @@ -{ - "com.acme.avro.aggregationtest::Key1Unused": [ - 0, - 9999 - ], - "com.acme.avro.OuterRecord::DecValue00": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue01": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue02": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue03": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue04": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue05": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue06": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue07": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue08": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue09": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue10": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue11": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue12": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue13": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue14": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue15": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue16": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue17": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue18": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue19": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue20": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue21": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue22": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue23": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue24": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue25": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue26": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue27": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue28": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue29": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue30": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue31": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue32": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue33": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue34": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue35": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue36": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue37": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue38": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue39": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue40": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue41": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue42": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue43": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue44": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue45": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue46": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue47": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue48": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue49": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue50": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue51": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue52": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue53": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue54": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue55": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue56": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue57": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue58": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue59": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue60": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue61": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue62": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue63": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue64": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue65": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue66": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue67": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue68": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue69": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue70": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue71": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue72": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue73": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue74": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue75": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue76": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue77": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue78": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue79": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue80": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue81": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue82": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue83": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue84": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue85": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue86": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue87": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue88": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue89": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue90": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue91": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue92": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue93": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue94": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue95": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue96": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue97": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue98": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue99": [ - 10000, - 1000000000 - ] -} diff --git a/test/bench/aggregations/generator/distributions/value-schema.json b/test/bench/aggregations/generator/distributions/value-schema.json deleted file mode 100644 index b437df2a387d1..0000000000000 --- a/test/bench/aggregations/generator/distributions/value-schema.json +++ /dev/null @@ -1,920 +0,0 @@ -{ - "name": "aggregationtest", - "type": "record", - "namespace": "com.acme.avro", - "fields": [ - { - "name": "Key1Unused", - "type": "long" - }, - { - "name": "OuterRecord", - "type": { - "name": "OuterRecord", - "type": "record", - "fields": [ - { - "name": "DecValue00", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue01", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue02", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue03", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue04", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue05", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue06", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue07", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue08", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue09", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue10", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue11", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue12", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue13", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue14", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue15", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue16", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue17", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue18", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue19", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue20", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue21", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue22", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue23", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue24", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue25", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue26", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue27", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue28", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue29", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue30", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue31", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue32", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue33", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue34", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue35", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue36", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue37", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue38", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue39", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue40", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue41", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue42", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue43", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue44", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue45", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue46", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue47", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue48", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue49", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue50", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue51", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue52", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue53", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue54", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue55", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue56", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue57", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue58", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue59", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue60", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue61", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue62", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue63", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue64", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue65", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue66", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue67", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue68", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue69", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue70", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue71", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue72", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue73", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue74", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue75", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue76", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue77", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue78", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue79", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue80", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue81", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue82", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue83", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue84", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue85", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue86", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue87", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue88", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue89", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue90", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue91", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue92", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue93", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue94", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue95", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue96", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue97", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue98", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue99", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - } - ] - } - } - ] -} diff --git a/test/bench/aggregations/generator/generate_data b/test/bench/aggregations/generator/generate_data deleted file mode 100755 index 29b1b766e4da6..0000000000000 --- a/test/bench/aggregations/generator/generate_data +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -import argparse -import multiprocessing -import pathlib -import subprocess -import sys - -DISTRIBUTIONS = "/usr/share/generator/distributions" - - -def run(args: argparse.Namespace) -> None: - """Run the generator, inserting args.num_records number of messages.""" - - records_per_process = int(args.num_records / args.parallelism) - - key_schema = pathlib.Path( - DISTRIBUTIONS, "key-schema.json").read_text().strip() - value_schema = pathlib.Path( - DISTRIBUTIONS, "value-schema.json").read_text().strip() - - distribution_dir = pathlib.Path(DISTRIBUTIONS, args.distribution) - key_distribution = ( - pathlib.Path(distribution_dir, - "key-distribution.json").read_text().strip() - ) - value_distribution = ( - pathlib.Path(distribution_dir, - "value-distribution.json").read_text().strip() - ) - - kafka_gen = [ - "/usr/local/bin/kgen", - "--quiet", - "--bootstrap-server", - "kafka:9092", - "--schema-registry-url", - "http://schema-registry:8081", - "--num-records", - str(records_per_process), - "--topic", - "aggregationtest", - "--keys", - "avro", - "--values", - "avro", - "--avro-schema", - value_schema, - "--avro-distribution", - value_distribution, - "--avro-key-schema", - key_schema, - "--avro-key-distribution", - key_distribution, - ] - - print( - f"Spawning {args.parallelism} generator processes, writing {records_per_process} messages each" - ) - procs = [subprocess.Popen(kafka_gen) for _ in range(0, args.parallelism)] - exit_code = 0 - for (i, p) in enumerate(procs): - p.wait() - print( - f"{i}/{args.parallelism} processes finished: pid={p.pid} returncode={p.returncode}" - ) - if p.returncode != 0: - exit_code = p.returncode - - sys.exit(exit_code) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument( - "-d", - "--distribution", - type=str, - default="benchmark", - help="Name of the distribution to use for generating random records", - choices=[p.name for p in pathlib.Path(DISTRIBUTIONS).iterdir()], - ) - - parser.add_argument( - "-n", - "--num-records", - type=int, - default=400000000, - help="Total number of messages to generate", - ) - parser.add_argument( - "-p", - "--parallelism", - type=int, - default=multiprocessing.cpu_count(), - help="Number of processes to spawn", - ) - - args = parser.parse_args() - if args.num_records % args.parallelism != 0: - print("ERROR: Number of records must divide evenly by number of processes") - sys.exit(1) - - run(args) diff --git a/test/bench/aggregations/generator/mzbuild.yml b/test/bench/aggregations/generator/mzbuild.yml deleted file mode 100644 index 403a141b884d4..0000000000000 --- a/test/bench/aggregations/generator/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: aggregation-data-generator -publish: false diff --git a/test/bench/aggregations/mzcompose b/test/bench/aggregations/mzcompose deleted file mode 100755 index 76f6976c3b4a5..0000000000000 --- a/test/bench/aggregations/mzcompose +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. -# -# mzcompose — runs Docker Compose with Materialize customizations. - -exec "$(dirname "$0")/../../../bin/mzcompose" "$@" diff --git a/test/bench/aggregations/mzcompose.yml b/test/bench/aggregations/mzcompose.yml deleted file mode 100644 index b2eaa7c50f603..0000000000000 --- a/test/bench/aggregations/mzcompose.yml +++ /dev/null @@ -1,290 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - - -# Map from host-port:internal port -# -# This mostly just shows all the ports that are available to the host system, if you want -# to change these you must restart the docker-compose cluster. -x-port-mappings: - - &kafka-internal ${KAFKA_PORT:-9092} - - &kafka-external 9093:9093 - - &schema-registry ${SR_PORT:-8081:8081} - - &materialized ${MZ_PORT:-6875} - - &grafana 3000:3000 - - &prometheus 9090 - - &perf-metrics ${PERF_METRICS_PORT:-8675} - - &perf-dash-web ${PERF_DASH_PORT:-8875:8875} - -version: '3.7' -services: - create-views: - mzbuild: aggregations-bench-create-views - perf-dash-metrics: - mzbuild: materialized - ports: - - *perf-metrics - command: - - --disable-telemetry - - --listen-addr=0.0.0.0:${PERF_METRICS_PORT:-8675} - - --logical-compaction-window=1ms - environment: - - MZ_DEV=1 - perf-dash-scraper: - mzbuild: perf-dash-scraper - command: scrape - perf-dash-create-views: - mzbuild: perf-dash-create-views - perf-dash-web: - mzbuild: perf-dash-web - ports: - - *perf-dash-web - materialized: - mzbuild: materialized - ports: - - *materialized - command: - - --workers=${MZ_WORKERS:-16} - - --logical-compaction-window=1ms - # We want this to eventually count up to the size of the largest batch in - # an arrangement. This number represents a tradeoff between proactive - # merging (which takes time) and low latency. - # - # 1000 was chosen by fair dice roll. - - --differential-idle-merge-effort=1000 - - --timely-progress-mode=${MZ_TIMELY_PROGRESS_MODE:-demand} - - --disable-telemetry - environment: - # You can, for example, add `pgwire=trace` or change `info` to `debug` to - # get more verbose logs. - - MZ_LOG_FILTER=pgwire=debug,info - - MZ_DEV=1 - zookeeper: - image: confluentinc/cp-zookeeper:5.5.4 - environment: - - ZOOKEEPER_CLIENT_PORT=2181 - kafka: - image: confluentinc/cp-enterprise-kafka:5.5.4 - ports: - - *kafka-internal - - *kafka-external - depends_on: [zookeeper] - environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT - - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092,EXTERNAL://0.0.0.0:9093 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://${KAFKA_HOST:-kafka}:9093 - - KAFKA_METRIC_REPORTERS=io.confluent.metrics.reporter.ConfluentMetricsReporter - - KAFKA_BROKER_ID=1 - - KAFKA_LOG_RETENTION_HOURS=-1 - - KAFKA_NUM_PARTITIONS=30 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS=kafka:9092 - - CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS=1 - - CONFLUENT_METRICS_REPORTER_TOPIC_CREATE=false - - KAFKA_JMX_PORT=9991 - schema-registry: - image: confluentinc/cp-schema-registry:5.5.4 - ports: - - *schema-registry - environment: - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 - - SCHEMA_REGISTRY_HOST_NAME=schema-registry - - SCHEMA_REGISTRY_LISTENERS=http://schema-registry:8081,http://localhost:8081 - depends_on: [zookeeper, kafka] - generator: - mzbuild: aggregation-data-generator - metric-verifier: - mzbuild: metric-verifier - environment: - - MZ_WORKERS=${MZ_WORKERS:-0} - - MZBENCH_GIT_REF=${MZBENCH_GIT_REF:-None} - - MZBENCH_ID=${MZBENCH_ID:-0} - # All monitoring - dashboard: - mzbuild: dashboard - propagate_uid_gid: true - environment: - - 'MATERIALIZED_URL=materialized:6875' - ports: - - *grafana - - *prometheus - volumes: - # ensure that data doesn't get lost across restarts - # data will be lost if you remove docker volumes (using nuke, for example) - - prometheus:/prometheus - # specialized configurations - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - -volumes: - prometheus: - -mzworkflows: - - start-services: - steps: - - step: start-services - services: [dashboard] - - step: start-services - services: [kafka, schema-registry, perf-dash-metrics] - - step: wait-for-tcp - host: kafka - port: 9092 - - step: wait-for-tcp - host: schema-registry - port: 8081 - - step: run - service: perf-dash-scraper - command: >- - create-topics - - step: run - service: metric-verifier - command: >- - create-topics - - step: run - service: perf-dash-create-views - - step: start-services - services: [perf-dash-web] - - setup-benchmark: - env: - AGGREGATION_GENERATOR_DISTRIBUTION: ${AGGREGATION_GENERATOR_DISTRIBUTION:-benchmark} - AGGREGATION_GENERATOR_NUM_RECORDS: ${AGGREGATION_GENERATOR_NUM_RECORDS:-175000000} - AGGREGATION_GENERATOR_PARALLELISM: ${AGGREGATION_GENERATOR_PARALLELISM:-40} - steps: - - step: workflow - workflow: start-services - - step: run - service: generator - command: >- - --parallelism=${AGGREGATION_GENERATOR_PARALLELISM} - --num-records=${AGGREGATION_GENERATOR_NUM_RECORDS} - --distribution=${AGGREGATION_GENERATOR_DISTRIBUTION} - - run-benchmark: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - METRIC_VERIFIER_TIMEOUT: ${METRIC_VERIFIER_TIMEOUT:-3600} - AGGREGATION_GENERATOR_NUM_RECORDS: ${AGGREGATION_GENERATOR_NUM_RECORDS:-175000000} - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: metric-verifier - daemon: true - command: >- - scrape - --timeout-seconds=${METRIC_VERIFIER_TIMEOUT} - "sum(mz_messages_ingested)" - ${AGGREGATION_GENERATOR_NUM_RECORDS} - - step: run - service: create-views - - step: wait - service: metric-verifier - expected_return_code: 0 - print_logs: true - - # The benchmark that we run in the cloud - benchmark-large: - steps: - - step: workflow - workflow: setup-benchmark-large - - step: workflow - workflow: run-benchmark-large - - setup-benchmark-large: - env: - AGGREGATION_GENERATOR_DISTRIBUTION: benchmark - AGGREGATION_GENERATOR_NUM_RECORDS: 7500000 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-large: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - AGGREGATION_GENERATOR_NUM_RECORDS: 7500000 - steps: - - step: workflow - workflow: run-benchmark - - # The benchmark that developers can run on their laptop - benchmark-medium: - steps: - - step: workflow - workflow: setup-benchmark-medium - - step: workflow - workflow: run-benchmark-medium - - setup-benchmark-medium: - env: - AGGREGATION_GENERATOR_DISTRIBUTION: medium - AGGREGATION_GENERATOR_NUM_RECORDS: 500000 - AGGREGATION_GENERATOR_PARALLELISM: 8 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-medium: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 900 - AGGREGATION_GENERATOR_NUM_RECORDS: 500000 - steps: - - step: workflow - workflow: run-benchmark - - # The smoketest benchmark that we run in CI - ci: - steps: - - step: workflow - workflow: setup-benchmark-ci - - step: workflow - workflow: run-benchmark-ci - - benchmark-ci: - steps: - - step: worfklow - workflow: ci - - setup-benchmark-ci: - env: - AGGREGATION_GENERATOR_DISTRIBUTION: small - AGGREGATION_GENERATOR_NUM_RECORDS: 100000 - AGGREGATION_GENERATOR_PARALLELISM: 4 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-ci: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 30 - AGGREGATION_GENERATOR_NUM_RECORDS: 100000 - steps: - - step: workflow - workflow: run-benchmark - - run-generator: - steps: - - step: run - service: generator - - rerun-benchmark: - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: create-views diff --git a/test/bench/aggregations/prometheus/prometheus.yml b/test/bench/aggregations/prometheus/prometheus.yml deleted file mode 100644 index 6950d2120bd27..0000000000000 --- a/test/bench/aggregations/prometheus/prometheus.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: materialized - scrape_interval: 1s - static_configs: - - targets: - - 'materialized:6875' diff --git a/test/bench/avro-insert/README.md b/test/bench/avro-insert/README.md deleted file mode 100644 index fee7efa8afbcb..0000000000000 --- a/test/bench/avro-insert/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# Avro Insert Benchmark - -This benchmark tests our ability to ingest Avro messages without any envelope type. In it's -current configuration, it assumes that Kafka topics should have 30 partitions and that you want 16 -worker threads. - -This benchmark is largely identical to the `avro-upsert` benchmark and was originally developed to -compare the relative performance of the `upsert` operator. - -## Running the Avro Insert Test - -To run the spec benchmark, please ensure that your machine has 16 cores and at least 96GB of -memory. Then run the following command: - - ./mzcompose run benchmark - -If you're running on a smaller machine, such as a laptop with 8 cores / 32GB of memory, you can -run the version of this benchmark designed for smaller machines: - - ./mzcompose run benchmark-medium - -You can also use [mzbench](../../../doc/developer/mzbench.md) to run comparative benchmarks on -different versions. See the `mzbench` documentation for more details but here is an example -invocation (run from the repository root): - - ./bin/mzbench -w --size medium avro-insert - -## Looking at Performance in Grafana - -Each benchmark will output something like the following: - -```sh -SUCCESS! seconds_taken=766, rows_per_sec=522193 -Grafana URL: http://localhost:3000/d/materialize-overview/materialize-overview?from=1612572459000&to=1612573285000&tz=UTC -``` - -The Grafana link will display performance metrics for materialize during the entirety of the -benchmark run. diff --git a/test/bench/avro-insert/create_views/Dockerfile b/test/bench/avro-insert/create_views/Dockerfile deleted file mode 100644 index 41007ae2b1144..0000000000000 --- a/test/bench/avro-insert/create_views/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -FROM postgres - -RUN mkdir /data -COPY views /data - -COPY create_views /usr/local/bin -ENTRYPOINT ["/usr/local/bin/create_views"] diff --git a/test/bench/avro-insert/create_views/create_views b/test/bench/avro-insert/create_views/create_views deleted file mode 100755 index e65f2437c98fe..0000000000000 --- a/test/bench/avro-insert/create_views/create_views +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -set -euo pipefail - -view_file="/data/${1:-insert_views.sql}" - -psql -h materialized -p 6875 -U materialize -f "${view_file}" materialize diff --git a/test/bench/avro-insert/create_views/mzbuild.yml b/test/bench/avro-insert/create_views/mzbuild.yml deleted file mode 100644 index 9dac8f160fbe4..0000000000000 --- a/test/bench/avro-insert/create_views/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: avro-insert-create-views -publish: false diff --git a/test/bench/avro-insert/create_views/views/insert_views.sql b/test/bench/avro-insert/create_views/views/insert_views.sql deleted file mode 100644 index a0bdc64eda533..0000000000000 --- a/test/bench/avro-insert/create_views/views/insert_views.sql +++ /dev/null @@ -1,26 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE source_insertavrotest -FROM KAFKA BROKER 'kafka:9092' -TOPIC 'insertavrotest' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE VIEW insertavrotest AS SELECT - "Key1Unused" AS Key1Unused, -- retaining AS long - "Key2Unused" AS Key2Unused, -- retaining AS long - (("OuterRecord")."Record1"."InnerRecord1"."Point")::float AS "Point1", -- converting long to float - (("OuterRecord")."Record1"."InnerRecord2"."Point")::float AS "Point2", -- converting long to float - (("OuterRecord")."Record2"."InnerRecord3"."Point")::float AS "Point3", -- converting long to float - (("OuterRecord")."Record2"."InnerRecord4"."Point")::float AS "Point4" -- converting long to float -FROM source_insertavrotest -WHERE "Key1Unused" = 0; - -CREATE INDEX insertavrotest_index ON insertavrotest(Key1Unused, Key2Unused); diff --git a/test/bench/avro-insert/mzcompose b/test/bench/avro-insert/mzcompose deleted file mode 100755 index 76f6976c3b4a5..0000000000000 --- a/test/bench/avro-insert/mzcompose +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. -# -# mzcompose — runs Docker Compose with Materialize customizations. - -exec "$(dirname "$0")/../../../bin/mzcompose" "$@" diff --git a/test/bench/avro-insert/mzcompose.yml b/test/bench/avro-insert/mzcompose.yml deleted file mode 100644 index 6972f1fc3d57f..0000000000000 --- a/test/bench/avro-insert/mzcompose.yml +++ /dev/null @@ -1,294 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - - -# Map from host-port:internal port -# -# This mostly just shows all the ports that are available to the host system, if you want -# to change these you must restart the docker-compose cluster. -x-port-mappings: - - &kafka-internal ${KAFKA_PORT:-9092} - - &kafka-external 9093:9093 - - &schema-registry ${SR_PORT:-8081:8081} - - &materialized ${MZ_PORT:-6875} - - &grafana 3000:3000 - - &prometheus 9090 - - &perf-metrics ${PERF_METRICS_PORT:-8675} - - &perf-dash-web ${PERF_DASH_PORT:-8875:8875} - -version: '3.7' -services: - create-views: - mzbuild: avro-insert-create-views - perf-dash-metrics: - mzbuild: materialized - ports: - - *perf-metrics - command: - - --disable-telemetry - - --listen-addr=0.0.0.0:${PERF_METRICS_PORT:-8675} - - --logical-compaction-window=1ms - environment: - - MZ_DEV=1 - perf-dash-scraper: - mzbuild: perf-dash-scraper - command: scrape - perf-dash-create-views: - mzbuild: perf-dash-create-views - perf-dash-web: - mzbuild: perf-dash-web - ports: - - *perf-dash-web - materialized: - mzbuild: materialized - ports: - - *materialized - command: - - --workers=${MZ_WORKERS:-16} - - --logical-compaction-window=1ms - # We want this to eventually count up to the size of the largest batch in - # an arrangement. This number represents a tradeoff between proactive - # merging (which takes time) and low latency. - # - # 1000 was chosen by fair dice roll. - - --differential-idle-merge-effort=1000 - - --timely-progress-mode=${MZ_TIMELY_PROGRESS_MODE:-demand} - - --disable-telemetry - environment: - # You can, for example, add `pgwire=trace` or change `info` to `debug` to - # get more verbose logs. - - MZ_LOG_FILTER=pgwire=debug,info - - MZ_DEV=1 - zookeeper: - image: confluentinc/cp-zookeeper:5.5.4 - environment: - - ZOOKEEPER_CLIENT_PORT=2181 - kafka: - image: confluentinc/cp-enterprise-kafka:5.5.4 - ports: - - *kafka-internal - - *kafka-external - depends_on: [zookeeper] - environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT - - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092,EXTERNAL://0.0.0.0:9093 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://${KAFKA_HOST:-kafka}:9093 - - KAFKA_METRIC_REPORTERS=io.confluent.metrics.reporter.ConfluentMetricsReporter - - KAFKA_BROKER_ID=1 - - KAFKA_LOG_RETENTION_HOURS=-1 - - KAFKA_NUM_PARTITIONS=30 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS=kafka:9092 - - CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS=1 - # To avoid race condition with control-center - - CONFLUENT_METRICS_REPORTER_TOPIC_CREATE=false - - KAFKA_JMX_PORT=9991 - schema-registry: - image: confluentinc/cp-schema-registry:5.5.4 - ports: - - *schema-registry - environment: - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 - - SCHEMA_REGISTRY_HOST_NAME=schema-registry - - SCHEMA_REGISTRY_LISTENERS=http://schema-registry:8081,http://localhost:8081 - depends_on: [zookeeper, kafka] - kafka-avro-generator: - mzbuild: kafka-avro-generator - metric-verifier: - mzbuild: metric-verifier - environment: - - MZ_WORKERS=${MZ_WORKERS:-0} - - MZBENCH_GIT_REF=${MZBENCH_GIT_REF:-None} - - MZBENCH_ID=${MZBENCH_ID:-0} - # All monitoring - dashboard: - mzbuild: dashboard - propagate_uid_gid: true - environment: - - 'MATERIALIZED_URL=materialized:6875' - ports: - - *grafana - - *prometheus - volumes: - # ensure that data doesn't get lost across restarts - # data will be lost if you remove docker volumes (using nuke, for example) - - prometheus:/prometheus - - grafana:/var/lib/grafana - # specialized configurations - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - -volumes: - grafana: - prometheus: - -mzworkflows: - - start-services: - steps: - - step: start-services - services: [dashboard] - - step: start-services - services: [kafka, schema-registry, perf-dash-metrics] - - step: wait-for-tcp - host: kafka - port: 9092 - - step: wait-for-tcp - host: schema-registry - port: 8081 - - step: run - service: perf-dash-scraper - command: >- - create-topics - - step: run - service: metric-verifier - command: >- - create-topics - - step: run - service: perf-dash-create-views - - step: start-services - services: [perf-dash-web] - - setup-benchmark: - env: - AVRO_GENERATOR_DISTRIBUTION: ${AVRO_GENERATOR_DISTRIBUTION:-benchmark} - AVRO_GENERATOR_NUM_RECORDS: ${AVRO_GENERATOR_NUM_RECORDS:-400000000} - AVRO_GENERATOR_PARALLELISM: ${AVRO_GENERATOR_PARALLELISM:-40} - steps: - - step: workflow - workflow: start-services - - step: run - service: kafka-avro-generator - command: >- - --parallelism=${AVRO_GENERATOR_PARALLELISM} - --num-records=${AVRO_GENERATOR_NUM_RECORDS} - --distribution=${AVRO_GENERATOR_DISTRIBUTION} - --topic=insertavrotest - - run-benchmark: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - METRIC_VERIFIER_TIMEOUT: ${METRIC_VERIFIER_TIMEOUT:-3600} - AVRO_GENERATOR_NUM_RECORDS: ${AVRO_GENERATOR_NUM_RECORDS:-400000000} - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: metric-verifier - daemon: true - command: >- - scrape - --timeout-seconds=${METRIC_VERIFIER_TIMEOUT} - "sum(mz_messages_ingested)" - ${AVRO_GENERATOR_NUM_RECORDS} - - step: run - service: create-views - - step: wait - service: metric-verifier - expected_return_code: 0 - print_logs: true - - # The benchmark that we run in the cloud - benchmark-large: - steps: - - step: workflow - workflow: setup-benchmark-large - - step: workflow - workflow: run-benchmark-large - - setup-benchmark-large: - env: - AVRO_GENERATOR_DISTRIBUTION: benchmark - AVRO_GENERATOR_NUM_RECORDS: 400000000 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-large: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - AVRO_GENERATOR_NUM_RECORDS: 400000000 - steps: - - step: workflow - workflow: run-benchmark - - # The benchmark that developers can run on their laptop - benchmark-medium: - steps: - - step: workflow - workflow: setup-benchmark-medium - - step: workflow - workflow: run-benchmark-medium - - setup-benchmark-medium: - env: - AVRO_GENERATOR_DISTRIBUTION: medium - AVRO_GENERATOR_NUM_RECORDS: 20000000 - AVRO_GENERATOR_PARALLELISM: 8 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-medium: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 900 - AVRO_GENERATOR_NUM_RECORDS: 20000000 - steps: - - step: workflow - workflow: run-benchmark - - # The smoketest benchmark that we run in CI - ci: - steps: - - step: workflow - workflow: setup-benchmark-ci - - step: workflow - workflow: run-benchmark-ci - - benchmark-ci: - steps: - - step: worfklow - workflow: ci - - setup-benchmark-ci: - env: - AVRO_GENERATOR_DISTRIBUTION: small - AVRO_GENERATOR_NUM_RECORDS: 100000 - AVRO_GENERATOR_PARALLELISM: 4 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-ci: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 30 - AVRO_GENERATOR_NUM_RECORDS: 100000 - steps: - - step: workflow - workflow: run-benchmark - - run-generator: - steps: - - step: run - service: kafka-avro-generator - - rerun-benchmark: - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: create-views diff --git a/test/bench/avro-insert/prometheus/prometheus.yml b/test/bench/avro-insert/prometheus/prometheus.yml deleted file mode 100644 index 6950d2120bd27..0000000000000 --- a/test/bench/avro-insert/prometheus/prometheus.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: materialized - scrape_interval: 1s - static_configs: - - targets: - - 'materialized:6875' diff --git a/test/bench/avro-upsert/README.md b/test/bench/avro-upsert/README.md deleted file mode 100644 index d38acd97d6001..0000000000000 --- a/test/bench/avro-upsert/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Avro Upsert Benchmark - -This benchmark tests our ability to ingest Avro Upsert messages. In it's current configuration, it -assumes that Kafka topics should have 30 partitions and that you want 16 worker threads. - -## Running the Avro Upsert Test - -To run the spec benchmark, please ensure that your machine has 16 cores and at least 96GB of -memory. Then run the following command: - - ./mzcompose run benchmark - -If you're running on a smaller machine, such as a laptop with 8 cores / 32GB of memory, you can -run the version of this benchmark designed for smaller machines: - - ./mzcompose run benchmark-medium - -You can also use [mzbench](../../../doc/developer/mzbench.md) to run comparative benchmarks on -different versions. See the `mzbench` documentation for more details but here is an example -invocation (run from the repository root): - - ./bin/mzbench -w --size medium avro-upsert - -## Looking at Performance - -Each benchmark will output something like the following: - -```sh -SUCCESS! seconds_taken=766, rows_per_sec=522193 -Grafana URL: http://localhost:3000/d/materialize-overview/materialize-overview?from=1612572459000&to=1612573285000&tz=UTC -``` - -The Grafana link will display performance metrics for materialize during the entirety of the -benchmark run. diff --git a/test/bench/avro-upsert/create_views/Dockerfile b/test/bench/avro-upsert/create_views/Dockerfile deleted file mode 100644 index 41007ae2b1144..0000000000000 --- a/test/bench/avro-upsert/create_views/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -FROM postgres - -RUN mkdir /data -COPY views /data - -COPY create_views /usr/local/bin -ENTRYPOINT ["/usr/local/bin/create_views"] diff --git a/test/bench/avro-upsert/create_views/create_views b/test/bench/avro-upsert/create_views/create_views deleted file mode 100755 index b868f7c2eb866..0000000000000 --- a/test/bench/avro-upsert/create_views/create_views +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -set -euo pipefail - -view_file="/data/${1:-upsert_views.sql}" - -psql -v ON_ERROR_STOP=1 -h materialized -p 6875 -U materialize -f "${view_file}" materialize diff --git a/test/bench/avro-upsert/create_views/mzbuild.yml b/test/bench/avro-upsert/create_views/mzbuild.yml deleted file mode 100644 index e8efc2ce2f2fa..0000000000000 --- a/test/bench/avro-upsert/create_views/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: avro-upsert-create-views -publish: false diff --git a/test/bench/avro-upsert/create_views/views/insert_views.sql b/test/bench/avro-upsert/create_views/views/insert_views.sql deleted file mode 100644 index 31bba5a9504ea..0000000000000 --- a/test/bench/avro-upsert/create_views/views/insert_views.sql +++ /dev/null @@ -1,26 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE source_insertavrotest -FROM KAFKA BROKER 'kafka:9092' -TOPIC 'upsertavrotest' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE VIEW insertavrotest AS SELECT - "Key1Unused" AS Key1Unused, -- retaining AS long - "Key2Unused" AS Key2Unused, -- retaining AS long - (("OuterRecord")."Record1"."InnerRecord1"."Point")::float AS "Point1", -- converting long to float - (("OuterRecord")."Record1"."InnerRecord2"."Point")::float AS "Point2", -- converting long to float - (("OuterRecord")."Record2"."InnerRecord3"."Point")::float AS "Point3", -- converting long to float - (("OuterRecord")."Record2"."InnerRecord4"."Point")::float AS "Point4" -- converting long to float -FROM source_insertavrotest -WHERE "Key1Unused" = 0; - -CREATE INDEX insertavrotest_index ON insertavrotest(Key1Unused, Key2Unused); diff --git a/test/bench/avro-upsert/create_views/views/upsert_views.sql b/test/bench/avro-upsert/create_views/views/upsert_views.sql deleted file mode 100644 index 7575984fd857c..0000000000000 --- a/test/bench/avro-upsert/create_views/views/upsert_views.sql +++ /dev/null @@ -1,26 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE source_upsertavrotest -FROM KAFKA BROKER 'kafka:9092' -TOPIC 'upsertavrotest' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE UPSERT -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081'; - -CREATE VIEW upsertavrotest AS SELECT - "Key1" AS Key1, -- retaining AS long - "Key2" AS Key2, -- retaining AS long - (("OuterRecord")."Record1"."InnerRecord1"."Point")::float AS "Point1", -- converting long to float - (("OuterRecord")."Record1"."InnerRecord2"."Point")::float AS "Point2", -- converting long to float - (("OuterRecord")."Record2"."InnerRecord3"."Point")::float AS "Point3", -- converting long to float - (("OuterRecord")."Record2"."InnerRecord4"."Point")::float AS "Point4" -- converting long to float -FROM source_upsertavrotest; - -CREATE INDEX upsertavrotest_index ON upsertavrotest(Key1, Key2); diff --git a/test/bench/avro-upsert/mzcompose b/test/bench/avro-upsert/mzcompose deleted file mode 100755 index 76f6976c3b4a5..0000000000000 --- a/test/bench/avro-upsert/mzcompose +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. -# -# mzcompose — runs Docker Compose with Materialize customizations. - -exec "$(dirname "$0")/../../../bin/mzcompose" "$@" diff --git a/test/bench/avro-upsert/mzcompose.yml b/test/bench/avro-upsert/mzcompose.yml deleted file mode 100644 index 6a5b71400171c..0000000000000 --- a/test/bench/avro-upsert/mzcompose.yml +++ /dev/null @@ -1,291 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - - -# Map from host-port:internal port -# -# This mostly just shows all the ports that are available to the host system, if you want -# to change these you must restart the docker-compose cluster. -x-port-mappings: - - &kafka-internal ${KAFKA_PORT:-9092} - - &kafka-external 9093:9093 - - &schema-registry ${SR_PORT:-8081:8081} - - &materialized ${MZ_PORT:-6875} - - &grafana 3000:3000 - - &prometheus 9090 - - &perf-metrics ${PERF_METRICS_PORT:-8675} - - &perf-dash-web ${PERF_DASH_PORT:-8875:8875} - -version: '3.7' -services: - create-views: - mzbuild: avro-upsert-create-views - perf-dash-metrics: - mzbuild: materialized - ports: - - *perf-metrics - command: - - --disable-telemetry - - --listen-addr=0.0.0.0:${PERF_METRICS_PORT:-8675} - - --logical-compaction-window=1ms - environment: - - MZ_DEV=1 - perf-dash-scraper: - mzbuild: perf-dash-scraper - command: scrape - perf-dash-create-views: - mzbuild: perf-dash-create-views - perf-dash-web: - mzbuild: perf-dash-web - ports: - - *perf-dash-web - materialized: - mzbuild: materialized - ports: - - *materialized - command: - - --workers=${MZ_WORKERS:-16} - - --logical-compaction-window=1ms - # We want this to eventually count up to the size of the largest batch in - # an arrangement. This number represents a tradeoff between proactive - # merging (which takes time) and low latency. - # - # 1000 was chosen by fair dice roll. - - --differential-idle-merge-effort=1000 - - --timely-progress-mode=${MZ_TIMELY_PROGRESS_MODE:-demand} - - --disable-telemetry - environment: - # You can, for example, add `pgwire=trace` or change `info` to `debug` to - # get more verbose logs. - - MZ_LOG_FILTER=pgwire=debug,info - - MZ_DEV=1 - zookeeper: - image: confluentinc/cp-zookeeper:5.5.4 - environment: - - ZOOKEEPER_CLIENT_PORT=2181 - kafka: - image: confluentinc/cp-enterprise-kafka:5.5.4 - ports: - - *kafka-internal - - *kafka-external - depends_on: [zookeeper] - environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT - - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092,EXTERNAL://0.0.0.0:9093 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://${KAFKA_HOST:-kafka}:9093 - - KAFKA_METRIC_REPORTERS=io.confluent.metrics.reporter.ConfluentMetricsReporter - - KAFKA_BROKER_ID=1 - - KAFKA_LOG_RETENTION_HOURS=-1 - - KAFKA_NUM_PARTITIONS=30 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS=kafka:9092 - - CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS=1 - - CONFLUENT_METRICS_REPORTER_TOPIC_CREATE=false - - KAFKA_JMX_PORT=9991 - schema-registry: - image: confluentinc/cp-schema-registry:5.5.4 - ports: - - *schema-registry - environment: - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 - - SCHEMA_REGISTRY_HOST_NAME=schema-registry - - SCHEMA_REGISTRY_LISTENERS=http://schema-registry:8081,http://localhost:8081 - depends_on: [zookeeper, kafka] - kafka-avro-generator: - mzbuild: kafka-avro-generator - metric-verifier: - mzbuild: metric-verifier - environment: - - MZ_WORKERS=${MZ_WORKERS:-0} - - MZBENCH_GIT_REF=${MZBENCH_GIT_REF:-None} - - MZBENCH_ID=${MZBENCH_ID:-0} - # All monitoring - dashboard: - mzbuild: dashboard - propagate_uid_gid: true - environment: - - 'MATERIALIZED_URL=materialized:6875' - ports: - - *grafana - - *prometheus - volumes: - # ensure that data doesn't get lost across restarts - # data will be lost if you remove docker volumes (using nuke, for example) - - prometheus:/prometheus - # specialized configurations - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - -volumes: - prometheus: - -mzworkflows: - - start-services: - steps: - - step: start-services - services: [dashboard] - - step: start-services - services: [kafka, schema-registry, perf-dash-metrics] - - step: wait-for-tcp - host: kafka - port: 9092 - - step: wait-for-tcp - host: schema-registry - port: 8081 - - step: run - service: perf-dash-scraper - command: >- - create-topics - - step: run - service: metric-verifier - command: >- - create-topics - - step: run - service: perf-dash-create-views - - step: start-services - services: [perf-dash-web] - - setup-benchmark: - env: - UPSERT_GENERATOR_DISTRIBUTION: ${UPSERT_GENERATOR_DISTRIBUTION:-benchmark} - UPSERT_GENERATOR_NUM_RECORDS: ${UPSERT_GENERATOR_NUM_RECORDS:-400000000} - UPSERT_GENERATOR_PARALLELISM: ${UPSERT_GENERATOR_PARALLELISM:-40} - steps: - - step: workflow - workflow: start-services - - step: run - service: kafka-avro-generator - command: >- - --parallelism=${UPSERT_GENERATOR_PARALLELISM} - --num-records=${UPSERT_GENERATOR_NUM_RECORDS} - --distribution=${UPSERT_GENERATOR_DISTRIBUTION} - --topic=upsertavrotest - - run-benchmark: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - METRIC_VERIFIER_TIMEOUT: ${METRIC_VERIFIER_TIMEOUT:-3600} - UPSERT_GENERATOR_NUM_RECORDS: ${UPSERT_GENERATOR_NUM_RECORDS:-400000000} - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: metric-verifier - daemon: true - command: >- - scrape - --timeout-seconds=${METRIC_VERIFIER_TIMEOUT} - "sum(mz_messages_ingested)" - ${UPSERT_GENERATOR_NUM_RECORDS} - - step: run - service: create-views - - step: wait - service: metric-verifier - expected_return_code: 0 - print_logs: true - - # The benchmark that we run in the cloud - benchmark-large: - steps: - - step: workflow - workflow: setup-benchmark-large - - step: workflow - workflow: run-benchmark-large - - setup-benchmark-large: - env: - UPSERT_GENERATOR_DISTRIBUTION: benchmark - UPSERT_GENERATOR_NUM_RECORDS: 400000000 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-large: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - UPSERT_GENERATOR_NUM_RECORDS: 400000000 - steps: - - step: workflow - workflow: run-benchmark - - # The benchmark that developers can run on their laptop - benchmark-medium: - steps: - - step: workflow - workflow: setup-benchmark-medium - - step: workflow - workflow: run-benchmark-medium - - setup-benchmark-medium: - env: - UPSERT_GENERATOR_DISTRIBUTION: medium - UPSERT_GENERATOR_NUM_RECORDS: 20000000 - UPSERT_GENERATOR_PARALLELISM: 8 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-medium: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 900 - UPSERT_GENERATOR_NUM_RECORDS: 20000000 - steps: - - step: workflow - workflow: run-benchmark - - # The smoketest benchmark that we run in CI - ci: - steps: - - step: workflow - workflow: setup-benchmark-ci - - step: workflow - workflow: run-benchmark-ci - - benchmark-ci: - steps: - - step: worfklow - workflow: ci - - setup-benchmark-ci: - env: - UPSERT_GENERATOR_DISTRIBUTION: small - UPSERT_GENERATOR_NUM_RECORDS: 100000 - UPSERT_GENERATOR_PARALLELISM: 4 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-ci: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 30 - UPSERT_GENERATOR_NUM_RECORDS: 100000 - steps: - - step: workflow - workflow: run-benchmark - - run-generator: - steps: - - step: run - service: kafka-avro-generator - - rerun-benchmark: - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: create-views diff --git a/test/bench/avro-upsert/prometheus/prometheus.yml b/test/bench/avro-upsert/prometheus/prometheus.yml deleted file mode 100644 index 6950d2120bd27..0000000000000 --- a/test/bench/avro-upsert/prometheus/prometheus.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: materialized - scrape_interval: 1s - static_configs: - - targets: - - 'materialized:6875' diff --git a/test/bench/kafka-sink-avro-debezium/README.md b/test/bench/kafka-sink-avro-debezium/README.md deleted file mode 100644 index 1ba5089da6196..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# Kafka Avro Debezium Sink Benchmark - -This benchmark tests our ability to write messages out to Kafka. In its -current configuration, it uses a topic with only one partition as input and a -topic with only one partition as output. We use only one input topic because -that is enough to saturate writing. We use only one output topic because the -current Kafka sink will only ever write from one timely worker. - -## Running the Kafka Avro Debezium Sink Test - -To run the spec benchmark run the following command: - - ./mzcompose run benchmark - -If you're running on a smaller machine, you can run the version of this -benchmark designed for smaller machines: - - ./mzcompose run benchmark-medium - -You can also use [mzbench](../../../doc/developer/mzbench.md) to run comparative benchmarks on -different versions. See the `mzbench` documentation for more details but here is an example -invocation (run from the repository root): - - ./bin/mzbench -w --size medium kafka-sink-avro-debezium - -## Looking at Performance - -Each benchmark will output something like the following: - -```sh -SUCCESS! seconds_taken=766, rows_per_sec=522193 -Grafana URL: http://localhost:3000/d/materialize-overview/materialize-overview?from=1612572459000&to=1612573285000&tz=UTC -``` - -The Grafana link will display performance metrics for materialize during the -entirety of the benchmark run. diff --git a/test/bench/kafka-sink-avro-debezium/create_sink/Dockerfile b/test/bench/kafka-sink-avro-debezium/create_sink/Dockerfile deleted file mode 100644 index 89c848aa1ba22..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/create_sink/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -FROM postgres - -RUN mkdir /data -COPY sinks /data - -COPY create_sink /usr/local/bin -ENTRYPOINT ["/usr/local/bin/create_sink"] diff --git a/test/bench/kafka-sink-avro-debezium/create_sink/create_sink b/test/bench/kafka-sink-avro-debezium/create_sink/create_sink deleted file mode 100755 index c14728e9b6f95..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/create_sink/create_sink +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -set -euo pipefail - -sink_file="/data/${1:-no-consistency-no-exactly-once-sink.sql}" - -psql -v ON_ERROR_STOP=1 -h materialized -p 6875 -U materialize -f "${sink_file}" materialize diff --git a/test/bench/kafka-sink-avro-debezium/create_sink/mzbuild.yml b/test/bench/kafka-sink-avro-debezium/create_sink/mzbuild.yml deleted file mode 100644 index 13f91800672a8..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/create_sink/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: kafka-sink-avro-debezium-create-sink -publish: false diff --git a/test/bench/kafka-sink-avro-debezium/create_sink/sinks/no-consistency-no-exactly-once-sink.sql b/test/bench/kafka-sink-avro-debezium/create_sink/sinks/no-consistency-no-exactly-once-sink.sql deleted file mode 100644 index 2b44264555d1b..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/create_sink/sinks/no-consistency-no-exactly-once-sink.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE append_source -FROM KAFKA BROKER 'kafka:9092' -TOPIC 'plain_avro_records' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE SINK kafka_sink FROM append_source -INTO KAFKA BROKER 'kafka:9092' TOPIC 'debezium_output' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE DEBEZIUM; diff --git a/test/bench/kafka-sink-avro-debezium/mzcompose b/test/bench/kafka-sink-avro-debezium/mzcompose deleted file mode 100755 index 76f6976c3b4a5..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/mzcompose +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. -# -# mzcompose — runs Docker Compose with Materialize customizations. - -exec "$(dirname "$0")/../../../bin/mzcompose" "$@" diff --git a/test/bench/kafka-sink-avro-debezium/mzcompose.yml b/test/bench/kafka-sink-avro-debezium/mzcompose.yml deleted file mode 100644 index b6038564bea43..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/mzcompose.yml +++ /dev/null @@ -1,311 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - - -# Map from host-port:internal port -# -# This mostly just shows all the ports that are available to the host system, if you want -# to change these you must restart the docker-compose cluster. -x-port-mappings: - - &kafka-internal ${KAFKA_PORT:-9092} - - &kafka-external 9093:9093 - - &schema-registry ${SR_PORT:-8081:8081} - - &materialized ${MZ_PORT:-6875} - - &materialized-external ${MZ_PORT:-6875:6875} - - &grafana 3000:3000 - - &prometheus 9090 - - &perf-metrics ${PERF_METRICS_PORT:-8675} - - &perf-dash-web ${PERF_DASH_PORT:-8875:8875} - -version: '3.7' -services: - create-sink: - mzbuild: kafka-sink-avro-debezium-create-sink - perf-dash-metrics: - mzbuild: materialized - ports: - - *perf-metrics - command: - - --disable-telemetry - - --listen-addr=0.0.0.0:${PERF_METRICS_PORT:-8675} - - --logical-compaction-window=1ms - environment: - - MZ_DEV=1 - perf-dash-scraper: - mzbuild: perf-dash-scraper - command: scrape - perf-dash-create-views: - mzbuild: perf-dash-create-views - perf-dash-web: - mzbuild: perf-dash-web - ports: - - *perf-dash-web - materialized: - mzbuild: materialized - ports: - - *materialized - - *materialized-external - command: - - --workers=${MZ_WORKERS:-16} - - --disable-telemetry - environment: - # You can, for example, add `pgwire=trace` or change `info` to `debug` to - # get more verbose logs. - - MZ_LOG_FILTER=dataflow::sink::kafka=debug,librdkafka=warn,rdkafka=warn,sql_parser=warn,hyper=warn,info - - MZ_DEV=1 - zookeeper: - image: confluentinc/cp-zookeeper:5.5.4 - environment: - - ZOOKEEPER_CLIENT_PORT=2181 - kafka: - image: confluentinc/cp-enterprise-kafka:5.5.4 - ports: - - *kafka-internal - - *kafka-external - depends_on: [zookeeper] - environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT - - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092,EXTERNAL://0.0.0.0:9093 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://${KAFKA_HOST:-kafka}:9093 - - KAFKA_METRIC_REPORTERS=io.confluent.metrics.reporter.ConfluentMetricsReporter - - KAFKA_BROKER_ID=1 - - KAFKA_LOG_RETENTION_HOURS=-1 - - KAFKA_NUM_PARTITIONS=1 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS=kafka:9092 - - CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS=1 - - CONFLUENT_METRICS_REPORTER_TOPIC_CREATE=false - - KAFKA_JMX_PORT=9991 - schema-registry: - image: confluentinc/cp-schema-registry:5.5.4 - ports: - - *schema-registry - environment: - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 - - SCHEMA_REGISTRY_HOST_NAME=schema-registry - - SCHEMA_REGISTRY_LISTENERS=http://schema-registry:8081,http://localhost:8081 - depends_on: [zookeeper, kafka] - kafka-avro-generator: - mzbuild: kafka-avro-generator - metric-verifier: - mzbuild: metric-verifier - environment: - - MZ_WORKERS=${MZ_WORKERS:-0} - - MZBENCH_GIT_REF=${MZBENCH_GIT_REF:-None} - - MZBENCH_ID=${MZBENCH_ID:-0} - # All monitoring - dashboard: - mzbuild: dashboard - propagate_uid_gid: true - environment: - - 'MATERIALIZED_URL=materialized:6875' - ports: - - *grafana - - *prometheus - volumes: - # ensure that data doesn't get lost across restarts - # data will be lost if you remove docker volumes (using nuke, for example) - - prometheus:/prometheus - # specialized configurations - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - -volumes: - prometheus: - -mzworkflows: - - start-services: - steps: - - step: start-services - services: [dashboard] - - step: start-services - services: [kafka, schema-registry, perf-dash-metrics] - - step: wait-for-tcp - host: kafka - port: 9092 - - step: wait-for-tcp - host: schema-registry - port: 8081 - - step: run - service: perf-dash-scraper - command: >- - create-topics - - step: run - service: metric-verifier - command: >- - create-topics - - step: run - service: perf-dash-create-views - - step: start-services - services: [perf-dash-web] - - setup-benchmark: - env: - DATA_GENERATOR_DISTRIBUTION: ${DATA_GENERATOR_DISTRIBUTION:-benchmark} - DATA_GENERATOR_NUM_RECORDS: ${DATA_GENERATOR_NUM_RECORDS:-400000000} - DATA_GENERATOR_PARALLELISM: ${DATA_GENERATOR_PARALLELISM:-40} - steps: - - step: workflow - workflow: start-services - - step: run - service: kafka-avro-generator - command: >- - --parallelism=${DATA_GENERATOR_PARALLELISM} - --num-records=${DATA_GENERATOR_NUM_RECORDS} - --distribution=${DATA_GENERATOR_DISTRIBUTION} - --topic=plain_avro_records - - run-benchmark: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - METRIC_VERIFIER_TIMEOUT: ${METRIC_VERIFIER_TIMEOUT:-3600} - DATA_GENERATOR_NUM_RECORDS: ${DATA_GENERATOR_NUM_RECORDS:-400000000} - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: metric-verifier - daemon: true - command: >- - scrape - --timeout-seconds=${METRIC_VERIFIER_TIMEOUT} - "sum(mz_kafka_messages_sent_total)" - ${DATA_GENERATOR_NUM_RECORDS} - - step: run - service: create-sink - - step: wait - service: metric-verifier - expected_return_code: 0 - print_logs: true - - # The benchmark that we run in the cloud - benchmark-large: - steps: - - step: workflow - workflow: setup-benchmark-large - - step: workflow - workflow: run-benchmark-large - - setup-benchmark-large: - env: - DATA_GENERATOR_DISTRIBUTION: benchmark - DATA_GENERATOR_NUM_RECORDS: 400000000 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-large: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - DATA_GENERATOR_NUM_RECORDS: 400000000 - steps: - - step: workflow - workflow: run-benchmark - - # The benchmark that developers can run on their laptop - benchmark-medium: - steps: - - step: workflow - workflow: setup-benchmark-medium - - step: workflow - workflow: run-benchmark-medium - - setup-benchmark-medium: - env: - DATA_GENERATOR_DISTRIBUTION: medium - DATA_GENERATOR_NUM_RECORDS: 20000000 - DATA_GENERATOR_PARALLELISM: 8 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-medium: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 900 - DATA_GENERATOR_NUM_RECORDS: 20000000 - steps: - - step: workflow - workflow: run-benchmark - - # Tiny dataset for quick iteration in dev build mode - benchmark-tiny: - steps: - - step: workflow - workflow: setup-benchmark-tiny - - step: workflow - workflow: run-benchmark-tiny - - setup-benchmark-tiny: - env: - DATA_GENERATOR_DISTRIBUTION: medium - DATA_GENERATOR_NUM_RECORDS: 2000 - DATA_GENERATOR_PARALLELISM: 1 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-tiny: - env: - MZ_WORKERS: ${MZ_WORKERS:-5} - METRIC_VERIFIER_TIMEOUT: 900 - DATA_GENERATOR_NUM_RECORDS: 2000 - steps: - - step: workflow - workflow: run-benchmark - - # The smoketest benchmark that we run in CI - ci: - steps: - - step: workflow - workflow: setup-benchmark-ci - - step: workflow - workflow: run-benchmark-ci - - benchmark-ci: - steps: - - step: worfklow - workflow: ci - - setup-benchmark-ci: - env: - DATA_GENERATOR_DISTRIBUTION: small - DATA_GENERATOR_NUM_RECORDS: 100000 - DATA_GENERATOR_PARALLELISM: 4 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-ci: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 30 - DATA_GENERATOR_NUM_RECORDS: 100000 - steps: - - step: workflow - workflow: run-benchmark - - run-generator: - steps: - - step: run - service: kafka-avro-generator - - rerun-benchmark: - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: create-sink diff --git a/test/bench/kafka-sink-avro-debezium/prometheus/prometheus.yml b/test/bench/kafka-sink-avro-debezium/prometheus/prometheus.yml deleted file mode 100644 index 6950d2120bd27..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/prometheus/prometheus.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: materialized - scrape_interval: 1s - static_configs: - - targets: - - 'materialized:6875' diff --git a/test/bench/util/kafka-avro-generator/Dockerfile b/test/bench/util/kafka-avro-generator/Dockerfile deleted file mode 100644 index d2e1a971f1c36..0000000000000 --- a/test/bench/util/kafka-avro-generator/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -MZFROM kgen - -FROM python:3.9-slim - -COPY --from=0 /usr/local/bin/kgen /usr/local/bin - -RUN mkdir -p /usr/share/generator/distributions -COPY distributions /usr/share/generator/distributions - -COPY generate_records /usr/local/bin -ENTRYPOINT ["generate_records"] diff --git a/test/bench/util/kafka-avro-generator/distributions/benchmark/key-distribution.json b/test/bench/util/kafka-avro-generator/distributions/benchmark/key-distribution.json deleted file mode 100644 index 2547ba99814ca..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/benchmark/key-distribution.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "com.acme.avro.testrecordkey::Key1": [0, 100], - "com.acme.avro.testrecordkey::Key2": [0, 250000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/benchmark/key-schema.json b/test/bench/util/kafka-avro-generator/distributions/benchmark/key-schema.json deleted file mode 100644 index c156c8ae85838..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/benchmark/key-schema.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "testrecordkey", - "type" : "record", - "namespace": "com.acme.avro", - "fields": [ - { - "name": "Key1", - "type": "long" - }, - { - "name": "Key2", - "type": "long" - } - ] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/benchmark/value-distribution.json b/test/bench/util/kafka-avro-generator/distributions/benchmark/value-distribution.json deleted file mode 100644 index 1d1f77be38375..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/benchmark/value-distribution.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "com.acme.avro.testrecord::Key1Unused": [0, 100], - "com.acme.avro.testrecord::Key2Unused": [0, 250000], - "com.acme.avro.InnerRecord1::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord2::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord3::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord4::Point": [10000, 10000000000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/benchmark/value-schema.json b/test/bench/util/kafka-avro-generator/distributions/benchmark/value-schema.json deleted file mode 100644 index 88c958d85a892..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/benchmark/value-schema.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "name": "testrecord", - "type": "record", - "namespace": "com.acme.avro", - "fields": [ - { - "name": "Key1Unused", - "type": "long" - }, - { - "name": "Key2Unused", - "type": "long" - }, - { - "name": "OuterRecord", - "type": { - "name": "OuterRecord", - "type": "record", - "fields": [ - { - "name": "Record1", - "type": { - "name": "Record1", - "type": "record", - "fields": [ - { - "name": "InnerRecord1", - "type": { - "name": "InnerRecord1", - "type": "record", - "fields": [ - { - "name": "Point", - "type": "long" - } - ] - } - }, - { - "name": "InnerRecord2", - "type": { - "name": "InnerRecord2", - "type": "record", - "fields": [ - { - "name": "Point", - "type": "long" - } - ] - } - } - ] - } - }, - { - "name": "Record2", - "type": { - "name": "Record2", - "type": "record", - "fields": [ - { - "name": "InnerRecord3", - "type": { - "name": "InnerRecord3", - "type": "record", - "fields": [ - { - "name": "Point", - "type": "long" - } - ] - } - }, - { - "name": "InnerRecord4", - "type": { - "name": "InnerRecord4", - "type": "record", - "fields": [ - { - "name": "Point", - "type": "long" - } - ] - } - } - ] - } - } - ] - } - } - ] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/big-records/key-distribution.json b/test/bench/util/kafka-avro-generator/distributions/big-records/key-distribution.json deleted file mode 100644 index 2547ba99814ca..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/big-records/key-distribution.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "com.acme.avro.testrecordkey::Key1": [0, 100], - "com.acme.avro.testrecordkey::Key2": [0, 250000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/big-records/key-schema.json b/test/bench/util/kafka-avro-generator/distributions/big-records/key-schema.json deleted file mode 100644 index c156c8ae85838..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/big-records/key-schema.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "testrecordkey", - "type" : "record", - "namespace": "com.acme.avro", - "fields": [ - { - "name": "Key1", - "type": "long" - }, - { - "name": "Key2", - "type": "long" - } - ] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/big-records/value-distribution.json b/test/bench/util/kafka-avro-generator/distributions/big-records/value-distribution.json deleted file mode 100644 index 570d0db5e9d29..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/big-records/value-distribution.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "com.acme.avro.inner::f1": [0, 1000000000], - "com.acme.avro.inner::f2.len": [0, 100], - "com.acme.avro.inner::f3": [1, 2], - "com.acme.avro.inner::f3.1.len": [0, 100], - "com.acme.avro.inner::f4": [0, 1], - "com.acme.avro.inner::f5": [10000, 100000000000], - "com.acme.avro.testrecord::outer_array.len": [0, 1000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/big-records/value-schema.json b/test/bench/util/kafka-avro-generator/distributions/big-records/value-schema.json deleted file mode 100644 index 6901f4369e571..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/big-records/value-schema.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "name": "testrecord", - "type": "record", - "namespace": "com.acme.avro", - "fields": [ - { - "name": "outer_array", - "type": { - "type": "array", - "items": { - "name": "inner", - "type": "record", - "fields": [ - { - "name": "f1", - "type": "int" - }, - { - "name": "f2", - "type": "string" - }, - { - "name": "f3", - "type": ["null", "string"] - }, - { - "name": "f4", - "type": "double" - }, - { - "name": "f5", - "type": "long" - } - ] - } - } - } - ] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/medium/key-distribution.json b/test/bench/util/kafka-avro-generator/distributions/medium/key-distribution.json deleted file mode 100644 index 3e3a2fc6809ca..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/medium/key-distribution.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "com.acme.avro.testrecordkey::Key1": [0, 100], - "com.acme.avro.testrecordkey::Key2": [0, 25000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/medium/key-schema.json b/test/bench/util/kafka-avro-generator/distributions/medium/key-schema.json deleted file mode 120000 index d9466bc11e6ae..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/medium/key-schema.json +++ /dev/null @@ -1 +0,0 @@ -../benchmark/key-schema.json \ No newline at end of file diff --git a/test/bench/util/kafka-avro-generator/distributions/medium/value-distribution.json b/test/bench/util/kafka-avro-generator/distributions/medium/value-distribution.json deleted file mode 100644 index 3ba3d0c924f19..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/medium/value-distribution.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "com.acme.avro.testrecord::Key1Unused": [0, 100], - "com.acme.avro.testrecord::Key2Unused": [0, 25000], - "com.acme.avro.InnerRecord1::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord2::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord3::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord4::Point": [10000, 10000000000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/medium/value-schema.json b/test/bench/util/kafka-avro-generator/distributions/medium/value-schema.json deleted file mode 120000 index a57a3848fd889..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/medium/value-schema.json +++ /dev/null @@ -1 +0,0 @@ -../benchmark/value-schema.json \ No newline at end of file diff --git a/test/bench/util/kafka-avro-generator/distributions/small/key-distribution.json b/test/bench/util/kafka-avro-generator/distributions/small/key-distribution.json deleted file mode 100644 index 638754202d23a..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/small/key-distribution.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "com.acme.avro.testrecordkey::Key1": [0, 10], - "com.acme.avro.testrecordkey::Key2": [0, 25] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/small/key-schema.json b/test/bench/util/kafka-avro-generator/distributions/small/key-schema.json deleted file mode 120000 index d9466bc11e6ae..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/small/key-schema.json +++ /dev/null @@ -1 +0,0 @@ -../benchmark/key-schema.json \ No newline at end of file diff --git a/test/bench/util/kafka-avro-generator/distributions/small/value-distribution.json b/test/bench/util/kafka-avro-generator/distributions/small/value-distribution.json deleted file mode 100644 index f8b9101c54e7e..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/small/value-distribution.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "com.acme.avro.testrecord::Key1Unused": [0, 10], - "com.acme.avro.testrecord::Key2Unused": [0, 25], - "com.acme.avro.InnerRecord1::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord2::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord3::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord4::Point": [10000, 10000000000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/small/value-schema.json b/test/bench/util/kafka-avro-generator/distributions/small/value-schema.json deleted file mode 120000 index a57a3848fd889..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/small/value-schema.json +++ /dev/null @@ -1 +0,0 @@ -../benchmark/value-schema.json \ No newline at end of file diff --git a/test/bench/util/kafka-avro-generator/generate_records b/test/bench/util/kafka-avro-generator/generate_records deleted file mode 100755 index 94142bfc09cb0..0000000000000 --- a/test/bench/util/kafka-avro-generator/generate_records +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -import argparse -import multiprocessing -import pathlib -import subprocess -import sys - -DISTRIBUTIONS = "/usr/share/generator/distributions" - -def run(args: argparse.Namespace) -> int: - """Run the generator, inserting args.num_records number of messages.""" - - records_per_process = int(args.num_records / args.parallelism) - - bootstrap_broker = args.bootstrap_broker - schema_registry_url = args.schema_registry_url - - topic= args.topic - - distribution_dir = pathlib.Path(DISTRIBUTIONS, args.distribution) - key_schema = pathlib.Path(distribution_dir, "key-schema.json").read_text().strip() - value_schema = pathlib.Path(distribution_dir, "value-schema.json").read_text().strip() - - key_distribution = ( - pathlib.Path(distribution_dir, "key-distribution.json").read_text().strip() - ) - value_distribution = ( - pathlib.Path(distribution_dir, "value-distribution.json").read_text().strip() - ) - - kafka_gen = [ - "/usr/local/bin/kgen", - "--quiet", - "--bootstrap-server", - bootstrap_broker, - "--schema-registry-url", - schema_registry_url, - "--num-records", - str(records_per_process), - "--topic", - topic, - "--keys", - "avro", - "--values", - "avro", - "--avro-schema", - value_schema, - "--avro-distribution", - value_distribution, - "--avro-key-schema", - key_schema, - "--avro-key-distribution", - key_distribution, - ] - - print( - f"Spawning {args.parallelism} generator processes, writing {records_per_process} messages each" - ) - procs = [subprocess.Popen(kafka_gen) for _ in range(0, args.parallelism)] - - exit_code = 0 - for (i, p) in enumerate(procs): - p.wait() - print( - f"{i}/{args.parallelism} processes finished: pid={p.pid} returncode={p.returncode}" - ) - - # If a process exited with an error, exit with the return code of the first such process - if p.returncode != 0 and exit_code == 0: - exit_code = p.returncode - - return exit_code - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument( - "-d", - "--distribution", - type=str, - default="benchmark", - help="Name of the distribution to use for generating random records", - choices=[p.name for p in pathlib.Path(DISTRIBUTIONS).iterdir()], - ) - - parser.add_argument( - "-n", - "--num-records", - type=int, - default=400000000, - help="Total number of messages to generate", - ) - parser.add_argument( - "-p", - "--parallelism", - type=int, - default=multiprocessing.cpu_count(), - help="Number of processes to spawn", - ) - - parser.add_argument( - "-b", - "--bootstrap-broker", - type=str, - default="kafka:9092", - help="Kafka bootstrap server", - ) - - parser.add_argument( - "-r", - "--schema-registry-url", - type=str, - default="http://schema-registry:8081", - help="Schema Registry url", - ) - - parser.add_argument( - "-t", - "--topic", - type=str, - help="Kafka topic", - ) - - args = parser.parse_args() - if args.num_records % args.parallelism != 0: - print("ERROR: Number of records must divide evenly by number of processes") - sys.exit(1) - - sys.exit(run(args)) diff --git a/test/bench/util/kafka-avro-generator/mzbuild.yml b/test/bench/util/kafka-avro-generator/mzbuild.yml deleted file mode 100644 index d415c5a7b0769..0000000000000 --- a/test/bench/util/kafka-avro-generator/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: kafka-avro-generator -publish: false diff --git a/test/bench/util/metric-verifier/Dockerfile b/test/bench/util/metric-verifier/Dockerfile deleted file mode 100644 index 7e95354490c02..0000000000000 --- a/test/bench/util/metric-verifier/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -MZFROM ubuntu-base - -ARG DEBIAN_FRONTEND="noninteractive" - -RUN apt-get update && apt-get install -y librdkafka-dev python3-dev python3 python3-pip tzdata && apt-get clean - -COPY requirements.txt /tmp -RUN pip3 install -r /tmp/requirements.txt - -RUN mkdir -p /usr/share/benchmarks/schemas -COPY schemas /usr/share/benchmarks/schemas - -COPY wait_metric /usr/local/bin -ENTRYPOINT ["wait_metric"] diff --git a/test/bench/util/metric-verifier/mzbuild.yml b/test/bench/util/metric-verifier/mzbuild.yml deleted file mode 100644 index bd593d9de80d2..0000000000000 --- a/test/bench/util/metric-verifier/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: metric-verifier -publish: false diff --git a/test/bench/util/metric-verifier/requirements.txt b/test/bench/util/metric-verifier/requirements.txt deleted file mode 100644 index e9e3fe5217b5b..0000000000000 --- a/test/bench/util/metric-verifier/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -confluent-kafka[avro]==1.5.0 -prometheus-api-client==0.4.2 -random_name==0.1.1 diff --git a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/key-schema.avsc b/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/key-schema.avsc deleted file mode 100644 index 527b31ca9c3cc..0000000000000 --- a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/key-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "key", - "type" : "record", - "namespace": "dev.mtrlz.benchmarks.runs.begin.v0", - "fields": [ - { - "name": "run_id", - "type": "string" - } - ] -} diff --git a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/value-schema.avsc b/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/value-schema.avsc deleted file mode 100644 index 0b4fdd7d7e8ba..0000000000000 --- a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/value-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "value", - "type" : "record", - "namespace": "dev.mtrlz.benchmarks.runs.begin.v0", - "fields": [ - { - "name": "start_ms", - "type": "long" - } - ] -} diff --git a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/key-schema.avsc b/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/key-schema.avsc deleted file mode 100644 index 8c97e1f6ac443..0000000000000 --- a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/key-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "key", - "type" : "record", - "namespace": "dev.mtrlz.benchmarks.runs.params.v0", - "fields": [ - { - "name": "run_id", - "type": "string" - } - ] -} diff --git a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/value-schema.avsc b/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/value-schema.avsc deleted file mode 100644 index 31eadb8a16e9f..0000000000000 --- a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/value-schema.avsc +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "value", - "type" : "record", - "namespace": "dev.mtrlz.benchmarks.runs.params.v0", - "fields": [ - { - "name": "benchmark_id", - "type": "string" - }, - { - "name": "git_ref", - "type": "string" - }, - { - "name": "mz_workers", - "type": "int" - } - ] -} diff --git a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/key-schema.avsc b/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/key-schema.avsc deleted file mode 100644 index 0d2f816cb7110..0000000000000 --- a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/key-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "key", - "type" : "record", - "namespace": "dev.mtrlz.benchmarks.runs.results.v0", - "fields": [ - { - "name": "run_id", - "type": "string" - } - ] -} diff --git a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/value-schema.avsc b/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/value-schema.avsc deleted file mode 100644 index 9d9bd8740f5c2..0000000000000 --- a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/value-schema.avsc +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "value", - "type" : "record", - "namespace": "dev.mtrlz.benchmarks.runs.results.v0", - "fields": [ - { - "name": "result", - "type": "string" - }, - { - "name": "end_ms", - "type": "long" - }, - { - "name": "rows_per_second", - "type": "long" - } - ] -} diff --git a/test/bench/util/metric-verifier/wait_metric b/test/bench/util/metric-verifier/wait_metric deleted file mode 100755 index a6f3c9c0b2271..0000000000000 --- a/test/bench/util/metric-verifier/wait_metric +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -import argparse -import datetime -import io -import math -import os -import pathlib -import sys -import time -import typing -import urllib -import uuid - -import confluent_kafka.admin -import confluent_kafka.avro -import confluent_kafka.schema_registry -import dateparser -import prometheus_api_client -import prometheus_api_client.utils -import requests - -METRICS_DIRECTORY = "/usr/share/benchmarks/schemas" - - -def create_topics(args: argparse.Namespace) -> None: - """Create the topics for each metric and exit.""" - - csr_config = {'url': f'http://{args.csr_host}:{args.csr_port}'} - csr = confluent_kafka.schema_registry.SchemaRegistryClient(csr_config) - - kafka_config = {'bootstrap.servers': f"{args.kafka_host}:{args.kafka_port}"} - admin_client = confluent_kafka.admin.AdminClient(kafka_config) - - for metric_dir in pathlib.Path(args.metrics_directory).iterdir(): - - key_schema_str = pathlib.Path(metric_dir, 'key-schema.avsc').read_text().strip() - key_schema = confluent_kafka.schema_registry.Schema(key_schema_str, 'AVRO') - value_schema_str = pathlib.Path(metric_dir, 'value-schema.avsc').read_text().strip() - value_schema = confluent_kafka.schema_registry.Schema(value_schema_str, 'AVRO') - - csr.register_schema(f"{metric_dir.name}-key", key_schema) - csr.register_schema(f"{metric_dir.name}-value", value_schema) - - # Create topics takes and returns a list of futures but we're going to call create topic - # while iterating on each directory, so these are just lists of 1 - topics = [confluent_kafka.admin.NewTopic(metric_dir.name, num_partitions=10, replication_factor=1)] - # Don't bother trying to catch the error, let's just fail startup and raise the error - [future.result() for future in admin_client.create_topics(topics).values()] - - -class Producer: - - def __init__(self, topic_dir: pathlib.Path, kafka_config: typing.Dict[str, str]): - """Construct a producer for writing to the given topic, using the given config.""" - self.topic_name = topic_dir.name - - key_schema = confluent_kafka.avro.load(pathlib.Path(topic_dir, 'key-schema.avsc')) - value_schema = confluent_kafka.avro.load(pathlib.Path(topic_dir, 'value-schema.avsc')) - - # Namespace for both schemas should be the name of the topic - assert key_schema.namespace == value_schema.namespace - assert self.topic_name == key_schema.namespace - - self.producer = confluent_kafka.avro.AvroProducer(kafka_config, - default_key_schema=key_schema, - default_value_schema=value_schema) - - def write_metric(self, key: typing.Any, value: typing.Any) -> None: - """Encode key and value using Avro and send event to Kafka.""" - self.producer.produce(topic=self.topic_name, key=key, value=value) - self.producer.flush() - - -class BenchmarkMetrics: - - def __init__(self, args: argparse.Namespace, schema_directory: str): - """Create an instance responsible for recording benchmark metrics.""" - self.args = args - self.schema_directory = schema_directory - self.producers: typing.Dict[str, Producer] = {} - - # See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - kafka_config = {'bootstrap.servers': f"{args.kafka_host}:{args.kafka_port}", - 'schema.registry.url': f'http://{args.csr_host}:{args.csr_port}'} - for topic_dir in pathlib.Path(self.schema_directory).iterdir(): - self.producers[topic_dir.name] = Producer(topic_dir, kafka_config) - - def write_metric(self, topic: str, key: typing.Any, value: typing.Any) -> None: - """Encode key and value using Avro and send event to Kafka.""" - self.producers[topic].write_metric(key, value) - - -def scrape(args: argparse.Namespace) -> int: - """Wait for the query to settle or timeout and then dump ingest metrics.""" - - start = dateparser.parse('now') - metrics = BenchmarkMetrics(args, args.metrics_directory) - - # Record this as two distinct metrics. Ideally, the first would be recorded by mzbench - # when it's populating the list of benchmarks to run. Then second would then be recorded by - # this runner, which may or may not be the same process / machine as the one enumerating the - # parameter space - metrics.write_metric('dev.mtrlz.benchmarks.runs.params.v0', - {"run_id": args.mzbench_run_id}, - {"benchmark_id": args.mzbench_id, - "git_ref": args.mzbench_git_ref, - "mz_workers": args.mz_workers}) - metrics.write_metric('dev.mtrlz.benchmarks.runs.begin.v0', - {"run_id": args.mzbench_run_id}, - {"start_ms": int(time.time() * 1000)}) - - try: - return wait_metric(args, metrics, start) - except: - record_results(args, metrics, start, result="error") - raise - - -def record_results(args: argparse.Namespace, metrics: BenchmarkMetrics, start: datetime.datetime, - rows_per_second: int = 0, result: str = "passed"): - """Write the results of this benchmark to the Kafka topic""" - metrics.write_metric('dev.mtrlz.benchmarks.runs.results.v0', - {"run_id": args.mzbench_run_id}, - {"end_ms": int(time.time() * 1000), - "rows_per_second": rows_per_second, - "result": result}) - - # TODO: Construct the URL externally from this script, maintain for compat with mzbench - # Create parameters to see a dashboard with the metrics from this benchmark run - # Add padding to make the charts nicer to read - # Grafana expects timestamps with milliseconds - path = '/d/materialize-overview/materialize-overview' - query = urllib.parse.urlencode( { - "from": round((start - datetime.timedelta(seconds=30)).timestamp()) * 1000, - "to": round(dateparser.parse('in 30 seconds').timestamp()) * 1000, - "tz": "UTC" - }) - dashboard_url = urllib.parse.urlunparse(('http', args.grafana_location, path, '', query, '')) - - print(f'Grafana URL: {dashboard_url}') - - -def wait_metric(args: argparse.Namespace, metrics: BenchmarkMetrics, start: datetime.datetime) -> int: - """Wait for the given metric, returning desired exit code (0 is success).""" - prom = prometheus_api_client.PrometheusConnect(f'http://{args.prometheus_host}:{args.prometheus_port}') - time_so_far = 0 - begin = int(time.monotonic()) - while time_so_far < args.timeout_seconds: - current_values = prom.custom_query(args.prom_query) - if args.verbose: - print(current_values) - - if len(current_values) > 1: - print('ERROR: Prometheus query must only return a zero or one results!') - prometheus_api_client.utils.pretty_print_metric(current_values) - return 1 - - # We aren't running query_range, so there should only be a single timestamp and point in the reponse - if current_values: - (ts, point) = [float(i) for i in current_values[0]['value']] - if point == args.expected_value: - rate = round(point / max(time_so_far, 1)) - print(f"SUCCESS! seconds_taken={time_so_far} rows_per_sec={rate}") - record_results(args, metrics, start, rate) - return 0 - - time.sleep(1) - time_so_far = int(time.monotonic()) - begin - - # Check this last because it's okay to have a 1-2 second grace period and we want the - # ability to print the most recent result - print(f"FAILED! Query response is '{point}' after {time_so_far} seconds") - return 1 - - -def run(args: argparse.Namespace) -> None: - """Run the desired command.""" - return args.action(args) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - - parser.add_argument( - "-v", - "--verbose", - action="store_true", - help="Enable verbose logging to print the results of each prometheus query", - ) - - parser.add_argument( - "--metrics-directory", - type=str, - default=METRICS_DIRECTORY, - help="Directory containing metrics definitions and source queries", - ) - - parser.add_argument( - "--kafka-host", - help="Name of the kafka broker", - type=str, - default="kafka", - ) - - parser.add_argument( - "--kafka-port", help="Port the connect to the broker over", type=int, default=9092 - ) - - parser.add_argument( - "--csr-host", help="Hostname of the schema registry", type=str, - default="schema-registry" - ) - parser.add_argument( - "--csr-port", help="Port that schema registry is listening on", type=int, - default=8081 - ) - - sub_parsers = parser.add_subparsers() - - # Create topics subcommand and flags - create_topic_parser = sub_parsers.add_parser("create-topics") - create_topic_parser.set_defaults(action=create_topics) - - # Scrape subcommand and flags - scrape_parser = sub_parsers.add_parser("scrape") - scrape_parser.set_defaults(action=scrape) - - scrape_parser.add_argument( - "--grafana-location", - type=str, - default='localhost:3000', - help="Default URL net location (host and port) for Grafana", - ) - - scrape_parser.add_argument( - "--mz-workers", - type=int, - default=os.environ.get('MZ_WORKERS', 0), - help="How many workers materialized is configured to use", - ) - - scrape_parser.add_argument( - "--mzbench-git-ref", - type=str, - default=os.environ.get('MZBENCH_GIT_REF', 'undefined'), - help="The materialized image tag", - ) - - scrape_parser.add_argument( - "--mzbench-id", - type=str, - default=os.environ.get('MZBENCH_ID', str(uuid.uuid4())), - help="Unique string that identifies the grouping of benchmark runs", - ) - - scrape_parser.add_argument( - "--mzbench-run-id", - type=str, - default=os.environ.get('MZBENCH_RUN_ID', str(uuid.uuid4())), - help="Unique string that identifies this particular benchmark run", - ) - - scrape_parser.add_argument( - "--prometheus-host", - type=str, - default='dashboard', - help="Hostname of the prometheus instance to query", - ) - - scrape_parser.add_argument( - "--prometheus-port", - type=int, - default=9090, - help="Port on which the prometheus instance is running", - ) - - scrape_parser.add_argument( - "-t", - "--timeout-seconds", - type=int, - default=900, - help="Length of time to wait until the metric reaches the specified value", - ) - - scrape_parser.add_argument( - "prom_query", - type=str, - help="Prometheus query to run", - ) - - scrape_parser.add_argument( - "expected_value", - type=float, - help="Expected value of the metric queried", - ) - - args = parser.parse_args() - return args - -if __name__ == '__main__': - sys.exit(run(parse_args()))