From 9c9239a2f95c8e1a7e3dbf385cbfbf1f41e64930 Mon Sep 17 00:00:00 2001 From: Nikhil Benesch Date: Tue, 28 Dec 2021 02:58:01 -0500 Subject: [PATCH] mzbench: remove mzbench has been obsoleted by the feature benchmark framework (in test/feature-benchmark) and the cloudbench tool (bin/cloudbench). The kafka-avro-generator tool has been obsoleted by parallelizing kgen directly (#9841). So this commit removes mzbench. To expound on the rational for removing mzbench: * mzbench configurations require an unmaintainable duplication of mzcompose.yml files. Each mzbench configuration contains 300+ lines of nearly identical definitions. There was talk of improving this (see #6676), but the plans never came to fruition. * The interplay between mzbench and mzcompose is unnecessarily delicate. mzbench expects a composition with workflows named just so, and then parses their output. This makes it very difficult to refactor the underlying compositions, since you don't know if you're breaking the contract with mzbench. I think most of mzbench's features could be recreated much more simply with an e.g. `--num-trials` parameter to mzcompose. * mzbench introduced quite a bit of complexity by trying to be both a demo of using Materialize to power a real-time dashboard [0] and a benchmarking framework. Experience suggests that this results in a tool that is a suboptimal dashboard and a suboptimal benchmarking framework. Better to have two separate tools optimized for their specific purpose. The new feature benchmarking framework resolves the above concerns. It is only focused on being a benchmarking framework and does not suffer from the code duplication problem. [0]: https://github.com/MaterializeInc/materialize/blob/45586f38a/doc/developer/mzbench.md#worker-balance-visualization --- bin/mzbench | 14 - ci/test/pipeline.template.yml | 27 - doc/developer/mzbench.md | 126 --- misc/perf-dash/app/Dockerfile | 20 - misc/perf-dash/app/mzbuild.yml | 11 - misc/perf-dash/app/templates/index.html | 203 ---- misc/perf-dash/create_views/Dockerfile | 17 - misc/perf-dash/create_views/create_views | 15 - misc/perf-dash/create_views/mzbuild.yml | 11 - .../create_views/views/benchmark_views.sql | 53 - .../create_views/views/system_views.sql | 36 - misc/perf-dash/scraper/Dockerfile | 24 - misc/perf-dash/scraper/README.md | 3 - .../key-schema.avsc | 11 - .../dev.mtrlz.time_per_operator.v0/query.sql | 22 - .../value-schema.avsc | 12 - .../key-schema.avsc | 11 - .../query.sql | 21 - .../value-schema.avsc | 13 - .../key-schema.avsc | 11 - .../dev.mtrlz.time_per_worker.v0/query.sql | 17 - .../value-schema.avsc | 11 - misc/perf-dash/scraper/mzbuild.yml | 11 - misc/perf-dash/scraper/requirements.txt | 3 - misc/perf-dash/scraper/scrape_system_metrics | 190 ---- test/bench/aggregations/README.md | 7 - .../aggregations/create_views/Dockerfile | 16 - .../aggregations/create_views/create_views | 15 - .../aggregations/create_views/mzbuild.yml | 11 - .../create_views/views/aggregation_views.sql | 118 --- test/bench/aggregations/generator/Dockerfile | 20 - .../benchmark/key-distribution.json | 6 - .../benchmark/value-distribution.json | 406 -------- .../generator/distributions/key-schema.json | 11 - .../medium/key-distribution.json | 6 - .../medium/value-distribution.json | 406 -------- .../distributions/small/key-distribution.json | 6 - .../small/value-distribution.json | 406 -------- .../generator/distributions/value-schema.json | 920 ------------------ .../aggregations/generator/generate_data | 114 --- test/bench/aggregations/generator/mzbuild.yml | 11 - test/bench/aggregations/mzcompose | 14 - test/bench/aggregations/mzcompose.yml | 290 ------ .../aggregations/prometheus/prometheus.yml | 19 - test/bench/avro-insert/README.md | 38 - .../bench/avro-insert/create_views/Dockerfile | 16 - .../avro-insert/create_views/create_views | 15 - .../avro-insert/create_views/mzbuild.yml | 11 - .../create_views/views/insert_views.sql | 26 - test/bench/avro-insert/mzcompose | 14 - test/bench/avro-insert/mzcompose.yml | 294 ------ .../avro-insert/prometheus/prometheus.yml | 19 - test/bench/avro-upsert/README.md | 34 - .../bench/avro-upsert/create_views/Dockerfile | 16 - .../avro-upsert/create_views/create_views | 15 - .../avro-upsert/create_views/mzbuild.yml | 11 - .../create_views/views/insert_views.sql | 26 - .../create_views/views/upsert_views.sql | 26 - test/bench/avro-upsert/mzcompose | 14 - test/bench/avro-upsert/mzcompose.yml | 291 ------ .../avro-upsert/prometheus/prometheus.yml | 19 - test/bench/kafka-sink-avro-debezium/README.md | 36 - .../create_sink/Dockerfile | 16 - .../create_sink/create_sink | 15 - .../create_sink/mzbuild.yml | 11 - .../no-consistency-no-exactly-once-sink.sql | 19 - test/bench/kafka-sink-avro-debezium/mzcompose | 14 - .../kafka-sink-avro-debezium/mzcompose.yml | 311 ------ .../prometheus/prometheus.yml | 19 - .../util/kafka-avro-generator/Dockerfile | 20 - .../benchmark/key-distribution.json | 4 - .../distributions/benchmark/key-schema.json | 15 - .../benchmark/value-distribution.json | 8 - .../distributions/benchmark/value-schema.json | 94 -- .../big-records/key-distribution.json | 4 - .../distributions/big-records/key-schema.json | 15 - .../big-records/value-distribution.json | 9 - .../big-records/value-schema.json | 39 - .../medium/key-distribution.json | 4 - .../distributions/medium/key-schema.json | 1 - .../medium/value-distribution.json | 8 - .../distributions/medium/value-schema.json | 1 - .../distributions/small/key-distribution.json | 4 - .../distributions/small/key-schema.json | 1 - .../small/value-distribution.json | 8 - .../distributions/small/value-schema.json | 1 - .../kafka-avro-generator/generate_records | 140 --- .../util/kafka-avro-generator/mzbuild.yml | 11 - test/bench/util/metric-verifier/Dockerfile | 23 - test/bench/util/metric-verifier/mzbuild.yml | 11 - .../util/metric-verifier/requirements.txt | 3 - .../key-schema.avsc | 11 - .../value-schema.avsc | 11 - .../key-schema.avsc | 11 - .../value-schema.avsc | 19 - .../key-schema.avsc | 11 - .../value-schema.avsc | 19 - test/bench/util/metric-verifier/wait_metric | 311 ------ 98 files changed, 5838 deletions(-) delete mode 100755 bin/mzbench delete mode 100644 doc/developer/mzbench.md delete mode 100644 misc/perf-dash/app/Dockerfile delete mode 100644 misc/perf-dash/app/mzbuild.yml delete mode 100644 misc/perf-dash/app/templates/index.html delete mode 100644 misc/perf-dash/create_views/Dockerfile delete mode 100755 misc/perf-dash/create_views/create_views delete mode 100644 misc/perf-dash/create_views/mzbuild.yml delete mode 100644 misc/perf-dash/create_views/views/benchmark_views.sql delete mode 100644 misc/perf-dash/create_views/views/system_views.sql delete mode 100644 misc/perf-dash/scraper/Dockerfile delete mode 100644 misc/perf-dash/scraper/README.md delete mode 100644 misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/key-schema.avsc delete mode 100644 misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/query.sql delete mode 100644 misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/value-schema.avsc delete mode 100644 misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/key-schema.avsc delete mode 100644 misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/query.sql delete mode 100644 misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/value-schema.avsc delete mode 100644 misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/key-schema.avsc delete mode 100644 misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/query.sql delete mode 100644 misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/value-schema.avsc delete mode 100644 misc/perf-dash/scraper/mzbuild.yml delete mode 100644 misc/perf-dash/scraper/requirements.txt delete mode 100755 misc/perf-dash/scraper/scrape_system_metrics delete mode 100644 test/bench/aggregations/README.md delete mode 100644 test/bench/aggregations/create_views/Dockerfile delete mode 100755 test/bench/aggregations/create_views/create_views delete mode 100644 test/bench/aggregations/create_views/mzbuild.yml delete mode 100644 test/bench/aggregations/create_views/views/aggregation_views.sql delete mode 100644 test/bench/aggregations/generator/Dockerfile delete mode 100644 test/bench/aggregations/generator/distributions/benchmark/key-distribution.json delete mode 100644 test/bench/aggregations/generator/distributions/benchmark/value-distribution.json delete mode 100644 test/bench/aggregations/generator/distributions/key-schema.json delete mode 100644 test/bench/aggregations/generator/distributions/medium/key-distribution.json delete mode 100644 test/bench/aggregations/generator/distributions/medium/value-distribution.json delete mode 100644 test/bench/aggregations/generator/distributions/small/key-distribution.json delete mode 100644 test/bench/aggregations/generator/distributions/small/value-distribution.json delete mode 100644 test/bench/aggregations/generator/distributions/value-schema.json delete mode 100755 test/bench/aggregations/generator/generate_data delete mode 100644 test/bench/aggregations/generator/mzbuild.yml delete mode 100755 test/bench/aggregations/mzcompose delete mode 100644 test/bench/aggregations/mzcompose.yml delete mode 100644 test/bench/aggregations/prometheus/prometheus.yml delete mode 100644 test/bench/avro-insert/README.md delete mode 100644 test/bench/avro-insert/create_views/Dockerfile delete mode 100755 test/bench/avro-insert/create_views/create_views delete mode 100644 test/bench/avro-insert/create_views/mzbuild.yml delete mode 100644 test/bench/avro-insert/create_views/views/insert_views.sql delete mode 100755 test/bench/avro-insert/mzcompose delete mode 100644 test/bench/avro-insert/mzcompose.yml delete mode 100644 test/bench/avro-insert/prometheus/prometheus.yml delete mode 100644 test/bench/avro-upsert/README.md delete mode 100644 test/bench/avro-upsert/create_views/Dockerfile delete mode 100755 test/bench/avro-upsert/create_views/create_views delete mode 100644 test/bench/avro-upsert/create_views/mzbuild.yml delete mode 100644 test/bench/avro-upsert/create_views/views/insert_views.sql delete mode 100644 test/bench/avro-upsert/create_views/views/upsert_views.sql delete mode 100755 test/bench/avro-upsert/mzcompose delete mode 100644 test/bench/avro-upsert/mzcompose.yml delete mode 100644 test/bench/avro-upsert/prometheus/prometheus.yml delete mode 100644 test/bench/kafka-sink-avro-debezium/README.md delete mode 100644 test/bench/kafka-sink-avro-debezium/create_sink/Dockerfile delete mode 100755 test/bench/kafka-sink-avro-debezium/create_sink/create_sink delete mode 100644 test/bench/kafka-sink-avro-debezium/create_sink/mzbuild.yml delete mode 100644 test/bench/kafka-sink-avro-debezium/create_sink/sinks/no-consistency-no-exactly-once-sink.sql delete mode 100755 test/bench/kafka-sink-avro-debezium/mzcompose delete mode 100644 test/bench/kafka-sink-avro-debezium/mzcompose.yml delete mode 100644 test/bench/kafka-sink-avro-debezium/prometheus/prometheus.yml delete mode 100644 test/bench/util/kafka-avro-generator/Dockerfile delete mode 100644 test/bench/util/kafka-avro-generator/distributions/benchmark/key-distribution.json delete mode 100644 test/bench/util/kafka-avro-generator/distributions/benchmark/key-schema.json delete mode 100644 test/bench/util/kafka-avro-generator/distributions/benchmark/value-distribution.json delete mode 100644 test/bench/util/kafka-avro-generator/distributions/benchmark/value-schema.json delete mode 100644 test/bench/util/kafka-avro-generator/distributions/big-records/key-distribution.json delete mode 100644 test/bench/util/kafka-avro-generator/distributions/big-records/key-schema.json delete mode 100644 test/bench/util/kafka-avro-generator/distributions/big-records/value-distribution.json delete mode 100644 test/bench/util/kafka-avro-generator/distributions/big-records/value-schema.json delete mode 100644 test/bench/util/kafka-avro-generator/distributions/medium/key-distribution.json delete mode 120000 test/bench/util/kafka-avro-generator/distributions/medium/key-schema.json delete mode 100644 test/bench/util/kafka-avro-generator/distributions/medium/value-distribution.json delete mode 120000 test/bench/util/kafka-avro-generator/distributions/medium/value-schema.json delete mode 100644 test/bench/util/kafka-avro-generator/distributions/small/key-distribution.json delete mode 120000 test/bench/util/kafka-avro-generator/distributions/small/key-schema.json delete mode 100644 test/bench/util/kafka-avro-generator/distributions/small/value-distribution.json delete mode 120000 test/bench/util/kafka-avro-generator/distributions/small/value-schema.json delete mode 100755 test/bench/util/kafka-avro-generator/generate_records delete mode 100644 test/bench/util/kafka-avro-generator/mzbuild.yml delete mode 100644 test/bench/util/metric-verifier/Dockerfile delete mode 100644 test/bench/util/metric-verifier/mzbuild.yml delete mode 100644 test/bench/util/metric-verifier/requirements.txt delete mode 100644 test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/key-schema.avsc delete mode 100644 test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/value-schema.avsc delete mode 100644 test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/key-schema.avsc delete mode 100644 test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/value-schema.avsc delete mode 100644 test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/key-schema.avsc delete mode 100644 test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/value-schema.avsc delete mode 100755 test/bench/util/metric-verifier/wait_metric diff --git a/bin/mzbench b/bin/mzbench deleted file mode 100755 index 87627085a9c52..0000000000000 --- a/bin/mzbench +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. -# -# mzcompose — runs Docker Compose with Materialize customizations. - -exec "$(dirname "$0")"/pyactivate --dev -m materialize.cli.mzbench "$@" diff --git a/ci/test/pipeline.template.yml b/ci/test/pipeline.template.yml index f5605a3f16822..b759fba261d5a 100644 --- a/ci/test/pipeline.template.yml +++ b/ci/test/pipeline.template.yml @@ -178,33 +178,6 @@ steps: run: ci timeout_in_minutes: 30 - - id: aggregations-benchmark - label: "aggregations benchmark sanity check" - depends_on: build-x86_64 - plugins: - - ./ci/plugins/mzcompose: - composition: aggregations - run: ci - timeout_in_minutes: 10 - - - id: avro-upsert-benchmark - label: "avro upsert benchmark sanity check" - depends_on: build-x86_64 - plugins: - - ./ci/plugins/mzcompose: - composition: avro-upsert - run: ci - timeout_in_minutes: 10 - - - id: kafka-sink-avro-debezium-benchmark - label: "kafka sink (avro debezium) benchmark sanity check" - depends_on: build-x86_64 - plugins: - - ./ci/plugins/mzcompose: - composition: kafka-sink-avro-debezium - run: ci - timeout_in_minutes: 10 - - id: catalog-compat label: ":book: catalog compatibility check" depends_on: build-x86_64 diff --git a/doc/developer/mzbench.md b/doc/developer/mzbench.md deleted file mode 100644 index 1c16f1512a382..0000000000000 --- a/doc/developer/mzbench.md +++ /dev/null @@ -1,126 +0,0 @@ -# mzbench - -mzbench is a tool that wraps mzcompose in order to repeatedly run benchmarks against one or more -versions of Materialize, with a matrix of variables to test. Currently, mzbench explicitly varies -the value for `MZ_WORKERS` and allows for choosing which materialize versions to test. Users can -also select the number of times to test each version, as a means of improving confidence in the -results. - -## Running a Benchmark - -Benchmarks are identified by the name of their composition and compositions are expected to -expose workflows with specific names. To run a benchmark, run `mzbench` from the root of your -materialize git checkout: - - ./bin/mzbench - -This script defaults to choosing the benchmark most suitable for developer machines -(`benchmark-medium` for fast laptops) and will run through a permutation of benchmark variables. -It will print a CSV that looks something like the following to your console: - - git_revision,num_workers,iteration,seconds_taken,rows_per_second,grafana_url - NONE,1,0,3,333333,http://localhost:3000/d/materialize-overview/materialize-overview?from=1612572459000&to=1612573285000&tz=UTC - -### Comparing Results Between Materialize Versions - -Benchmark results can be considered either absolute (X operations per second) or relative -(10% faster than the previous release). Our current crop of benchmarks are written to test -relative performance. To help engineers better understand materialized performance, mzbench -supports running the benchmark over multiple versions of materialized: - - ./bin/mzbench ... - -For example, here's how to compare the performance of materialized, built from your local working -directory, against `v0.6.1` and `v0.7.0`: - - ./bin/mzbench v0.7.0 v0.6.1 - -If you don't want to benchmark the version in your local working directory, such as when -benchmarking in the cloud, you can supply the `--no-benchmark-this-checkout` flag: - - ./bin/mzbench --no-benchmark-this-checkout origin/main v0.7.0 v0.6.1 - -### Controlling the Number of Iterations - -Our benchmarks are not sufficiently advanced to provide an accurate picture after only a single -run. By default, mzbench will run each measurement 6 times. This can be controlled via the -`--num-measurements` flag: - - ./bin/mzbench --num-measurements 3 - -## Visualizing Results - -If the benchmark is configured to start `perf-dash` services during the setup step, then you can -visualize results in your browser. These results are update in real-time, as the results are -written to the persistent datastore for perf-dash. To view the benchmark results in your browser, -you can run `mzbench` with the `--web` flag: - - ./bin/mzbench --web ... - -This will open a new browser window, pointed at the web interface for perf-dash, once the services -have started up. - -## Adding New Benchmarks (Compositions) - -`mzbench` expects that benchmarks expose 3 different sizes of benchmarks: - -- `benchmark-ci` - for tests that are intended to verify correctness, not performance. -- `benchmark-medium` - for tests that are intended for developers to understand the performance - changes between two versions of the code. -- `benchmark-large` - for tests that are intended to verify performance in absolute terms. These - are the tests that we run in our cloud based benchmarks. - -Users of `mzbench` can choose which variant to run by using the `--size` flag. - -To run a given benchmark, `mzbench` assumes that there are 2 corresponding workflows for each -benchmark: - -- `setup-` - this job is run once at mzbench startup to initialize any supporting - resources, such as a Kafka cluster or other source of data. -- `run-` - this job is run once for iteration of the benchmark. This workflow is - responsible for clearing any materialized state and making sure that materialized is restarted - with the new parameters set by the benchmark iteration. - -This means that, to write a new benchmark that supports running in CI, on laptops and in the -cloud, 6 workflows must exist: - -- `setup-benchmark-ci` -- `run-benchmark-ci` -- `setup-benchmark-medium` -- `run-benchmark-medium` -- `setup-benchmark-large` -- `run-benchmark-large` - -## Worker Balance Visualization - -If run with the `--web` flag, `mzbench` will open a browser to the results visualizations. In -addition to the results visualization, there is a demonstration visualization calling "Elapsed -Time Per Worker" (you may need to scroll down to see it). This visualization is a "technology -demostration" for replicating system tables to another materialized instance to power a real-time -visualization. - -The code to make this all work lives under `misc/perf-dash` and consists of 3 major components: - -1. `perf-dash-scraper`. A utility that periodically queries system tables, encodes them as Avro and - writes them to a Kafka topic. -2. `perf-dash-metrics`: A materialized instanced configured to read metrics from Kafka and - maintain a materialized view called `time_per_worker`. -3. `perf-dash-web`: A web server configured serve the visualization, pushing updates to the - browser directly from tail. - -Similar metrics can be inferred by using Prometheus / Grafana, as many of the system table -aggregations are also exported as prometheus metrics. So why go through the work of scraping -system tables, writing avro schemas, writing materialized views and custom visualizations? There -are a few motivations: - -1. Demonstrate materialize feature parity with an existing metrics / reporting stack. -2. Demonstrate the value in replicating system tables as SQL to another materialized instance for - telemetry purposes. -3. Demonstrate how to write an event-sourced application using materialized / using materialized - as a denormalizatin engine. -4. Demonstrate the value of real-time visualizations over 5-second refresh, even for internal - engineering projects. -5. Provide real data that can be used as a test-bed for features, particularly around temporal - features (aggregations, sliding / tumbling windows, etc). -6. Lossless metrics collection. Instead of relying on Prometheus scrape intervals, metrics are - persistently recorded in Kafka. No more gaps in our metrics. diff --git a/misc/perf-dash/app/Dockerfile b/misc/perf-dash/app/Dockerfile deleted file mode 100644 index 87bf89b1505b8..0000000000000 --- a/misc/perf-dash/app/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -MZFROM wikirecent-server - -# Override the wikirecent HTML / JS template files with our own -COPY templates /app/templates - -# Override the views that we expose via TAIL -CMD ["--dbhost=perf-dash-metrics", \ - "--dbport=8675", \ - "benchmark_results", \ - "benchmark_progress", \ - "time_per_worker"] diff --git a/misc/perf-dash/app/mzbuild.yml b/misc/perf-dash/app/mzbuild.yml deleted file mode 100644 index faf0e91a11178..0000000000000 --- a/misc/perf-dash/app/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: perf-dash-web -publish: false diff --git a/misc/perf-dash/app/templates/index.html b/misc/perf-dash/app/templates/index.html deleted file mode 100644 index 707c89555b559..0000000000000 --- a/misc/perf-dash/app/templates/index.html +++ /dev/null @@ -1,203 +0,0 @@ - - - {% block title %}Materialize Performance Dashboard{% end %} - - - - - - - -
-
-
- - - - - - diff --git a/misc/perf-dash/create_views/Dockerfile b/misc/perf-dash/create_views/Dockerfile deleted file mode 100644 index 440e69e9ff3a2..0000000000000 --- a/misc/perf-dash/create_views/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -FROM postgres - -RUN mkdir /data -WORKDIR /data -COPY views . - -COPY create_views /usr/local/bin -ENTRYPOINT ["/usr/local/bin/create_views"] diff --git a/misc/perf-dash/create_views/create_views b/misc/perf-dash/create_views/create_views deleted file mode 100755 index 0072452cf0466..0000000000000 --- a/misc/perf-dash/create_views/create_views +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -set -euo pipefail - -for view_file in *.sql; do - psql -v ON_ERROR_STOP=1 -h perf-dash-metrics -p 8675 -U materialize -f "${view_file}" materialize -done diff --git a/misc/perf-dash/create_views/mzbuild.yml b/misc/perf-dash/create_views/mzbuild.yml deleted file mode 100644 index fac0df167db99..0000000000000 --- a/misc/perf-dash/create_views/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: perf-dash-create-views -publish: false diff --git a/misc/perf-dash/create_views/views/benchmark_views.sql b/misc/perf-dash/create_views/views/benchmark_views.sql deleted file mode 100644 index dea3e910fe35e..0000000000000 --- a/misc/perf-dash/create_views/views/benchmark_views.sql +++ /dev/null @@ -1,53 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE benchmark_run_params_v0 -FROM KAFKA BROKER 'kafka:9093' -TOPIC 'dev.mtrlz.benchmarks.runs.params.v0' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE UPSERT -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081'; - -CREATE SOURCE benchmark_run_begin_v0 -FROM KAFKA BROKER 'kafka:9093' -TOPIC 'dev.mtrlz.benchmarks.runs.begin.v0' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE UPSERT -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081'; - -CREATE SOURCE benchmark_run_results_v0 -FROM KAFKA BROKER 'kafka:9093' -TOPIC 'dev.mtrlz.benchmarks.runs.results.v0' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE UPSERT -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081'; - -CREATE OR REPLACE MATERIALIZED VIEW benchmark_progress AS - SELECT bp.benchmark_id, - bb.run_id, - bb.start_ms, - CASE WHEN br.end_ms IS NOT NULL THEN br.end_ms - bb.start_ms ELSE null END AS runtime, - bp.git_ref, - bp.mz_workers, - br.end_ms, - CASE WHEN br.result IS NOT NULL THEN br.result ELSE 'running' END AS result - FROM benchmark_run_begin_v0 AS bb - INNER JOIN benchmark_run_params_v0 AS bp ON bb.run_id = bp.run_id - LEFT JOIN benchmark_run_results_v0 AS br ON bb.run_id = br.run_id; - -CREATE OR REPLACE MATERIALIZED VIEW benchmark_results AS - SELECT bp.benchmark_id, - bb.run_id, - bp.git_ref, - bp.mz_workers, - br.result, - br.rows_per_second - FROM benchmark_run_begin_v0 AS bb - INNER JOIN benchmark_run_params_v0 AS bp ON bb.run_id = bp.run_id - INNER JOIN benchmark_run_results_v0 AS br ON bb.run_id = br.run_id; diff --git a/misc/perf-dash/create_views/views/system_views.sql b/misc/perf-dash/create_views/views/system_views.sql deleted file mode 100644 index f3f02277921f2..0000000000000 --- a/misc/perf-dash/create_views/views/system_views.sql +++ /dev/null @@ -1,36 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE time_per_operator_v0 -FROM KAFKA BROKER 'kafka:9093' -TOPIC 'dev.mtrlz.time_per_operator.v0' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE SOURCE time_per_operator_per_worker_v0 -FROM KAFKA BROKER 'kafka:9093' -TOPIC 'dev.mtrlz.time_per_operator_per_worker.v0' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE SOURCE time_per_worker_v0 -FROM KAFKA BROKER 'kafka:9093' -TOPIC 'dev.mtrlz.time_per_worker.v0' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE MATERIALIZED VIEW IF NOT EXISTS time_per_operator_per_worker AS - SELECT * FROM time_per_operator_per_worker_v0; - -CREATE MATERIALIZED VIEW IF NOT EXISTS time_per_operator AS - SELECT * FROM time_per_operator_v0; - -CREATE MATERIALIZED VIEW IF NOT EXISTS time_per_worker AS - SELECT mz_cluster_id, mz_scheduling_worker_id, mz_scheduling_elapsed_ns FROM time_per_worker_v0 - WHERE mz_logical_timestamp = (SELECT max(mz_logical_timestamp) FROM time_per_worker_v0); diff --git a/misc/perf-dash/scraper/Dockerfile b/misc/perf-dash/scraper/Dockerfile deleted file mode 100644 index 8bf1457393c59..0000000000000 --- a/misc/perf-dash/scraper/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -MZFROM ubuntu-base - -ARG DEBIAN_FRONTEND="noninteractive" - -RUN apt-get update && apt-get install -y librdkafka-dev python3-dev python3 python3-pip tzdata && apt-get clean - -COPY requirements.txt /tmp -RUN pip3 install -r /tmp/requirements.txt - -RUN mkdir -p /usr/share/perf-scraper/metrics -COPY metrics /usr/share/perf-scraper/metrics - -COPY scrape_system_metrics /usr/local/bin -ENTRYPOINT ["scrape_system_metrics"] -CMD ["scrape"] diff --git a/misc/perf-dash/scraper/README.md b/misc/perf-dash/scraper/README.md deleted file mode 100644 index ed9192464bd65..0000000000000 --- a/misc/perf-dash/scraper/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# scraper - -Captures metrics from Materialize and dumps them to a per-metric Kafka topic. diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/key-schema.avsc b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/key-schema.avsc deleted file mode 100644 index 1a6289d935987..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/key-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "key", - "type" : "record", - "namespace": "dev.mtrlz.time_per_operator.v0", - "fields": [ - { - "name": "mz_cluster_id", - "type": "string" - } - ] -} diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/query.sql b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/query.sql deleted file mode 100644 index aba45e81a6c46..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/query.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - --- Get time spent in each Operator -SELECT mz_cluster_id() AS mz_cluster_id, - mz_logical_timestamp()::float AS mz_logical_timestamp, - mz_dataflow_operators.id AS mz_dataflow_operator_id, - mz_dataflow_operators.name AS mz_dataflow_operator_name, - sum(mz_scheduling_elapsed.elapsed_ns)::bigint AS mz_scheduling_elapsed_ns -FROM mz_scheduling_elapsed, - mz_dataflow_operators -WHERE mz_scheduling_elapsed.id = mz_dataflow_operators.id - AND mz_scheduling_elapsed.worker = mz_dataflow_operators.worker -GROUP BY mz_dataflow_operators.id, - mz_dataflow_operators.name -ORDER BY mz_dataflow_operators.id ASC; diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/value-schema.avsc b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/value-schema.avsc deleted file mode 100644 index 20882dba9bafe..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator.v0/value-schema.avsc +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "value", - "type" : "record", - "namespace": "dev.mtrlz.time_per_operator.v0", - "fields": [ - {"name": "mz_cluster_id", "type": "string"}, - {"name": "mz_logical_timestamp", "type": "double"}, - {"name": "mz_dataflow_operator_id", "type": "int"}, - {"name": "mz_dataflow_operator_name", "type": "string"}, - {"name": "mz_scheduling_elapsed_ns", "type": "long"} - ] -} diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/key-schema.avsc b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/key-schema.avsc deleted file mode 100644 index 6016e2ddeb697..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/key-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "key", - "type" : "record", - "namespace": "dev.mtrlz.time_per_operator_per_worker.v0", - "fields": [ - { - "name": "mz_cluster_id", - "type": "string" - } - ] -} diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/query.sql b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/query.sql deleted file mode 100644 index c40d7fc14cc06..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/query.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - --- Get time spent in each Operator per Worker -SELECT mz_cluster_id() AS mz_cluster_id, - mz_logical_timestamp()::float AS mz_logical_timestamp, - mz_dataflow_operators.id AS mz_dataflow_operator_id, - mz_dataflow_operators.worker AS mz_dataflow_operator_worker, - mz_dataflow_operators.name AS mz_dataflow_operator_name, - mz_scheduling_elapsed.elapsed_ns mz_scheduling_elapsed_ns -FROM mz_scheduling_elapsed, - mz_dataflow_operators -WHERE mz_scheduling_elapsed.id = mz_dataflow_operators.id - AND mz_scheduling_elapsed.worker = mz_dataflow_operators.worker -ORDER BY mz_dataflow_operators.id ASC; diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/value-schema.avsc b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/value-schema.avsc deleted file mode 100644 index f2ccbc7d5a8d0..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_operator_per_worker.v0/value-schema.avsc +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "value", - "type" : "record", - "namespace": "dev.mtrlz.time_per_operator_per_worker.v0", - "fields": [ - {"name": "mz_cluster_id", "type": "string"}, - {"name": "mz_logical_timestamp", "type": "double"}, - {"name": "mz_dataflow_operator_id", "type": "int"}, - {"name": "mz_dataflow_operator_worker", "type": "int"}, - {"name": "mz_dataflow_operator_name", "type": "string"}, - {"name": "mz_scheduling_elapsed_ns", "type": "long"} - ] -} diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/key-schema.avsc b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/key-schema.avsc deleted file mode 100644 index efd7c2e8599df..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/key-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "key", - "type" : "record", - "namespace": "dev.mtrlz.time_per_worker.v0", - "fields": [ - { - "name": "mz_cluster_id", - "type": "string" - } - ] -} diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/query.sql b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/query.sql deleted file mode 100644 index 57a8aa2a1bf7e..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/query.sql +++ /dev/null @@ -1,17 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - --- Get time spent in each Worker -SELECT mz_cluster_id() AS mz_cluster_id, - mz_logical_timestamp()::float AS mz_logical_timestamp, - mz_scheduling_elapsed.worker AS mz_scheduling_worker_id, - sum(mz_scheduling_elapsed.elapsed_ns)::bigint AS mz_scheduling_elapsed_ns -FROM mz_scheduling_elapsed -GROUP BY mz_scheduling_elapsed.worker -ORDER BY mz_scheduling_elapsed.worker ASC; diff --git a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/value-schema.avsc b/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/value-schema.avsc deleted file mode 100644 index 10a8c6ec1175a..0000000000000 --- a/misc/perf-dash/scraper/metrics/dev.mtrlz.time_per_worker.v0/value-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "value", - "type" : "record", - "namespace": "dev.mtrlz.time_per_worker.v0", - "fields": [ - {"name": "mz_cluster_id", "type": "string"}, - {"name": "mz_logical_timestamp", "type": "double"}, - {"name": "mz_scheduling_worker_id", "type": "int"}, - {"name": "mz_scheduling_elapsed_ns", "type": "long"} - ] -} diff --git a/misc/perf-dash/scraper/mzbuild.yml b/misc/perf-dash/scraper/mzbuild.yml deleted file mode 100644 index 49fda457ea979..0000000000000 --- a/misc/perf-dash/scraper/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: perf-dash-scraper -publish: false diff --git a/misc/perf-dash/scraper/requirements.txt b/misc/perf-dash/scraper/requirements.txt deleted file mode 100644 index d7f2fb2ecebb1..0000000000000 --- a/misc/perf-dash/scraper/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -confluent-kafka[avro]==1.5.0 -dateparser==1.0.0 -psycopg2-binary==2.8.6 diff --git a/misc/perf-dash/scraper/scrape_system_metrics b/misc/perf-dash/scraper/scrape_system_metrics deleted file mode 100755 index 9c368e9cde040..0000000000000 --- a/misc/perf-dash/scraper/scrape_system_metrics +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -import argparse -import datetime -import io -import os -import pathlib -import sys -import time -import typing -import urllib - -import confluent_kafka.admin -import confluent_kafka.avro -import confluent_kafka.schema_registry -import dateparser -import psycopg2 -import psycopg2.extensions -import psycopg2.extras - -METRICS_DIRECTORY = "/usr/share/perf-scraper/metrics" - - -class Scraper: - - def __init__(self, metric_dir: pathlib.Path, kafka_config: typing.Dict[str, str]): - """Construct a producer for writing to the given topic, using the given config.""" - self.topic_name = metric_dir.name - - key_schema = confluent_kafka.avro.load(pathlib.Path(metric_dir, 'key-schema.avsc')) - value_schema = confluent_kafka.avro.load(pathlib.Path(metric_dir, 'value-schema.avsc')) - - self.query = pathlib.Path(metric_dir, 'query.sql').read_text().strip() - - # Namespace for both schemas should be the name of the topic - assert key_schema.namespace == value_schema.namespace - assert self.topic_name == key_schema.namespace - - self.producer = confluent_kafka.avro.AvroProducer(kafka_config, - default_key_schema=key_schema, - default_value_schema=value_schema) - - def scrape(self, cursor: psycopg2.extensions.cursor) -> None: - """Issue a query against the system tables and write the data to Kafka.""" - cursor.execute(self.query) - for row in cursor: - key = {"mz_cluster_id": row["mz_cluster_id"]} - self.producer.produce(topic=self.topic_name, key=key, value=row) - self.producer.flush() - - -class MetricsScrapers: - - def __init__(self, args: argparse.Namespace): - """Create an instance responsible for recording benchmark metrics.""" - self.scrapers: typing.Dict[str, Scraper] = {} - - # See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - self.kafka_config = {'bootstrap.servers': f"{args.kafka_host}:{args.kafka_port}", - 'schema.registry.url': f'http://{args.csr_host}:{args.csr_port}'} - for metric_dir in pathlib.Path(args.metrics_directory).iterdir(): - self.scrapers[metric_dir.name] = Scraper(metric_dir, self.kafka_config) - - def scrape(self, dsn: str) -> None: - """Encode key and value using Avro and send event to Kafka.""" - with psycopg2.connect(dsn) as conn: - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cursor: - for scraper in self.scrapers.values(): - scraper.scrape(cursor) - - -def create_topics(args: argparse.Namespace) -> None: - """Create the topics for each metric and exit.""" - - csr_config = {'url': f'http://{args.csr_host}:{args.csr_port}'} - csr = confluent_kafka.schema_registry.SchemaRegistryClient(csr_config) - - kafka_config = {'bootstrap.servers': f"{args.kafka_host}:{args.kafka_port}"} - admin_client = confluent_kafka.admin.AdminClient(kafka_config) - - for metric_dir in pathlib.Path(args.metrics_directory).iterdir(): - - key_schema_str = pathlib.Path(metric_dir, 'key-schema.avsc').read_text().strip() - key_schema = confluent_kafka.schema_registry.Schema(key_schema_str, 'AVRO') - value_schema_str = pathlib.Path(metric_dir, 'value-schema.avsc').read_text().strip() - value_schema = confluent_kafka.schema_registry.Schema(value_schema_str, 'AVRO') - - csr.register_schema(f"{metric_dir.name}-key", key_schema) - csr.register_schema(f"{metric_dir.name}-value", value_schema) - - # Create topics takes and returns a list of futures but we're going to call create topic - # while iterating on each directory, so these are just lists of 1 - topics = [confluent_kafka.admin.NewTopic(metric_dir.name, num_partitions=10, replication_factor=1)] - # Don't bother trying to catch the error, let's just fail startup and raise the error - [future.result() for future in admin_client.create_topics(topics).values()] - - -def scrape(args: argparse.Namespace) -> None: - """Grab system metrics from the configured materialized instance, replicating them to Kafka""" - dsn = f'postgresql://{args.dbuser}@{args.dbhost}:{args.dbport}/{args.dbname}' - scrapers = MetricsScrapers(args) - while True: - scrapers.scrape(dsn) - time.sleep(args.scrape_interval) - - -def run(args: argparse.Namespace) -> None: - """Wait for the query to settle or timeout and then dump ingest metrics.""" - args.action(args) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - - parser.add_argument( - "-v", - "--verbose", - action="store_true", - help="Enable verbose logging to print the results of each prometheus query", - ) - - parser.add_argument( - "--metrics-directory", - type=str, - default=METRICS_DIRECTORY, - help="Directory containing metrics definitions and source queries", - ) - - parser.add_argument( - "--kafka-host", - help="Name of the kafka broker", - type=str, - default="kafka", - ) - - parser.add_argument( - "--kafka-port", help="Port the connect to the broker over", type=int, default=9092 - ) - - parser.add_argument( - "--csr-host", help="Hostname of the schema registry", type=str, - default="schema-registry" - ) - parser.add_argument( - "--csr-port", help="Port that schema registry is listening on", type=int, - default=8081 - ) - - sub_parsers = parser.add_subparsers() - - # Create Topics Subcommand and Flags - create_topic_parser = sub_parsers.add_parser("create-topics") - create_topic_parser.set_defaults(action=create_topics) - - # Scrape Subcommand and Flags - scrape_parser = sub_parsers.add_parser("scrape") - scrape_parser.set_defaults(action=scrape) - - scrape_parser.add_argument( - "--dbhost", help="materialized hostname", default="materialized", type=str - ) - scrape_parser.add_argument( - "--dbname", help="materialized database name", default="materialize", type=str - ) - scrape_parser.add_argument( - "--dbport", help="materialized port number", default=6875, type=int - ) - scrape_parser.add_argument( - "--dbuser", help="materialized username", default="materialize", type=str - ) - - scrape_parser.add_argument( - "--scrape-interval", help="How often, in seconds, to scrape metrics", type=int, - default=1 - ) - - return parser.parse_args() - - -if __name__ == '__main__': - run(parse_args()) diff --git a/test/bench/aggregations/README.md b/test/bench/aggregations/README.md deleted file mode 100644 index 3516c43c37136..0000000000000 --- a/test/bench/aggregations/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Aggregations Benchmark - -This benchmark tests the performance for aggregations using the Decimal datatype. - -## Running the Aggregations Benchmark - -To run this benchmark, use [mzbench](../../../doc/developer/mzbench.md). diff --git a/test/bench/aggregations/create_views/Dockerfile b/test/bench/aggregations/create_views/Dockerfile deleted file mode 100644 index 41007ae2b1144..0000000000000 --- a/test/bench/aggregations/create_views/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -FROM postgres - -RUN mkdir /data -COPY views /data - -COPY create_views /usr/local/bin -ENTRYPOINT ["/usr/local/bin/create_views"] diff --git a/test/bench/aggregations/create_views/create_views b/test/bench/aggregations/create_views/create_views deleted file mode 100755 index be4129677ade2..0000000000000 --- a/test/bench/aggregations/create_views/create_views +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -set -euo pipefail - -view_file="/data/${1:-aggregation_views.sql}" - -psql -v ON_ERROR_STOP=1 -h materialized -p 6875 -U materialize -f "${view_file}" materialize diff --git a/test/bench/aggregations/create_views/mzbuild.yml b/test/bench/aggregations/create_views/mzbuild.yml deleted file mode 100644 index dede67cb19883..0000000000000 --- a/test/bench/aggregations/create_views/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: aggregations-bench-create-views -publish: false diff --git a/test/bench/aggregations/create_views/views/aggregation_views.sql b/test/bench/aggregations/create_views/views/aggregation_views.sql deleted file mode 100644 index ceb8cd1e45ee2..0000000000000 --- a/test/bench/aggregations/create_views/views/aggregation_views.sql +++ /dev/null @@ -1,118 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE source_aggregationtest -FROM KAFKA BROKER 'kafka:9092' -TOPIC 'aggregationtest' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE MATERIALIZED VIEW aggregationtest AS SELECT - sum(("OuterRecord")."DecValue00") AS dec00_sum, - sum(("OuterRecord")."DecValue01") AS dec01_sum, - sum(("OuterRecord")."DecValue02") AS dec02_sum, - sum(("OuterRecord")."DecValue03") AS dec03_sum, - sum(("OuterRecord")."DecValue04") AS dec04_sum, - sum(("OuterRecord")."DecValue05") AS dec05_sum, - sum(("OuterRecord")."DecValue06") AS dec06_sum, - sum(("OuterRecord")."DecValue07") AS dec07_sum, - sum(("OuterRecord")."DecValue08") AS dec08_sum, - sum(("OuterRecord")."DecValue09") AS dec09_sum, - sum(("OuterRecord")."DecValue10") AS dec10_sum, - sum(("OuterRecord")."DecValue11") AS dec11_sum, - sum(("OuterRecord")."DecValue12") AS dec12_sum, - sum(("OuterRecord")."DecValue13") AS dec13_sum, - sum(("OuterRecord")."DecValue14") AS dec14_sum, - sum(("OuterRecord")."DecValue15") AS dec15_sum, - sum(("OuterRecord")."DecValue16") AS dec16_sum, - sum(("OuterRecord")."DecValue17") AS dec17_sum, - sum(("OuterRecord")."DecValue18") AS dec18_sum, - sum(("OuterRecord")."DecValue19") AS dec19_sum, - sum(("OuterRecord")."DecValue20") AS dec20_sum, - sum(("OuterRecord")."DecValue21") AS dec21_sum, - sum(("OuterRecord")."DecValue22") AS dec22_sum, - sum(("OuterRecord")."DecValue23") AS dec23_sum, - sum(("OuterRecord")."DecValue24") AS dec24_sum, - sum(("OuterRecord")."DecValue25") AS dec25_sum, - sum(("OuterRecord")."DecValue26") AS dec26_sum, - sum(("OuterRecord")."DecValue27") AS dec27_sum, - sum(("OuterRecord")."DecValue28") AS dec28_sum, - sum(("OuterRecord")."DecValue29") AS dec29_sum, - sum(("OuterRecord")."DecValue30") AS dec30_sum, - sum(("OuterRecord")."DecValue31") AS dec31_sum, - sum(("OuterRecord")."DecValue32") AS dec32_sum, - sum(("OuterRecord")."DecValue33") AS dec33_sum, - sum(("OuterRecord")."DecValue34") AS dec34_sum, - sum(("OuterRecord")."DecValue35") AS dec35_sum, - sum(("OuterRecord")."DecValue36") AS dec36_sum, - sum(("OuterRecord")."DecValue37") AS dec37_sum, - sum(("OuterRecord")."DecValue38") AS dec38_sum, - sum(("OuterRecord")."DecValue39") AS dec39_sum, - sum(("OuterRecord")."DecValue40") AS dec40_sum, - sum(("OuterRecord")."DecValue41") AS dec41_sum, - sum(("OuterRecord")."DecValue42") AS dec42_sum, - sum(("OuterRecord")."DecValue43") AS dec43_sum, - sum(("OuterRecord")."DecValue44") AS dec44_sum, - sum(("OuterRecord")."DecValue45") AS dec45_sum, - sum(("OuterRecord")."DecValue46") AS dec46_sum, - sum(("OuterRecord")."DecValue47") AS dec47_sum, - sum(("OuterRecord")."DecValue48") AS dec48_sum, - sum(("OuterRecord")."DecValue49") AS dec49_sum, - sum(("OuterRecord")."DecValue50") AS dec50_sum, - sum(("OuterRecord")."DecValue51") AS dec51_sum, - sum(("OuterRecord")."DecValue52") AS dec52_sum, - sum(("OuterRecord")."DecValue53") AS dec53_sum, - sum(("OuterRecord")."DecValue54") AS dec54_sum, - sum(("OuterRecord")."DecValue55") AS dec55_sum, - sum(("OuterRecord")."DecValue56") AS dec56_sum, - sum(("OuterRecord")."DecValue57") AS dec57_sum, - sum(("OuterRecord")."DecValue58") AS dec58_sum, - sum(("OuterRecord")."DecValue59") AS dec59_sum, - sum(("OuterRecord")."DecValue60") AS dec60_sum, - sum(("OuterRecord")."DecValue61") AS dec61_sum, - sum(("OuterRecord")."DecValue62") AS dec62_sum, - sum(("OuterRecord")."DecValue63") AS dec63_sum, - sum(("OuterRecord")."DecValue64") AS dec64_sum, - sum(("OuterRecord")."DecValue65") AS dec65_sum, - sum(("OuterRecord")."DecValue66") AS dec66_sum, - sum(("OuterRecord")."DecValue67") AS dec67_sum, - sum(("OuterRecord")."DecValue68") AS dec68_sum, - sum(("OuterRecord")."DecValue69") AS dec69_sum, - sum(("OuterRecord")."DecValue70") AS dec70_sum, - sum(("OuterRecord")."DecValue71") AS dec71_sum, - sum(("OuterRecord")."DecValue72") AS dec72_sum, - sum(("OuterRecord")."DecValue73") AS dec73_sum, - sum(("OuterRecord")."DecValue74") AS dec74_sum, - sum(("OuterRecord")."DecValue75") AS dec75_sum, - sum(("OuterRecord")."DecValue76") AS dec76_sum, - sum(("OuterRecord")."DecValue77") AS dec77_sum, - sum(("OuterRecord")."DecValue78") AS dec78_sum, - sum(("OuterRecord")."DecValue79") AS dec79_sum, - sum(("OuterRecord")."DecValue80") AS dec80_sum, - sum(("OuterRecord")."DecValue81") AS dec81_sum, - sum(("OuterRecord")."DecValue82") AS dec82_sum, - sum(("OuterRecord")."DecValue83") AS dec83_sum, - sum(("OuterRecord")."DecValue84") AS dec84_sum, - sum(("OuterRecord")."DecValue85") AS dec85_sum, - sum(("OuterRecord")."DecValue86") AS dec86_sum, - sum(("OuterRecord")."DecValue87") AS dec87_sum, - sum(("OuterRecord")."DecValue88") AS dec88_sum, - sum(("OuterRecord")."DecValue89") AS dec89_sum, - sum(("OuterRecord")."DecValue90") AS dec90_sum, - sum(("OuterRecord")."DecValue91") AS dec91_sum, - sum(("OuterRecord")."DecValue92") AS dec92_sum, - sum(("OuterRecord")."DecValue93") AS dec93_sum, - sum(("OuterRecord")."DecValue94") AS dec94_sum, - sum(("OuterRecord")."DecValue95") AS dec95_sum, - sum(("OuterRecord")."DecValue96") AS dec96_sum, - sum(("OuterRecord")."DecValue97") AS dec97_sum, - sum(("OuterRecord")."DecValue98") AS dec98_sum, - sum(("OuterRecord")."DecValue99") AS dec99_sum -FROM source_aggregationtest -GROUP BY ("Key1Unused"); diff --git a/test/bench/aggregations/generator/Dockerfile b/test/bench/aggregations/generator/Dockerfile deleted file mode 100644 index bee7c48185e93..0000000000000 --- a/test/bench/aggregations/generator/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -MZFROM kgen - -FROM python:3.9-slim - -COPY --from=0 /usr/local/bin/kgen /usr/local/bin - -RUN mkdir -p /usr/share/generator/distributions -COPY distributions /usr/share/generator/distributions - -COPY generate_data /usr/local/bin -ENTRYPOINT ["generate_data"] diff --git a/test/bench/aggregations/generator/distributions/benchmark/key-distribution.json b/test/bench/aggregations/generator/distributions/benchmark/key-distribution.json deleted file mode 100644 index 42d53cb7aeb7c..0000000000000 --- a/test/bench/aggregations/generator/distributions/benchmark/key-distribution.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "com.acme.avro.aggregationtestkey::Key1": [ - 0, - 9999 - ] -} diff --git a/test/bench/aggregations/generator/distributions/benchmark/value-distribution.json b/test/bench/aggregations/generator/distributions/benchmark/value-distribution.json deleted file mode 100644 index 8a0af05281aec..0000000000000 --- a/test/bench/aggregations/generator/distributions/benchmark/value-distribution.json +++ /dev/null @@ -1,406 +0,0 @@ -{ - "com.acme.avro.aggregationtest::Key1Unused": [ - 0, - 9999 - ], - "com.acme.avro.OuterRecord::DecValue00": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue01": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue02": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue03": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue04": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue05": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue06": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue07": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue08": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue09": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue10": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue11": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue12": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue13": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue14": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue15": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue16": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue17": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue18": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue19": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue20": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue21": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue22": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue23": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue24": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue25": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue26": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue27": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue28": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue29": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue30": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue31": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue32": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue33": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue34": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue35": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue36": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue37": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue38": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue39": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue40": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue41": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue42": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue43": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue44": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue45": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue46": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue47": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue48": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue49": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue50": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue51": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue52": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue53": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue54": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue55": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue56": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue57": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue58": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue59": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue60": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue61": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue62": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue63": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue64": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue65": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue66": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue67": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue68": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue69": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue70": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue71": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue72": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue73": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue74": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue75": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue76": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue77": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue78": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue79": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue80": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue81": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue82": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue83": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue84": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue85": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue86": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue87": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue88": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue89": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue90": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue91": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue92": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue93": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue94": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue95": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue96": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue97": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue98": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue99": [ - 10000, - 1000000000 - ] -} diff --git a/test/bench/aggregations/generator/distributions/key-schema.json b/test/bench/aggregations/generator/distributions/key-schema.json deleted file mode 100644 index 2fac5d8195cbe..0000000000000 --- a/test/bench/aggregations/generator/distributions/key-schema.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "aggregationtestkey", - "type": "record", - "namespace": "com.acme.avro", - "fields": [ - { - "name": "Key1", - "type": "long" - } - ] -} diff --git a/test/bench/aggregations/generator/distributions/medium/key-distribution.json b/test/bench/aggregations/generator/distributions/medium/key-distribution.json deleted file mode 100644 index 2dbf5c851e833..0000000000000 --- a/test/bench/aggregations/generator/distributions/medium/key-distribution.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "com.acme.avro.aggregationtestkey::Key1": [ - 0, - 999 - ] -} diff --git a/test/bench/aggregations/generator/distributions/medium/value-distribution.json b/test/bench/aggregations/generator/distributions/medium/value-distribution.json deleted file mode 100644 index 8a0af05281aec..0000000000000 --- a/test/bench/aggregations/generator/distributions/medium/value-distribution.json +++ /dev/null @@ -1,406 +0,0 @@ -{ - "com.acme.avro.aggregationtest::Key1Unused": [ - 0, - 9999 - ], - "com.acme.avro.OuterRecord::DecValue00": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue01": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue02": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue03": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue04": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue05": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue06": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue07": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue08": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue09": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue10": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue11": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue12": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue13": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue14": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue15": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue16": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue17": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue18": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue19": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue20": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue21": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue22": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue23": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue24": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue25": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue26": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue27": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue28": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue29": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue30": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue31": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue32": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue33": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue34": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue35": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue36": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue37": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue38": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue39": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue40": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue41": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue42": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue43": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue44": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue45": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue46": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue47": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue48": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue49": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue50": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue51": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue52": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue53": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue54": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue55": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue56": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue57": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue58": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue59": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue60": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue61": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue62": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue63": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue64": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue65": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue66": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue67": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue68": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue69": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue70": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue71": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue72": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue73": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue74": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue75": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue76": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue77": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue78": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue79": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue80": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue81": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue82": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue83": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue84": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue85": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue86": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue87": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue88": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue89": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue90": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue91": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue92": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue93": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue94": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue95": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue96": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue97": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue98": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue99": [ - 10000, - 1000000000 - ] -} diff --git a/test/bench/aggregations/generator/distributions/small/key-distribution.json b/test/bench/aggregations/generator/distributions/small/key-distribution.json deleted file mode 100644 index 0cac4b6dc9d5e..0000000000000 --- a/test/bench/aggregations/generator/distributions/small/key-distribution.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "com.acme.avro.aggregationtestkey::Key1": [ - 0, - 9 - ] -} diff --git a/test/bench/aggregations/generator/distributions/small/value-distribution.json b/test/bench/aggregations/generator/distributions/small/value-distribution.json deleted file mode 100644 index 8a0af05281aec..0000000000000 --- a/test/bench/aggregations/generator/distributions/small/value-distribution.json +++ /dev/null @@ -1,406 +0,0 @@ -{ - "com.acme.avro.aggregationtest::Key1Unused": [ - 0, - 9999 - ], - "com.acme.avro.OuterRecord::DecValue00": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue01": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue02": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue03": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue04": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue05": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue06": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue07": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue08": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue09": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue10": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue11": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue12": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue13": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue14": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue15": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue16": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue17": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue18": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue19": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue20": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue21": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue22": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue23": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue24": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue25": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue26": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue27": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue28": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue29": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue30": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue31": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue32": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue33": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue34": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue35": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue36": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue37": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue38": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue39": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue40": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue41": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue42": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue43": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue44": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue45": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue46": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue47": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue48": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue49": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue50": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue51": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue52": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue53": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue54": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue55": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue56": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue57": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue58": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue59": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue60": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue61": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue62": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue63": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue64": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue65": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue66": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue67": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue68": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue69": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue70": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue71": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue72": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue73": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue74": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue75": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue76": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue77": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue78": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue79": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue80": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue81": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue82": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue83": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue84": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue85": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue86": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue87": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue88": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue89": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue90": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue91": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue92": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue93": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue94": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue95": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue96": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue97": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue98": [ - 10000, - 1000000000 - ], - "com.acme.avro.OuterRecord::DecValue99": [ - 10000, - 1000000000 - ] -} diff --git a/test/bench/aggregations/generator/distributions/value-schema.json b/test/bench/aggregations/generator/distributions/value-schema.json deleted file mode 100644 index b437df2a387d1..0000000000000 --- a/test/bench/aggregations/generator/distributions/value-schema.json +++ /dev/null @@ -1,920 +0,0 @@ -{ - "name": "aggregationtest", - "type": "record", - "namespace": "com.acme.avro", - "fields": [ - { - "name": "Key1Unused", - "type": "long" - }, - { - "name": "OuterRecord", - "type": { - "name": "OuterRecord", - "type": "record", - "fields": [ - { - "name": "DecValue00", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue01", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue02", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue03", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue04", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue05", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue06", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue07", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue08", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue09", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue10", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue11", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue12", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue13", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue14", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue15", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue16", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue17", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue18", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue19", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue20", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue21", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue22", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue23", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue24", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue25", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue26", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue27", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue28", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue29", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue30", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue31", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue32", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue33", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue34", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue35", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue36", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue37", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue38", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue39", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue40", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue41", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue42", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue43", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue44", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue45", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue46", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue47", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue48", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue49", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue50", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue51", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue52", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue53", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue54", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue55", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue56", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue57", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue58", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue59", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue60", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue61", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue62", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue63", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue64", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue65", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue66", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue67", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue68", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue69", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue70", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue71", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue72", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue73", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue74", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue75", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue76", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue77", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue78", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue79", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue80", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue81", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue82", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue83", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue84", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue85", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue86", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue87", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue88", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue89", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue90", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue91", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue92", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue93", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue94", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue95", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue96", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue97", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue98", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - }, - { - "name": "DecValue99", - "type": { - "type": "bytes", - "scale": 2, - "precision": 15, - "logicalType": "decimal" - } - } - ] - } - } - ] -} diff --git a/test/bench/aggregations/generator/generate_data b/test/bench/aggregations/generator/generate_data deleted file mode 100755 index 29b1b766e4da6..0000000000000 --- a/test/bench/aggregations/generator/generate_data +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -import argparse -import multiprocessing -import pathlib -import subprocess -import sys - -DISTRIBUTIONS = "/usr/share/generator/distributions" - - -def run(args: argparse.Namespace) -> None: - """Run the generator, inserting args.num_records number of messages.""" - - records_per_process = int(args.num_records / args.parallelism) - - key_schema = pathlib.Path( - DISTRIBUTIONS, "key-schema.json").read_text().strip() - value_schema = pathlib.Path( - DISTRIBUTIONS, "value-schema.json").read_text().strip() - - distribution_dir = pathlib.Path(DISTRIBUTIONS, args.distribution) - key_distribution = ( - pathlib.Path(distribution_dir, - "key-distribution.json").read_text().strip() - ) - value_distribution = ( - pathlib.Path(distribution_dir, - "value-distribution.json").read_text().strip() - ) - - kafka_gen = [ - "/usr/local/bin/kgen", - "--quiet", - "--bootstrap-server", - "kafka:9092", - "--schema-registry-url", - "http://schema-registry:8081", - "--num-records", - str(records_per_process), - "--topic", - "aggregationtest", - "--keys", - "avro", - "--values", - "avro", - "--avro-schema", - value_schema, - "--avro-distribution", - value_distribution, - "--avro-key-schema", - key_schema, - "--avro-key-distribution", - key_distribution, - ] - - print( - f"Spawning {args.parallelism} generator processes, writing {records_per_process} messages each" - ) - procs = [subprocess.Popen(kafka_gen) for _ in range(0, args.parallelism)] - exit_code = 0 - for (i, p) in enumerate(procs): - p.wait() - print( - f"{i}/{args.parallelism} processes finished: pid={p.pid} returncode={p.returncode}" - ) - if p.returncode != 0: - exit_code = p.returncode - - sys.exit(exit_code) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument( - "-d", - "--distribution", - type=str, - default="benchmark", - help="Name of the distribution to use for generating random records", - choices=[p.name for p in pathlib.Path(DISTRIBUTIONS).iterdir()], - ) - - parser.add_argument( - "-n", - "--num-records", - type=int, - default=400000000, - help="Total number of messages to generate", - ) - parser.add_argument( - "-p", - "--parallelism", - type=int, - default=multiprocessing.cpu_count(), - help="Number of processes to spawn", - ) - - args = parser.parse_args() - if args.num_records % args.parallelism != 0: - print("ERROR: Number of records must divide evenly by number of processes") - sys.exit(1) - - run(args) diff --git a/test/bench/aggregations/generator/mzbuild.yml b/test/bench/aggregations/generator/mzbuild.yml deleted file mode 100644 index 403a141b884d4..0000000000000 --- a/test/bench/aggregations/generator/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: aggregation-data-generator -publish: false diff --git a/test/bench/aggregations/mzcompose b/test/bench/aggregations/mzcompose deleted file mode 100755 index 76f6976c3b4a5..0000000000000 --- a/test/bench/aggregations/mzcompose +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. -# -# mzcompose — runs Docker Compose with Materialize customizations. - -exec "$(dirname "$0")/../../../bin/mzcompose" "$@" diff --git a/test/bench/aggregations/mzcompose.yml b/test/bench/aggregations/mzcompose.yml deleted file mode 100644 index b2eaa7c50f603..0000000000000 --- a/test/bench/aggregations/mzcompose.yml +++ /dev/null @@ -1,290 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - - -# Map from host-port:internal port -# -# This mostly just shows all the ports that are available to the host system, if you want -# to change these you must restart the docker-compose cluster. -x-port-mappings: - - &kafka-internal ${KAFKA_PORT:-9092} - - &kafka-external 9093:9093 - - &schema-registry ${SR_PORT:-8081:8081} - - &materialized ${MZ_PORT:-6875} - - &grafana 3000:3000 - - &prometheus 9090 - - &perf-metrics ${PERF_METRICS_PORT:-8675} - - &perf-dash-web ${PERF_DASH_PORT:-8875:8875} - -version: '3.7' -services: - create-views: - mzbuild: aggregations-bench-create-views - perf-dash-metrics: - mzbuild: materialized - ports: - - *perf-metrics - command: - - --disable-telemetry - - --listen-addr=0.0.0.0:${PERF_METRICS_PORT:-8675} - - --logical-compaction-window=1ms - environment: - - MZ_DEV=1 - perf-dash-scraper: - mzbuild: perf-dash-scraper - command: scrape - perf-dash-create-views: - mzbuild: perf-dash-create-views - perf-dash-web: - mzbuild: perf-dash-web - ports: - - *perf-dash-web - materialized: - mzbuild: materialized - ports: - - *materialized - command: - - --workers=${MZ_WORKERS:-16} - - --logical-compaction-window=1ms - # We want this to eventually count up to the size of the largest batch in - # an arrangement. This number represents a tradeoff between proactive - # merging (which takes time) and low latency. - # - # 1000 was chosen by fair dice roll. - - --differential-idle-merge-effort=1000 - - --timely-progress-mode=${MZ_TIMELY_PROGRESS_MODE:-demand} - - --disable-telemetry - environment: - # You can, for example, add `pgwire=trace` or change `info` to `debug` to - # get more verbose logs. - - MZ_LOG_FILTER=pgwire=debug,info - - MZ_DEV=1 - zookeeper: - image: confluentinc/cp-zookeeper:5.5.4 - environment: - - ZOOKEEPER_CLIENT_PORT=2181 - kafka: - image: confluentinc/cp-enterprise-kafka:5.5.4 - ports: - - *kafka-internal - - *kafka-external - depends_on: [zookeeper] - environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT - - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092,EXTERNAL://0.0.0.0:9093 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://${KAFKA_HOST:-kafka}:9093 - - KAFKA_METRIC_REPORTERS=io.confluent.metrics.reporter.ConfluentMetricsReporter - - KAFKA_BROKER_ID=1 - - KAFKA_LOG_RETENTION_HOURS=-1 - - KAFKA_NUM_PARTITIONS=30 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS=kafka:9092 - - CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS=1 - - CONFLUENT_METRICS_REPORTER_TOPIC_CREATE=false - - KAFKA_JMX_PORT=9991 - schema-registry: - image: confluentinc/cp-schema-registry:5.5.4 - ports: - - *schema-registry - environment: - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 - - SCHEMA_REGISTRY_HOST_NAME=schema-registry - - SCHEMA_REGISTRY_LISTENERS=http://schema-registry:8081,http://localhost:8081 - depends_on: [zookeeper, kafka] - generator: - mzbuild: aggregation-data-generator - metric-verifier: - mzbuild: metric-verifier - environment: - - MZ_WORKERS=${MZ_WORKERS:-0} - - MZBENCH_GIT_REF=${MZBENCH_GIT_REF:-None} - - MZBENCH_ID=${MZBENCH_ID:-0} - # All monitoring - dashboard: - mzbuild: dashboard - propagate_uid_gid: true - environment: - - 'MATERIALIZED_URL=materialized:6875' - ports: - - *grafana - - *prometheus - volumes: - # ensure that data doesn't get lost across restarts - # data will be lost if you remove docker volumes (using nuke, for example) - - prometheus:/prometheus - # specialized configurations - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - -volumes: - prometheus: - -mzworkflows: - - start-services: - steps: - - step: start-services - services: [dashboard] - - step: start-services - services: [kafka, schema-registry, perf-dash-metrics] - - step: wait-for-tcp - host: kafka - port: 9092 - - step: wait-for-tcp - host: schema-registry - port: 8081 - - step: run - service: perf-dash-scraper - command: >- - create-topics - - step: run - service: metric-verifier - command: >- - create-topics - - step: run - service: perf-dash-create-views - - step: start-services - services: [perf-dash-web] - - setup-benchmark: - env: - AGGREGATION_GENERATOR_DISTRIBUTION: ${AGGREGATION_GENERATOR_DISTRIBUTION:-benchmark} - AGGREGATION_GENERATOR_NUM_RECORDS: ${AGGREGATION_GENERATOR_NUM_RECORDS:-175000000} - AGGREGATION_GENERATOR_PARALLELISM: ${AGGREGATION_GENERATOR_PARALLELISM:-40} - steps: - - step: workflow - workflow: start-services - - step: run - service: generator - command: >- - --parallelism=${AGGREGATION_GENERATOR_PARALLELISM} - --num-records=${AGGREGATION_GENERATOR_NUM_RECORDS} - --distribution=${AGGREGATION_GENERATOR_DISTRIBUTION} - - run-benchmark: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - METRIC_VERIFIER_TIMEOUT: ${METRIC_VERIFIER_TIMEOUT:-3600} - AGGREGATION_GENERATOR_NUM_RECORDS: ${AGGREGATION_GENERATOR_NUM_RECORDS:-175000000} - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: metric-verifier - daemon: true - command: >- - scrape - --timeout-seconds=${METRIC_VERIFIER_TIMEOUT} - "sum(mz_messages_ingested)" - ${AGGREGATION_GENERATOR_NUM_RECORDS} - - step: run - service: create-views - - step: wait - service: metric-verifier - expected_return_code: 0 - print_logs: true - - # The benchmark that we run in the cloud - benchmark-large: - steps: - - step: workflow - workflow: setup-benchmark-large - - step: workflow - workflow: run-benchmark-large - - setup-benchmark-large: - env: - AGGREGATION_GENERATOR_DISTRIBUTION: benchmark - AGGREGATION_GENERATOR_NUM_RECORDS: 7500000 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-large: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - AGGREGATION_GENERATOR_NUM_RECORDS: 7500000 - steps: - - step: workflow - workflow: run-benchmark - - # The benchmark that developers can run on their laptop - benchmark-medium: - steps: - - step: workflow - workflow: setup-benchmark-medium - - step: workflow - workflow: run-benchmark-medium - - setup-benchmark-medium: - env: - AGGREGATION_GENERATOR_DISTRIBUTION: medium - AGGREGATION_GENERATOR_NUM_RECORDS: 500000 - AGGREGATION_GENERATOR_PARALLELISM: 8 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-medium: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 900 - AGGREGATION_GENERATOR_NUM_RECORDS: 500000 - steps: - - step: workflow - workflow: run-benchmark - - # The smoketest benchmark that we run in CI - ci: - steps: - - step: workflow - workflow: setup-benchmark-ci - - step: workflow - workflow: run-benchmark-ci - - benchmark-ci: - steps: - - step: worfklow - workflow: ci - - setup-benchmark-ci: - env: - AGGREGATION_GENERATOR_DISTRIBUTION: small - AGGREGATION_GENERATOR_NUM_RECORDS: 100000 - AGGREGATION_GENERATOR_PARALLELISM: 4 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-ci: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 30 - AGGREGATION_GENERATOR_NUM_RECORDS: 100000 - steps: - - step: workflow - workflow: run-benchmark - - run-generator: - steps: - - step: run - service: generator - - rerun-benchmark: - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: create-views diff --git a/test/bench/aggregations/prometheus/prometheus.yml b/test/bench/aggregations/prometheus/prometheus.yml deleted file mode 100644 index 6950d2120bd27..0000000000000 --- a/test/bench/aggregations/prometheus/prometheus.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: materialized - scrape_interval: 1s - static_configs: - - targets: - - 'materialized:6875' diff --git a/test/bench/avro-insert/README.md b/test/bench/avro-insert/README.md deleted file mode 100644 index fee7efa8afbcb..0000000000000 --- a/test/bench/avro-insert/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# Avro Insert Benchmark - -This benchmark tests our ability to ingest Avro messages without any envelope type. In it's -current configuration, it assumes that Kafka topics should have 30 partitions and that you want 16 -worker threads. - -This benchmark is largely identical to the `avro-upsert` benchmark and was originally developed to -compare the relative performance of the `upsert` operator. - -## Running the Avro Insert Test - -To run the spec benchmark, please ensure that your machine has 16 cores and at least 96GB of -memory. Then run the following command: - - ./mzcompose run benchmark - -If you're running on a smaller machine, such as a laptop with 8 cores / 32GB of memory, you can -run the version of this benchmark designed for smaller machines: - - ./mzcompose run benchmark-medium - -You can also use [mzbench](../../../doc/developer/mzbench.md) to run comparative benchmarks on -different versions. See the `mzbench` documentation for more details but here is an example -invocation (run from the repository root): - - ./bin/mzbench -w --size medium avro-insert - -## Looking at Performance in Grafana - -Each benchmark will output something like the following: - -```sh -SUCCESS! seconds_taken=766, rows_per_sec=522193 -Grafana URL: http://localhost:3000/d/materialize-overview/materialize-overview?from=1612572459000&to=1612573285000&tz=UTC -``` - -The Grafana link will display performance metrics for materialize during the entirety of the -benchmark run. diff --git a/test/bench/avro-insert/create_views/Dockerfile b/test/bench/avro-insert/create_views/Dockerfile deleted file mode 100644 index 41007ae2b1144..0000000000000 --- a/test/bench/avro-insert/create_views/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -FROM postgres - -RUN mkdir /data -COPY views /data - -COPY create_views /usr/local/bin -ENTRYPOINT ["/usr/local/bin/create_views"] diff --git a/test/bench/avro-insert/create_views/create_views b/test/bench/avro-insert/create_views/create_views deleted file mode 100755 index e65f2437c98fe..0000000000000 --- a/test/bench/avro-insert/create_views/create_views +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -set -euo pipefail - -view_file="/data/${1:-insert_views.sql}" - -psql -h materialized -p 6875 -U materialize -f "${view_file}" materialize diff --git a/test/bench/avro-insert/create_views/mzbuild.yml b/test/bench/avro-insert/create_views/mzbuild.yml deleted file mode 100644 index 9dac8f160fbe4..0000000000000 --- a/test/bench/avro-insert/create_views/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: avro-insert-create-views -publish: false diff --git a/test/bench/avro-insert/create_views/views/insert_views.sql b/test/bench/avro-insert/create_views/views/insert_views.sql deleted file mode 100644 index a0bdc64eda533..0000000000000 --- a/test/bench/avro-insert/create_views/views/insert_views.sql +++ /dev/null @@ -1,26 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE source_insertavrotest -FROM KAFKA BROKER 'kafka:9092' -TOPIC 'insertavrotest' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE VIEW insertavrotest AS SELECT - "Key1Unused" AS Key1Unused, -- retaining AS long - "Key2Unused" AS Key2Unused, -- retaining AS long - (("OuterRecord")."Record1"."InnerRecord1"."Point")::float AS "Point1", -- converting long to float - (("OuterRecord")."Record1"."InnerRecord2"."Point")::float AS "Point2", -- converting long to float - (("OuterRecord")."Record2"."InnerRecord3"."Point")::float AS "Point3", -- converting long to float - (("OuterRecord")."Record2"."InnerRecord4"."Point")::float AS "Point4" -- converting long to float -FROM source_insertavrotest -WHERE "Key1Unused" = 0; - -CREATE INDEX insertavrotest_index ON insertavrotest(Key1Unused, Key2Unused); diff --git a/test/bench/avro-insert/mzcompose b/test/bench/avro-insert/mzcompose deleted file mode 100755 index 76f6976c3b4a5..0000000000000 --- a/test/bench/avro-insert/mzcompose +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. -# -# mzcompose — runs Docker Compose with Materialize customizations. - -exec "$(dirname "$0")/../../../bin/mzcompose" "$@" diff --git a/test/bench/avro-insert/mzcompose.yml b/test/bench/avro-insert/mzcompose.yml deleted file mode 100644 index 6972f1fc3d57f..0000000000000 --- a/test/bench/avro-insert/mzcompose.yml +++ /dev/null @@ -1,294 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - - -# Map from host-port:internal port -# -# This mostly just shows all the ports that are available to the host system, if you want -# to change these you must restart the docker-compose cluster. -x-port-mappings: - - &kafka-internal ${KAFKA_PORT:-9092} - - &kafka-external 9093:9093 - - &schema-registry ${SR_PORT:-8081:8081} - - &materialized ${MZ_PORT:-6875} - - &grafana 3000:3000 - - &prometheus 9090 - - &perf-metrics ${PERF_METRICS_PORT:-8675} - - &perf-dash-web ${PERF_DASH_PORT:-8875:8875} - -version: '3.7' -services: - create-views: - mzbuild: avro-insert-create-views - perf-dash-metrics: - mzbuild: materialized - ports: - - *perf-metrics - command: - - --disable-telemetry - - --listen-addr=0.0.0.0:${PERF_METRICS_PORT:-8675} - - --logical-compaction-window=1ms - environment: - - MZ_DEV=1 - perf-dash-scraper: - mzbuild: perf-dash-scraper - command: scrape - perf-dash-create-views: - mzbuild: perf-dash-create-views - perf-dash-web: - mzbuild: perf-dash-web - ports: - - *perf-dash-web - materialized: - mzbuild: materialized - ports: - - *materialized - command: - - --workers=${MZ_WORKERS:-16} - - --logical-compaction-window=1ms - # We want this to eventually count up to the size of the largest batch in - # an arrangement. This number represents a tradeoff between proactive - # merging (which takes time) and low latency. - # - # 1000 was chosen by fair dice roll. - - --differential-idle-merge-effort=1000 - - --timely-progress-mode=${MZ_TIMELY_PROGRESS_MODE:-demand} - - --disable-telemetry - environment: - # You can, for example, add `pgwire=trace` or change `info` to `debug` to - # get more verbose logs. - - MZ_LOG_FILTER=pgwire=debug,info - - MZ_DEV=1 - zookeeper: - image: confluentinc/cp-zookeeper:5.5.4 - environment: - - ZOOKEEPER_CLIENT_PORT=2181 - kafka: - image: confluentinc/cp-enterprise-kafka:5.5.4 - ports: - - *kafka-internal - - *kafka-external - depends_on: [zookeeper] - environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT - - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092,EXTERNAL://0.0.0.0:9093 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://${KAFKA_HOST:-kafka}:9093 - - KAFKA_METRIC_REPORTERS=io.confluent.metrics.reporter.ConfluentMetricsReporter - - KAFKA_BROKER_ID=1 - - KAFKA_LOG_RETENTION_HOURS=-1 - - KAFKA_NUM_PARTITIONS=30 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS=kafka:9092 - - CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS=1 - # To avoid race condition with control-center - - CONFLUENT_METRICS_REPORTER_TOPIC_CREATE=false - - KAFKA_JMX_PORT=9991 - schema-registry: - image: confluentinc/cp-schema-registry:5.5.4 - ports: - - *schema-registry - environment: - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 - - SCHEMA_REGISTRY_HOST_NAME=schema-registry - - SCHEMA_REGISTRY_LISTENERS=http://schema-registry:8081,http://localhost:8081 - depends_on: [zookeeper, kafka] - kafka-avro-generator: - mzbuild: kafka-avro-generator - metric-verifier: - mzbuild: metric-verifier - environment: - - MZ_WORKERS=${MZ_WORKERS:-0} - - MZBENCH_GIT_REF=${MZBENCH_GIT_REF:-None} - - MZBENCH_ID=${MZBENCH_ID:-0} - # All monitoring - dashboard: - mzbuild: dashboard - propagate_uid_gid: true - environment: - - 'MATERIALIZED_URL=materialized:6875' - ports: - - *grafana - - *prometheus - volumes: - # ensure that data doesn't get lost across restarts - # data will be lost if you remove docker volumes (using nuke, for example) - - prometheus:/prometheus - - grafana:/var/lib/grafana - # specialized configurations - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - -volumes: - grafana: - prometheus: - -mzworkflows: - - start-services: - steps: - - step: start-services - services: [dashboard] - - step: start-services - services: [kafka, schema-registry, perf-dash-metrics] - - step: wait-for-tcp - host: kafka - port: 9092 - - step: wait-for-tcp - host: schema-registry - port: 8081 - - step: run - service: perf-dash-scraper - command: >- - create-topics - - step: run - service: metric-verifier - command: >- - create-topics - - step: run - service: perf-dash-create-views - - step: start-services - services: [perf-dash-web] - - setup-benchmark: - env: - AVRO_GENERATOR_DISTRIBUTION: ${AVRO_GENERATOR_DISTRIBUTION:-benchmark} - AVRO_GENERATOR_NUM_RECORDS: ${AVRO_GENERATOR_NUM_RECORDS:-400000000} - AVRO_GENERATOR_PARALLELISM: ${AVRO_GENERATOR_PARALLELISM:-40} - steps: - - step: workflow - workflow: start-services - - step: run - service: kafka-avro-generator - command: >- - --parallelism=${AVRO_GENERATOR_PARALLELISM} - --num-records=${AVRO_GENERATOR_NUM_RECORDS} - --distribution=${AVRO_GENERATOR_DISTRIBUTION} - --topic=insertavrotest - - run-benchmark: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - METRIC_VERIFIER_TIMEOUT: ${METRIC_VERIFIER_TIMEOUT:-3600} - AVRO_GENERATOR_NUM_RECORDS: ${AVRO_GENERATOR_NUM_RECORDS:-400000000} - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: metric-verifier - daemon: true - command: >- - scrape - --timeout-seconds=${METRIC_VERIFIER_TIMEOUT} - "sum(mz_messages_ingested)" - ${AVRO_GENERATOR_NUM_RECORDS} - - step: run - service: create-views - - step: wait - service: metric-verifier - expected_return_code: 0 - print_logs: true - - # The benchmark that we run in the cloud - benchmark-large: - steps: - - step: workflow - workflow: setup-benchmark-large - - step: workflow - workflow: run-benchmark-large - - setup-benchmark-large: - env: - AVRO_GENERATOR_DISTRIBUTION: benchmark - AVRO_GENERATOR_NUM_RECORDS: 400000000 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-large: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - AVRO_GENERATOR_NUM_RECORDS: 400000000 - steps: - - step: workflow - workflow: run-benchmark - - # The benchmark that developers can run on their laptop - benchmark-medium: - steps: - - step: workflow - workflow: setup-benchmark-medium - - step: workflow - workflow: run-benchmark-medium - - setup-benchmark-medium: - env: - AVRO_GENERATOR_DISTRIBUTION: medium - AVRO_GENERATOR_NUM_RECORDS: 20000000 - AVRO_GENERATOR_PARALLELISM: 8 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-medium: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 900 - AVRO_GENERATOR_NUM_RECORDS: 20000000 - steps: - - step: workflow - workflow: run-benchmark - - # The smoketest benchmark that we run in CI - ci: - steps: - - step: workflow - workflow: setup-benchmark-ci - - step: workflow - workflow: run-benchmark-ci - - benchmark-ci: - steps: - - step: worfklow - workflow: ci - - setup-benchmark-ci: - env: - AVRO_GENERATOR_DISTRIBUTION: small - AVRO_GENERATOR_NUM_RECORDS: 100000 - AVRO_GENERATOR_PARALLELISM: 4 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-ci: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 30 - AVRO_GENERATOR_NUM_RECORDS: 100000 - steps: - - step: workflow - workflow: run-benchmark - - run-generator: - steps: - - step: run - service: kafka-avro-generator - - rerun-benchmark: - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: create-views diff --git a/test/bench/avro-insert/prometheus/prometheus.yml b/test/bench/avro-insert/prometheus/prometheus.yml deleted file mode 100644 index 6950d2120bd27..0000000000000 --- a/test/bench/avro-insert/prometheus/prometheus.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: materialized - scrape_interval: 1s - static_configs: - - targets: - - 'materialized:6875' diff --git a/test/bench/avro-upsert/README.md b/test/bench/avro-upsert/README.md deleted file mode 100644 index d38acd97d6001..0000000000000 --- a/test/bench/avro-upsert/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Avro Upsert Benchmark - -This benchmark tests our ability to ingest Avro Upsert messages. In it's current configuration, it -assumes that Kafka topics should have 30 partitions and that you want 16 worker threads. - -## Running the Avro Upsert Test - -To run the spec benchmark, please ensure that your machine has 16 cores and at least 96GB of -memory. Then run the following command: - - ./mzcompose run benchmark - -If you're running on a smaller machine, such as a laptop with 8 cores / 32GB of memory, you can -run the version of this benchmark designed for smaller machines: - - ./mzcompose run benchmark-medium - -You can also use [mzbench](../../../doc/developer/mzbench.md) to run comparative benchmarks on -different versions. See the `mzbench` documentation for more details but here is an example -invocation (run from the repository root): - - ./bin/mzbench -w --size medium avro-upsert - -## Looking at Performance - -Each benchmark will output something like the following: - -```sh -SUCCESS! seconds_taken=766, rows_per_sec=522193 -Grafana URL: http://localhost:3000/d/materialize-overview/materialize-overview?from=1612572459000&to=1612573285000&tz=UTC -``` - -The Grafana link will display performance metrics for materialize during the entirety of the -benchmark run. diff --git a/test/bench/avro-upsert/create_views/Dockerfile b/test/bench/avro-upsert/create_views/Dockerfile deleted file mode 100644 index 41007ae2b1144..0000000000000 --- a/test/bench/avro-upsert/create_views/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -FROM postgres - -RUN mkdir /data -COPY views /data - -COPY create_views /usr/local/bin -ENTRYPOINT ["/usr/local/bin/create_views"] diff --git a/test/bench/avro-upsert/create_views/create_views b/test/bench/avro-upsert/create_views/create_views deleted file mode 100755 index b868f7c2eb866..0000000000000 --- a/test/bench/avro-upsert/create_views/create_views +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -set -euo pipefail - -view_file="/data/${1:-upsert_views.sql}" - -psql -v ON_ERROR_STOP=1 -h materialized -p 6875 -U materialize -f "${view_file}" materialize diff --git a/test/bench/avro-upsert/create_views/mzbuild.yml b/test/bench/avro-upsert/create_views/mzbuild.yml deleted file mode 100644 index e8efc2ce2f2fa..0000000000000 --- a/test/bench/avro-upsert/create_views/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: avro-upsert-create-views -publish: false diff --git a/test/bench/avro-upsert/create_views/views/insert_views.sql b/test/bench/avro-upsert/create_views/views/insert_views.sql deleted file mode 100644 index 31bba5a9504ea..0000000000000 --- a/test/bench/avro-upsert/create_views/views/insert_views.sql +++ /dev/null @@ -1,26 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE source_insertavrotest -FROM KAFKA BROKER 'kafka:9092' -TOPIC 'upsertavrotest' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE VIEW insertavrotest AS SELECT - "Key1Unused" AS Key1Unused, -- retaining AS long - "Key2Unused" AS Key2Unused, -- retaining AS long - (("OuterRecord")."Record1"."InnerRecord1"."Point")::float AS "Point1", -- converting long to float - (("OuterRecord")."Record1"."InnerRecord2"."Point")::float AS "Point2", -- converting long to float - (("OuterRecord")."Record2"."InnerRecord3"."Point")::float AS "Point3", -- converting long to float - (("OuterRecord")."Record2"."InnerRecord4"."Point")::float AS "Point4" -- converting long to float -FROM source_insertavrotest -WHERE "Key1Unused" = 0; - -CREATE INDEX insertavrotest_index ON insertavrotest(Key1Unused, Key2Unused); diff --git a/test/bench/avro-upsert/create_views/views/upsert_views.sql b/test/bench/avro-upsert/create_views/views/upsert_views.sql deleted file mode 100644 index 7575984fd857c..0000000000000 --- a/test/bench/avro-upsert/create_views/views/upsert_views.sql +++ /dev/null @@ -1,26 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE source_upsertavrotest -FROM KAFKA BROKER 'kafka:9092' -TOPIC 'upsertavrotest' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE UPSERT -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081'; - -CREATE VIEW upsertavrotest AS SELECT - "Key1" AS Key1, -- retaining AS long - "Key2" AS Key2, -- retaining AS long - (("OuterRecord")."Record1"."InnerRecord1"."Point")::float AS "Point1", -- converting long to float - (("OuterRecord")."Record1"."InnerRecord2"."Point")::float AS "Point2", -- converting long to float - (("OuterRecord")."Record2"."InnerRecord3"."Point")::float AS "Point3", -- converting long to float - (("OuterRecord")."Record2"."InnerRecord4"."Point")::float AS "Point4" -- converting long to float -FROM source_upsertavrotest; - -CREATE INDEX upsertavrotest_index ON upsertavrotest(Key1, Key2); diff --git a/test/bench/avro-upsert/mzcompose b/test/bench/avro-upsert/mzcompose deleted file mode 100755 index 76f6976c3b4a5..0000000000000 --- a/test/bench/avro-upsert/mzcompose +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. -# -# mzcompose — runs Docker Compose with Materialize customizations. - -exec "$(dirname "$0")/../../../bin/mzcompose" "$@" diff --git a/test/bench/avro-upsert/mzcompose.yml b/test/bench/avro-upsert/mzcompose.yml deleted file mode 100644 index 6a5b71400171c..0000000000000 --- a/test/bench/avro-upsert/mzcompose.yml +++ /dev/null @@ -1,291 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - - -# Map from host-port:internal port -# -# This mostly just shows all the ports that are available to the host system, if you want -# to change these you must restart the docker-compose cluster. -x-port-mappings: - - &kafka-internal ${KAFKA_PORT:-9092} - - &kafka-external 9093:9093 - - &schema-registry ${SR_PORT:-8081:8081} - - &materialized ${MZ_PORT:-6875} - - &grafana 3000:3000 - - &prometheus 9090 - - &perf-metrics ${PERF_METRICS_PORT:-8675} - - &perf-dash-web ${PERF_DASH_PORT:-8875:8875} - -version: '3.7' -services: - create-views: - mzbuild: avro-upsert-create-views - perf-dash-metrics: - mzbuild: materialized - ports: - - *perf-metrics - command: - - --disable-telemetry - - --listen-addr=0.0.0.0:${PERF_METRICS_PORT:-8675} - - --logical-compaction-window=1ms - environment: - - MZ_DEV=1 - perf-dash-scraper: - mzbuild: perf-dash-scraper - command: scrape - perf-dash-create-views: - mzbuild: perf-dash-create-views - perf-dash-web: - mzbuild: perf-dash-web - ports: - - *perf-dash-web - materialized: - mzbuild: materialized - ports: - - *materialized - command: - - --workers=${MZ_WORKERS:-16} - - --logical-compaction-window=1ms - # We want this to eventually count up to the size of the largest batch in - # an arrangement. This number represents a tradeoff between proactive - # merging (which takes time) and low latency. - # - # 1000 was chosen by fair dice roll. - - --differential-idle-merge-effort=1000 - - --timely-progress-mode=${MZ_TIMELY_PROGRESS_MODE:-demand} - - --disable-telemetry - environment: - # You can, for example, add `pgwire=trace` or change `info` to `debug` to - # get more verbose logs. - - MZ_LOG_FILTER=pgwire=debug,info - - MZ_DEV=1 - zookeeper: - image: confluentinc/cp-zookeeper:5.5.4 - environment: - - ZOOKEEPER_CLIENT_PORT=2181 - kafka: - image: confluentinc/cp-enterprise-kafka:5.5.4 - ports: - - *kafka-internal - - *kafka-external - depends_on: [zookeeper] - environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT - - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092,EXTERNAL://0.0.0.0:9093 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://${KAFKA_HOST:-kafka}:9093 - - KAFKA_METRIC_REPORTERS=io.confluent.metrics.reporter.ConfluentMetricsReporter - - KAFKA_BROKER_ID=1 - - KAFKA_LOG_RETENTION_HOURS=-1 - - KAFKA_NUM_PARTITIONS=30 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS=kafka:9092 - - CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS=1 - - CONFLUENT_METRICS_REPORTER_TOPIC_CREATE=false - - KAFKA_JMX_PORT=9991 - schema-registry: - image: confluentinc/cp-schema-registry:5.5.4 - ports: - - *schema-registry - environment: - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 - - SCHEMA_REGISTRY_HOST_NAME=schema-registry - - SCHEMA_REGISTRY_LISTENERS=http://schema-registry:8081,http://localhost:8081 - depends_on: [zookeeper, kafka] - kafka-avro-generator: - mzbuild: kafka-avro-generator - metric-verifier: - mzbuild: metric-verifier - environment: - - MZ_WORKERS=${MZ_WORKERS:-0} - - MZBENCH_GIT_REF=${MZBENCH_GIT_REF:-None} - - MZBENCH_ID=${MZBENCH_ID:-0} - # All monitoring - dashboard: - mzbuild: dashboard - propagate_uid_gid: true - environment: - - 'MATERIALIZED_URL=materialized:6875' - ports: - - *grafana - - *prometheus - volumes: - # ensure that data doesn't get lost across restarts - # data will be lost if you remove docker volumes (using nuke, for example) - - prometheus:/prometheus - # specialized configurations - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - -volumes: - prometheus: - -mzworkflows: - - start-services: - steps: - - step: start-services - services: [dashboard] - - step: start-services - services: [kafka, schema-registry, perf-dash-metrics] - - step: wait-for-tcp - host: kafka - port: 9092 - - step: wait-for-tcp - host: schema-registry - port: 8081 - - step: run - service: perf-dash-scraper - command: >- - create-topics - - step: run - service: metric-verifier - command: >- - create-topics - - step: run - service: perf-dash-create-views - - step: start-services - services: [perf-dash-web] - - setup-benchmark: - env: - UPSERT_GENERATOR_DISTRIBUTION: ${UPSERT_GENERATOR_DISTRIBUTION:-benchmark} - UPSERT_GENERATOR_NUM_RECORDS: ${UPSERT_GENERATOR_NUM_RECORDS:-400000000} - UPSERT_GENERATOR_PARALLELISM: ${UPSERT_GENERATOR_PARALLELISM:-40} - steps: - - step: workflow - workflow: start-services - - step: run - service: kafka-avro-generator - command: >- - --parallelism=${UPSERT_GENERATOR_PARALLELISM} - --num-records=${UPSERT_GENERATOR_NUM_RECORDS} - --distribution=${UPSERT_GENERATOR_DISTRIBUTION} - --topic=upsertavrotest - - run-benchmark: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - METRIC_VERIFIER_TIMEOUT: ${METRIC_VERIFIER_TIMEOUT:-3600} - UPSERT_GENERATOR_NUM_RECORDS: ${UPSERT_GENERATOR_NUM_RECORDS:-400000000} - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: metric-verifier - daemon: true - command: >- - scrape - --timeout-seconds=${METRIC_VERIFIER_TIMEOUT} - "sum(mz_messages_ingested)" - ${UPSERT_GENERATOR_NUM_RECORDS} - - step: run - service: create-views - - step: wait - service: metric-verifier - expected_return_code: 0 - print_logs: true - - # The benchmark that we run in the cloud - benchmark-large: - steps: - - step: workflow - workflow: setup-benchmark-large - - step: workflow - workflow: run-benchmark-large - - setup-benchmark-large: - env: - UPSERT_GENERATOR_DISTRIBUTION: benchmark - UPSERT_GENERATOR_NUM_RECORDS: 400000000 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-large: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - UPSERT_GENERATOR_NUM_RECORDS: 400000000 - steps: - - step: workflow - workflow: run-benchmark - - # The benchmark that developers can run on their laptop - benchmark-medium: - steps: - - step: workflow - workflow: setup-benchmark-medium - - step: workflow - workflow: run-benchmark-medium - - setup-benchmark-medium: - env: - UPSERT_GENERATOR_DISTRIBUTION: medium - UPSERT_GENERATOR_NUM_RECORDS: 20000000 - UPSERT_GENERATOR_PARALLELISM: 8 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-medium: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 900 - UPSERT_GENERATOR_NUM_RECORDS: 20000000 - steps: - - step: workflow - workflow: run-benchmark - - # The smoketest benchmark that we run in CI - ci: - steps: - - step: workflow - workflow: setup-benchmark-ci - - step: workflow - workflow: run-benchmark-ci - - benchmark-ci: - steps: - - step: worfklow - workflow: ci - - setup-benchmark-ci: - env: - UPSERT_GENERATOR_DISTRIBUTION: small - UPSERT_GENERATOR_NUM_RECORDS: 100000 - UPSERT_GENERATOR_PARALLELISM: 4 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-ci: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 30 - UPSERT_GENERATOR_NUM_RECORDS: 100000 - steps: - - step: workflow - workflow: run-benchmark - - run-generator: - steps: - - step: run - service: kafka-avro-generator - - rerun-benchmark: - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: create-views diff --git a/test/bench/avro-upsert/prometheus/prometheus.yml b/test/bench/avro-upsert/prometheus/prometheus.yml deleted file mode 100644 index 6950d2120bd27..0000000000000 --- a/test/bench/avro-upsert/prometheus/prometheus.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: materialized - scrape_interval: 1s - static_configs: - - targets: - - 'materialized:6875' diff --git a/test/bench/kafka-sink-avro-debezium/README.md b/test/bench/kafka-sink-avro-debezium/README.md deleted file mode 100644 index 1ba5089da6196..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# Kafka Avro Debezium Sink Benchmark - -This benchmark tests our ability to write messages out to Kafka. In its -current configuration, it uses a topic with only one partition as input and a -topic with only one partition as output. We use only one input topic because -that is enough to saturate writing. We use only one output topic because the -current Kafka sink will only ever write from one timely worker. - -## Running the Kafka Avro Debezium Sink Test - -To run the spec benchmark run the following command: - - ./mzcompose run benchmark - -If you're running on a smaller machine, you can run the version of this -benchmark designed for smaller machines: - - ./mzcompose run benchmark-medium - -You can also use [mzbench](../../../doc/developer/mzbench.md) to run comparative benchmarks on -different versions. See the `mzbench` documentation for more details but here is an example -invocation (run from the repository root): - - ./bin/mzbench -w --size medium kafka-sink-avro-debezium - -## Looking at Performance - -Each benchmark will output something like the following: - -```sh -SUCCESS! seconds_taken=766, rows_per_sec=522193 -Grafana URL: http://localhost:3000/d/materialize-overview/materialize-overview?from=1612572459000&to=1612573285000&tz=UTC -``` - -The Grafana link will display performance metrics for materialize during the -entirety of the benchmark run. diff --git a/test/bench/kafka-sink-avro-debezium/create_sink/Dockerfile b/test/bench/kafka-sink-avro-debezium/create_sink/Dockerfile deleted file mode 100644 index 89c848aa1ba22..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/create_sink/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -FROM postgres - -RUN mkdir /data -COPY sinks /data - -COPY create_sink /usr/local/bin -ENTRYPOINT ["/usr/local/bin/create_sink"] diff --git a/test/bench/kafka-sink-avro-debezium/create_sink/create_sink b/test/bench/kafka-sink-avro-debezium/create_sink/create_sink deleted file mode 100755 index c14728e9b6f95..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/create_sink/create_sink +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -set -euo pipefail - -sink_file="/data/${1:-no-consistency-no-exactly-once-sink.sql}" - -psql -v ON_ERROR_STOP=1 -h materialized -p 6875 -U materialize -f "${sink_file}" materialize diff --git a/test/bench/kafka-sink-avro-debezium/create_sink/mzbuild.yml b/test/bench/kafka-sink-avro-debezium/create_sink/mzbuild.yml deleted file mode 100644 index 13f91800672a8..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/create_sink/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: kafka-sink-avro-debezium-create-sink -publish: false diff --git a/test/bench/kafka-sink-avro-debezium/create_sink/sinks/no-consistency-no-exactly-once-sink.sql b/test/bench/kafka-sink-avro-debezium/create_sink/sinks/no-consistency-no-exactly-once-sink.sql deleted file mode 100644 index 2b44264555d1b..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/create_sink/sinks/no-consistency-no-exactly-once-sink.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Copyright Materialize, Inc. and contributors. All rights reserved. --- --- Use of this software is governed by the Business Source License --- included in the LICENSE file at the root of this repository. --- --- As of the Change Date specified in that file, in accordance with --- the Business Source License, use of this software will be governed --- by the Apache License, Version 2.0. - -CREATE SOURCE append_source -FROM KAFKA BROKER 'kafka:9092' -TOPIC 'plain_avro_records' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE NONE; - -CREATE SINK kafka_sink FROM append_source -INTO KAFKA BROKER 'kafka:9092' TOPIC 'debezium_output' -FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://schema-registry:8081' -ENVELOPE DEBEZIUM; diff --git a/test/bench/kafka-sink-avro-debezium/mzcompose b/test/bench/kafka-sink-avro-debezium/mzcompose deleted file mode 100755 index 76f6976c3b4a5..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/mzcompose +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. -# -# mzcompose — runs Docker Compose with Materialize customizations. - -exec "$(dirname "$0")/../../../bin/mzcompose" "$@" diff --git a/test/bench/kafka-sink-avro-debezium/mzcompose.yml b/test/bench/kafka-sink-avro-debezium/mzcompose.yml deleted file mode 100644 index b6038564bea43..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/mzcompose.yml +++ /dev/null @@ -1,311 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - - -# Map from host-port:internal port -# -# This mostly just shows all the ports that are available to the host system, if you want -# to change these you must restart the docker-compose cluster. -x-port-mappings: - - &kafka-internal ${KAFKA_PORT:-9092} - - &kafka-external 9093:9093 - - &schema-registry ${SR_PORT:-8081:8081} - - &materialized ${MZ_PORT:-6875} - - &materialized-external ${MZ_PORT:-6875:6875} - - &grafana 3000:3000 - - &prometheus 9090 - - &perf-metrics ${PERF_METRICS_PORT:-8675} - - &perf-dash-web ${PERF_DASH_PORT:-8875:8875} - -version: '3.7' -services: - create-sink: - mzbuild: kafka-sink-avro-debezium-create-sink - perf-dash-metrics: - mzbuild: materialized - ports: - - *perf-metrics - command: - - --disable-telemetry - - --listen-addr=0.0.0.0:${PERF_METRICS_PORT:-8675} - - --logical-compaction-window=1ms - environment: - - MZ_DEV=1 - perf-dash-scraper: - mzbuild: perf-dash-scraper - command: scrape - perf-dash-create-views: - mzbuild: perf-dash-create-views - perf-dash-web: - mzbuild: perf-dash-web - ports: - - *perf-dash-web - materialized: - mzbuild: materialized - ports: - - *materialized - - *materialized-external - command: - - --workers=${MZ_WORKERS:-16} - - --disable-telemetry - environment: - # You can, for example, add `pgwire=trace` or change `info` to `debug` to - # get more verbose logs. - - MZ_LOG_FILTER=dataflow::sink::kafka=debug,librdkafka=warn,rdkafka=warn,sql_parser=warn,hyper=warn,info - - MZ_DEV=1 - zookeeper: - image: confluentinc/cp-zookeeper:5.5.4 - environment: - - ZOOKEEPER_CLIENT_PORT=2181 - kafka: - image: confluentinc/cp-enterprise-kafka:5.5.4 - ports: - - *kafka-internal - - *kafka-external - depends_on: [zookeeper] - environment: - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT - - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092,EXTERNAL://0.0.0.0:9093 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://${KAFKA_HOST:-kafka}:9093 - - KAFKA_METRIC_REPORTERS=io.confluent.metrics.reporter.ConfluentMetricsReporter - - KAFKA_BROKER_ID=1 - - KAFKA_LOG_RETENTION_HOURS=-1 - - KAFKA_NUM_PARTITIONS=1 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS=kafka:9092 - - CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS=1 - - CONFLUENT_METRICS_REPORTER_TOPIC_CREATE=false - - KAFKA_JMX_PORT=9991 - schema-registry: - image: confluentinc/cp-schema-registry:5.5.4 - ports: - - *schema-registry - environment: - - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 - - SCHEMA_REGISTRY_HOST_NAME=schema-registry - - SCHEMA_REGISTRY_LISTENERS=http://schema-registry:8081,http://localhost:8081 - depends_on: [zookeeper, kafka] - kafka-avro-generator: - mzbuild: kafka-avro-generator - metric-verifier: - mzbuild: metric-verifier - environment: - - MZ_WORKERS=${MZ_WORKERS:-0} - - MZBENCH_GIT_REF=${MZBENCH_GIT_REF:-None} - - MZBENCH_ID=${MZBENCH_ID:-0} - # All monitoring - dashboard: - mzbuild: dashboard - propagate_uid_gid: true - environment: - - 'MATERIALIZED_URL=materialized:6875' - ports: - - *grafana - - *prometheus - volumes: - # ensure that data doesn't get lost across restarts - # data will be lost if you remove docker volumes (using nuke, for example) - - prometheus:/prometheus - # specialized configurations - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - -volumes: - prometheus: - -mzworkflows: - - start-services: - steps: - - step: start-services - services: [dashboard] - - step: start-services - services: [kafka, schema-registry, perf-dash-metrics] - - step: wait-for-tcp - host: kafka - port: 9092 - - step: wait-for-tcp - host: schema-registry - port: 8081 - - step: run - service: perf-dash-scraper - command: >- - create-topics - - step: run - service: metric-verifier - command: >- - create-topics - - step: run - service: perf-dash-create-views - - step: start-services - services: [perf-dash-web] - - setup-benchmark: - env: - DATA_GENERATOR_DISTRIBUTION: ${DATA_GENERATOR_DISTRIBUTION:-benchmark} - DATA_GENERATOR_NUM_RECORDS: ${DATA_GENERATOR_NUM_RECORDS:-400000000} - DATA_GENERATOR_PARALLELISM: ${DATA_GENERATOR_PARALLELISM:-40} - steps: - - step: workflow - workflow: start-services - - step: run - service: kafka-avro-generator - command: >- - --parallelism=${DATA_GENERATOR_PARALLELISM} - --num-records=${DATA_GENERATOR_NUM_RECORDS} - --distribution=${DATA_GENERATOR_DISTRIBUTION} - --topic=plain_avro_records - - run-benchmark: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - METRIC_VERIFIER_TIMEOUT: ${METRIC_VERIFIER_TIMEOUT:-3600} - DATA_GENERATOR_NUM_RECORDS: ${DATA_GENERATOR_NUM_RECORDS:-400000000} - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: metric-verifier - daemon: true - command: >- - scrape - --timeout-seconds=${METRIC_VERIFIER_TIMEOUT} - "sum(mz_kafka_messages_sent_total)" - ${DATA_GENERATOR_NUM_RECORDS} - - step: run - service: create-sink - - step: wait - service: metric-verifier - expected_return_code: 0 - print_logs: true - - # The benchmark that we run in the cloud - benchmark-large: - steps: - - step: workflow - workflow: setup-benchmark-large - - step: workflow - workflow: run-benchmark-large - - setup-benchmark-large: - env: - DATA_GENERATOR_DISTRIBUTION: benchmark - DATA_GENERATOR_NUM_RECORDS: 400000000 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-large: - env: - MZ_WORKERS: ${MZ_WORKERS:-16} - DATA_GENERATOR_NUM_RECORDS: 400000000 - steps: - - step: workflow - workflow: run-benchmark - - # The benchmark that developers can run on their laptop - benchmark-medium: - steps: - - step: workflow - workflow: setup-benchmark-medium - - step: workflow - workflow: run-benchmark-medium - - setup-benchmark-medium: - env: - DATA_GENERATOR_DISTRIBUTION: medium - DATA_GENERATOR_NUM_RECORDS: 20000000 - DATA_GENERATOR_PARALLELISM: 8 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-medium: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 900 - DATA_GENERATOR_NUM_RECORDS: 20000000 - steps: - - step: workflow - workflow: run-benchmark - - # Tiny dataset for quick iteration in dev build mode - benchmark-tiny: - steps: - - step: workflow - workflow: setup-benchmark-tiny - - step: workflow - workflow: run-benchmark-tiny - - setup-benchmark-tiny: - env: - DATA_GENERATOR_DISTRIBUTION: medium - DATA_GENERATOR_NUM_RECORDS: 2000 - DATA_GENERATOR_PARALLELISM: 1 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-tiny: - env: - MZ_WORKERS: ${MZ_WORKERS:-5} - METRIC_VERIFIER_TIMEOUT: 900 - DATA_GENERATOR_NUM_RECORDS: 2000 - steps: - - step: workflow - workflow: run-benchmark - - # The smoketest benchmark that we run in CI - ci: - steps: - - step: workflow - workflow: setup-benchmark-ci - - step: workflow - workflow: run-benchmark-ci - - benchmark-ci: - steps: - - step: worfklow - workflow: ci - - setup-benchmark-ci: - env: - DATA_GENERATOR_DISTRIBUTION: small - DATA_GENERATOR_NUM_RECORDS: 100000 - DATA_GENERATOR_PARALLELISM: 4 - steps: - - step: workflow - workflow: setup-benchmark - - run-benchmark-ci: - env: - MZ_WORKERS: ${MZ_WORKERS:-8} - METRIC_VERIFIER_TIMEOUT: 30 - DATA_GENERATOR_NUM_RECORDS: 100000 - steps: - - step: workflow - workflow: run-benchmark - - run-generator: - steps: - - step: run - service: kafka-avro-generator - - rerun-benchmark: - steps: - - step: remove-services - services: [materialized, perf-dash-scraper] - destroy_volumes: true - - step: start-services - services: [materialized, perf-dash-scraper] - - step: run - service: create-sink diff --git a/test/bench/kafka-sink-avro-debezium/prometheus/prometheus.yml b/test/bench/kafka-sink-avro-debezium/prometheus/prometheus.yml deleted file mode 100644 index 6950d2120bd27..0000000000000 --- a/test/bench/kafka-sink-avro-debezium/prometheus/prometheus.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: materialized - scrape_interval: 1s - static_configs: - - targets: - - 'materialized:6875' diff --git a/test/bench/util/kafka-avro-generator/Dockerfile b/test/bench/util/kafka-avro-generator/Dockerfile deleted file mode 100644 index d2e1a971f1c36..0000000000000 --- a/test/bench/util/kafka-avro-generator/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -MZFROM kgen - -FROM python:3.9-slim - -COPY --from=0 /usr/local/bin/kgen /usr/local/bin - -RUN mkdir -p /usr/share/generator/distributions -COPY distributions /usr/share/generator/distributions - -COPY generate_records /usr/local/bin -ENTRYPOINT ["generate_records"] diff --git a/test/bench/util/kafka-avro-generator/distributions/benchmark/key-distribution.json b/test/bench/util/kafka-avro-generator/distributions/benchmark/key-distribution.json deleted file mode 100644 index 2547ba99814ca..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/benchmark/key-distribution.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "com.acme.avro.testrecordkey::Key1": [0, 100], - "com.acme.avro.testrecordkey::Key2": [0, 250000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/benchmark/key-schema.json b/test/bench/util/kafka-avro-generator/distributions/benchmark/key-schema.json deleted file mode 100644 index c156c8ae85838..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/benchmark/key-schema.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "testrecordkey", - "type" : "record", - "namespace": "com.acme.avro", - "fields": [ - { - "name": "Key1", - "type": "long" - }, - { - "name": "Key2", - "type": "long" - } - ] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/benchmark/value-distribution.json b/test/bench/util/kafka-avro-generator/distributions/benchmark/value-distribution.json deleted file mode 100644 index 1d1f77be38375..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/benchmark/value-distribution.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "com.acme.avro.testrecord::Key1Unused": [0, 100], - "com.acme.avro.testrecord::Key2Unused": [0, 250000], - "com.acme.avro.InnerRecord1::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord2::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord3::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord4::Point": [10000, 10000000000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/benchmark/value-schema.json b/test/bench/util/kafka-avro-generator/distributions/benchmark/value-schema.json deleted file mode 100644 index 88c958d85a892..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/benchmark/value-schema.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "name": "testrecord", - "type": "record", - "namespace": "com.acme.avro", - "fields": [ - { - "name": "Key1Unused", - "type": "long" - }, - { - "name": "Key2Unused", - "type": "long" - }, - { - "name": "OuterRecord", - "type": { - "name": "OuterRecord", - "type": "record", - "fields": [ - { - "name": "Record1", - "type": { - "name": "Record1", - "type": "record", - "fields": [ - { - "name": "InnerRecord1", - "type": { - "name": "InnerRecord1", - "type": "record", - "fields": [ - { - "name": "Point", - "type": "long" - } - ] - } - }, - { - "name": "InnerRecord2", - "type": { - "name": "InnerRecord2", - "type": "record", - "fields": [ - { - "name": "Point", - "type": "long" - } - ] - } - } - ] - } - }, - { - "name": "Record2", - "type": { - "name": "Record2", - "type": "record", - "fields": [ - { - "name": "InnerRecord3", - "type": { - "name": "InnerRecord3", - "type": "record", - "fields": [ - { - "name": "Point", - "type": "long" - } - ] - } - }, - { - "name": "InnerRecord4", - "type": { - "name": "InnerRecord4", - "type": "record", - "fields": [ - { - "name": "Point", - "type": "long" - } - ] - } - } - ] - } - } - ] - } - } - ] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/big-records/key-distribution.json b/test/bench/util/kafka-avro-generator/distributions/big-records/key-distribution.json deleted file mode 100644 index 2547ba99814ca..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/big-records/key-distribution.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "com.acme.avro.testrecordkey::Key1": [0, 100], - "com.acme.avro.testrecordkey::Key2": [0, 250000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/big-records/key-schema.json b/test/bench/util/kafka-avro-generator/distributions/big-records/key-schema.json deleted file mode 100644 index c156c8ae85838..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/big-records/key-schema.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "testrecordkey", - "type" : "record", - "namespace": "com.acme.avro", - "fields": [ - { - "name": "Key1", - "type": "long" - }, - { - "name": "Key2", - "type": "long" - } - ] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/big-records/value-distribution.json b/test/bench/util/kafka-avro-generator/distributions/big-records/value-distribution.json deleted file mode 100644 index 570d0db5e9d29..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/big-records/value-distribution.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "com.acme.avro.inner::f1": [0, 1000000000], - "com.acme.avro.inner::f2.len": [0, 100], - "com.acme.avro.inner::f3": [1, 2], - "com.acme.avro.inner::f3.1.len": [0, 100], - "com.acme.avro.inner::f4": [0, 1], - "com.acme.avro.inner::f5": [10000, 100000000000], - "com.acme.avro.testrecord::outer_array.len": [0, 1000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/big-records/value-schema.json b/test/bench/util/kafka-avro-generator/distributions/big-records/value-schema.json deleted file mode 100644 index 6901f4369e571..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/big-records/value-schema.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "name": "testrecord", - "type": "record", - "namespace": "com.acme.avro", - "fields": [ - { - "name": "outer_array", - "type": { - "type": "array", - "items": { - "name": "inner", - "type": "record", - "fields": [ - { - "name": "f1", - "type": "int" - }, - { - "name": "f2", - "type": "string" - }, - { - "name": "f3", - "type": ["null", "string"] - }, - { - "name": "f4", - "type": "double" - }, - { - "name": "f5", - "type": "long" - } - ] - } - } - } - ] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/medium/key-distribution.json b/test/bench/util/kafka-avro-generator/distributions/medium/key-distribution.json deleted file mode 100644 index 3e3a2fc6809ca..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/medium/key-distribution.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "com.acme.avro.testrecordkey::Key1": [0, 100], - "com.acme.avro.testrecordkey::Key2": [0, 25000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/medium/key-schema.json b/test/bench/util/kafka-avro-generator/distributions/medium/key-schema.json deleted file mode 120000 index d9466bc11e6ae..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/medium/key-schema.json +++ /dev/null @@ -1 +0,0 @@ -../benchmark/key-schema.json \ No newline at end of file diff --git a/test/bench/util/kafka-avro-generator/distributions/medium/value-distribution.json b/test/bench/util/kafka-avro-generator/distributions/medium/value-distribution.json deleted file mode 100644 index 3ba3d0c924f19..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/medium/value-distribution.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "com.acme.avro.testrecord::Key1Unused": [0, 100], - "com.acme.avro.testrecord::Key2Unused": [0, 25000], - "com.acme.avro.InnerRecord1::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord2::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord3::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord4::Point": [10000, 10000000000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/medium/value-schema.json b/test/bench/util/kafka-avro-generator/distributions/medium/value-schema.json deleted file mode 120000 index a57a3848fd889..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/medium/value-schema.json +++ /dev/null @@ -1 +0,0 @@ -../benchmark/value-schema.json \ No newline at end of file diff --git a/test/bench/util/kafka-avro-generator/distributions/small/key-distribution.json b/test/bench/util/kafka-avro-generator/distributions/small/key-distribution.json deleted file mode 100644 index 638754202d23a..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/small/key-distribution.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "com.acme.avro.testrecordkey::Key1": [0, 10], - "com.acme.avro.testrecordkey::Key2": [0, 25] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/small/key-schema.json b/test/bench/util/kafka-avro-generator/distributions/small/key-schema.json deleted file mode 120000 index d9466bc11e6ae..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/small/key-schema.json +++ /dev/null @@ -1 +0,0 @@ -../benchmark/key-schema.json \ No newline at end of file diff --git a/test/bench/util/kafka-avro-generator/distributions/small/value-distribution.json b/test/bench/util/kafka-avro-generator/distributions/small/value-distribution.json deleted file mode 100644 index f8b9101c54e7e..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/small/value-distribution.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "com.acme.avro.testrecord::Key1Unused": [0, 10], - "com.acme.avro.testrecord::Key2Unused": [0, 25], - "com.acme.avro.InnerRecord1::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord2::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord3::Point": [10000, 1000000000], - "com.acme.avro.InnerRecord4::Point": [10000, 10000000000] -} diff --git a/test/bench/util/kafka-avro-generator/distributions/small/value-schema.json b/test/bench/util/kafka-avro-generator/distributions/small/value-schema.json deleted file mode 120000 index a57a3848fd889..0000000000000 --- a/test/bench/util/kafka-avro-generator/distributions/small/value-schema.json +++ /dev/null @@ -1 +0,0 @@ -../benchmark/value-schema.json \ No newline at end of file diff --git a/test/bench/util/kafka-avro-generator/generate_records b/test/bench/util/kafka-avro-generator/generate_records deleted file mode 100755 index 94142bfc09cb0..0000000000000 --- a/test/bench/util/kafka-avro-generator/generate_records +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -import argparse -import multiprocessing -import pathlib -import subprocess -import sys - -DISTRIBUTIONS = "/usr/share/generator/distributions" - -def run(args: argparse.Namespace) -> int: - """Run the generator, inserting args.num_records number of messages.""" - - records_per_process = int(args.num_records / args.parallelism) - - bootstrap_broker = args.bootstrap_broker - schema_registry_url = args.schema_registry_url - - topic= args.topic - - distribution_dir = pathlib.Path(DISTRIBUTIONS, args.distribution) - key_schema = pathlib.Path(distribution_dir, "key-schema.json").read_text().strip() - value_schema = pathlib.Path(distribution_dir, "value-schema.json").read_text().strip() - - key_distribution = ( - pathlib.Path(distribution_dir, "key-distribution.json").read_text().strip() - ) - value_distribution = ( - pathlib.Path(distribution_dir, "value-distribution.json").read_text().strip() - ) - - kafka_gen = [ - "/usr/local/bin/kgen", - "--quiet", - "--bootstrap-server", - bootstrap_broker, - "--schema-registry-url", - schema_registry_url, - "--num-records", - str(records_per_process), - "--topic", - topic, - "--keys", - "avro", - "--values", - "avro", - "--avro-schema", - value_schema, - "--avro-distribution", - value_distribution, - "--avro-key-schema", - key_schema, - "--avro-key-distribution", - key_distribution, - ] - - print( - f"Spawning {args.parallelism} generator processes, writing {records_per_process} messages each" - ) - procs = [subprocess.Popen(kafka_gen) for _ in range(0, args.parallelism)] - - exit_code = 0 - for (i, p) in enumerate(procs): - p.wait() - print( - f"{i}/{args.parallelism} processes finished: pid={p.pid} returncode={p.returncode}" - ) - - # If a process exited with an error, exit with the return code of the first such process - if p.returncode != 0 and exit_code == 0: - exit_code = p.returncode - - return exit_code - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument( - "-d", - "--distribution", - type=str, - default="benchmark", - help="Name of the distribution to use for generating random records", - choices=[p.name for p in pathlib.Path(DISTRIBUTIONS).iterdir()], - ) - - parser.add_argument( - "-n", - "--num-records", - type=int, - default=400000000, - help="Total number of messages to generate", - ) - parser.add_argument( - "-p", - "--parallelism", - type=int, - default=multiprocessing.cpu_count(), - help="Number of processes to spawn", - ) - - parser.add_argument( - "-b", - "--bootstrap-broker", - type=str, - default="kafka:9092", - help="Kafka bootstrap server", - ) - - parser.add_argument( - "-r", - "--schema-registry-url", - type=str, - default="http://schema-registry:8081", - help="Schema Registry url", - ) - - parser.add_argument( - "-t", - "--topic", - type=str, - help="Kafka topic", - ) - - args = parser.parse_args() - if args.num_records % args.parallelism != 0: - print("ERROR: Number of records must divide evenly by number of processes") - sys.exit(1) - - sys.exit(run(args)) diff --git a/test/bench/util/kafka-avro-generator/mzbuild.yml b/test/bench/util/kafka-avro-generator/mzbuild.yml deleted file mode 100644 index d415c5a7b0769..0000000000000 --- a/test/bench/util/kafka-avro-generator/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: kafka-avro-generator -publish: false diff --git a/test/bench/util/metric-verifier/Dockerfile b/test/bench/util/metric-verifier/Dockerfile deleted file mode 100644 index 7e95354490c02..0000000000000 --- a/test/bench/util/metric-verifier/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -MZFROM ubuntu-base - -ARG DEBIAN_FRONTEND="noninteractive" - -RUN apt-get update && apt-get install -y librdkafka-dev python3-dev python3 python3-pip tzdata && apt-get clean - -COPY requirements.txt /tmp -RUN pip3 install -r /tmp/requirements.txt - -RUN mkdir -p /usr/share/benchmarks/schemas -COPY schemas /usr/share/benchmarks/schemas - -COPY wait_metric /usr/local/bin -ENTRYPOINT ["wait_metric"] diff --git a/test/bench/util/metric-verifier/mzbuild.yml b/test/bench/util/metric-verifier/mzbuild.yml deleted file mode 100644 index bd593d9de80d2..0000000000000 --- a/test/bench/util/metric-verifier/mzbuild.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -name: metric-verifier -publish: false diff --git a/test/bench/util/metric-verifier/requirements.txt b/test/bench/util/metric-verifier/requirements.txt deleted file mode 100644 index e9e3fe5217b5b..0000000000000 --- a/test/bench/util/metric-verifier/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -confluent-kafka[avro]==1.5.0 -prometheus-api-client==0.4.2 -random_name==0.1.1 diff --git a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/key-schema.avsc b/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/key-schema.avsc deleted file mode 100644 index 527b31ca9c3cc..0000000000000 --- a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/key-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "key", - "type" : "record", - "namespace": "dev.mtrlz.benchmarks.runs.begin.v0", - "fields": [ - { - "name": "run_id", - "type": "string" - } - ] -} diff --git a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/value-schema.avsc b/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/value-schema.avsc deleted file mode 100644 index 0b4fdd7d7e8ba..0000000000000 --- a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.begin.v0/value-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "value", - "type" : "record", - "namespace": "dev.mtrlz.benchmarks.runs.begin.v0", - "fields": [ - { - "name": "start_ms", - "type": "long" - } - ] -} diff --git a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/key-schema.avsc b/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/key-schema.avsc deleted file mode 100644 index 8c97e1f6ac443..0000000000000 --- a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/key-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "key", - "type" : "record", - "namespace": "dev.mtrlz.benchmarks.runs.params.v0", - "fields": [ - { - "name": "run_id", - "type": "string" - } - ] -} diff --git a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/value-schema.avsc b/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/value-schema.avsc deleted file mode 100644 index 31eadb8a16e9f..0000000000000 --- a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.params.v0/value-schema.avsc +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "value", - "type" : "record", - "namespace": "dev.mtrlz.benchmarks.runs.params.v0", - "fields": [ - { - "name": "benchmark_id", - "type": "string" - }, - { - "name": "git_ref", - "type": "string" - }, - { - "name": "mz_workers", - "type": "int" - } - ] -} diff --git a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/key-schema.avsc b/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/key-schema.avsc deleted file mode 100644 index 0d2f816cb7110..0000000000000 --- a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/key-schema.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "key", - "type" : "record", - "namespace": "dev.mtrlz.benchmarks.runs.results.v0", - "fields": [ - { - "name": "run_id", - "type": "string" - } - ] -} diff --git a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/value-schema.avsc b/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/value-schema.avsc deleted file mode 100644 index 9d9bd8740f5c2..0000000000000 --- a/test/bench/util/metric-verifier/schemas/dev.mtrlz.benchmarks.runs.results.v0/value-schema.avsc +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "value", - "type" : "record", - "namespace": "dev.mtrlz.benchmarks.runs.results.v0", - "fields": [ - { - "name": "result", - "type": "string" - }, - { - "name": "end_ms", - "type": "long" - }, - { - "name": "rows_per_second", - "type": "long" - } - ] -} diff --git a/test/bench/util/metric-verifier/wait_metric b/test/bench/util/metric-verifier/wait_metric deleted file mode 100755 index a6f3c9c0b2271..0000000000000 --- a/test/bench/util/metric-verifier/wait_metric +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright Materialize, Inc. and contributors. All rights reserved. -# -# Use of this software is governed by the Business Source License -# included in the LICENSE file at the root of this repository. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0. - -import argparse -import datetime -import io -import math -import os -import pathlib -import sys -import time -import typing -import urllib -import uuid - -import confluent_kafka.admin -import confluent_kafka.avro -import confluent_kafka.schema_registry -import dateparser -import prometheus_api_client -import prometheus_api_client.utils -import requests - -METRICS_DIRECTORY = "/usr/share/benchmarks/schemas" - - -def create_topics(args: argparse.Namespace) -> None: - """Create the topics for each metric and exit.""" - - csr_config = {'url': f'http://{args.csr_host}:{args.csr_port}'} - csr = confluent_kafka.schema_registry.SchemaRegistryClient(csr_config) - - kafka_config = {'bootstrap.servers': f"{args.kafka_host}:{args.kafka_port}"} - admin_client = confluent_kafka.admin.AdminClient(kafka_config) - - for metric_dir in pathlib.Path(args.metrics_directory).iterdir(): - - key_schema_str = pathlib.Path(metric_dir, 'key-schema.avsc').read_text().strip() - key_schema = confluent_kafka.schema_registry.Schema(key_schema_str, 'AVRO') - value_schema_str = pathlib.Path(metric_dir, 'value-schema.avsc').read_text().strip() - value_schema = confluent_kafka.schema_registry.Schema(value_schema_str, 'AVRO') - - csr.register_schema(f"{metric_dir.name}-key", key_schema) - csr.register_schema(f"{metric_dir.name}-value", value_schema) - - # Create topics takes and returns a list of futures but we're going to call create topic - # while iterating on each directory, so these are just lists of 1 - topics = [confluent_kafka.admin.NewTopic(metric_dir.name, num_partitions=10, replication_factor=1)] - # Don't bother trying to catch the error, let's just fail startup and raise the error - [future.result() for future in admin_client.create_topics(topics).values()] - - -class Producer: - - def __init__(self, topic_dir: pathlib.Path, kafka_config: typing.Dict[str, str]): - """Construct a producer for writing to the given topic, using the given config.""" - self.topic_name = topic_dir.name - - key_schema = confluent_kafka.avro.load(pathlib.Path(topic_dir, 'key-schema.avsc')) - value_schema = confluent_kafka.avro.load(pathlib.Path(topic_dir, 'value-schema.avsc')) - - # Namespace for both schemas should be the name of the topic - assert key_schema.namespace == value_schema.namespace - assert self.topic_name == key_schema.namespace - - self.producer = confluent_kafka.avro.AvroProducer(kafka_config, - default_key_schema=key_schema, - default_value_schema=value_schema) - - def write_metric(self, key: typing.Any, value: typing.Any) -> None: - """Encode key and value using Avro and send event to Kafka.""" - self.producer.produce(topic=self.topic_name, key=key, value=value) - self.producer.flush() - - -class BenchmarkMetrics: - - def __init__(self, args: argparse.Namespace, schema_directory: str): - """Create an instance responsible for recording benchmark metrics.""" - self.args = args - self.schema_directory = schema_directory - self.producers: typing.Dict[str, Producer] = {} - - # See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - kafka_config = {'bootstrap.servers': f"{args.kafka_host}:{args.kafka_port}", - 'schema.registry.url': f'http://{args.csr_host}:{args.csr_port}'} - for topic_dir in pathlib.Path(self.schema_directory).iterdir(): - self.producers[topic_dir.name] = Producer(topic_dir, kafka_config) - - def write_metric(self, topic: str, key: typing.Any, value: typing.Any) -> None: - """Encode key and value using Avro and send event to Kafka.""" - self.producers[topic].write_metric(key, value) - - -def scrape(args: argparse.Namespace) -> int: - """Wait for the query to settle or timeout and then dump ingest metrics.""" - - start = dateparser.parse('now') - metrics = BenchmarkMetrics(args, args.metrics_directory) - - # Record this as two distinct metrics. Ideally, the first would be recorded by mzbench - # when it's populating the list of benchmarks to run. Then second would then be recorded by - # this runner, which may or may not be the same process / machine as the one enumerating the - # parameter space - metrics.write_metric('dev.mtrlz.benchmarks.runs.params.v0', - {"run_id": args.mzbench_run_id}, - {"benchmark_id": args.mzbench_id, - "git_ref": args.mzbench_git_ref, - "mz_workers": args.mz_workers}) - metrics.write_metric('dev.mtrlz.benchmarks.runs.begin.v0', - {"run_id": args.mzbench_run_id}, - {"start_ms": int(time.time() * 1000)}) - - try: - return wait_metric(args, metrics, start) - except: - record_results(args, metrics, start, result="error") - raise - - -def record_results(args: argparse.Namespace, metrics: BenchmarkMetrics, start: datetime.datetime, - rows_per_second: int = 0, result: str = "passed"): - """Write the results of this benchmark to the Kafka topic""" - metrics.write_metric('dev.mtrlz.benchmarks.runs.results.v0', - {"run_id": args.mzbench_run_id}, - {"end_ms": int(time.time() * 1000), - "rows_per_second": rows_per_second, - "result": result}) - - # TODO: Construct the URL externally from this script, maintain for compat with mzbench - # Create parameters to see a dashboard with the metrics from this benchmark run - # Add padding to make the charts nicer to read - # Grafana expects timestamps with milliseconds - path = '/d/materialize-overview/materialize-overview' - query = urllib.parse.urlencode( { - "from": round((start - datetime.timedelta(seconds=30)).timestamp()) * 1000, - "to": round(dateparser.parse('in 30 seconds').timestamp()) * 1000, - "tz": "UTC" - }) - dashboard_url = urllib.parse.urlunparse(('http', args.grafana_location, path, '', query, '')) - - print(f'Grafana URL: {dashboard_url}') - - -def wait_metric(args: argparse.Namespace, metrics: BenchmarkMetrics, start: datetime.datetime) -> int: - """Wait for the given metric, returning desired exit code (0 is success).""" - prom = prometheus_api_client.PrometheusConnect(f'http://{args.prometheus_host}:{args.prometheus_port}') - time_so_far = 0 - begin = int(time.monotonic()) - while time_so_far < args.timeout_seconds: - current_values = prom.custom_query(args.prom_query) - if args.verbose: - print(current_values) - - if len(current_values) > 1: - print('ERROR: Prometheus query must only return a zero or one results!') - prometheus_api_client.utils.pretty_print_metric(current_values) - return 1 - - # We aren't running query_range, so there should only be a single timestamp and point in the reponse - if current_values: - (ts, point) = [float(i) for i in current_values[0]['value']] - if point == args.expected_value: - rate = round(point / max(time_so_far, 1)) - print(f"SUCCESS! seconds_taken={time_so_far} rows_per_sec={rate}") - record_results(args, metrics, start, rate) - return 0 - - time.sleep(1) - time_so_far = int(time.monotonic()) - begin - - # Check this last because it's okay to have a 1-2 second grace period and we want the - # ability to print the most recent result - print(f"FAILED! Query response is '{point}' after {time_so_far} seconds") - return 1 - - -def run(args: argparse.Namespace) -> None: - """Run the desired command.""" - return args.action(args) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - - parser.add_argument( - "-v", - "--verbose", - action="store_true", - help="Enable verbose logging to print the results of each prometheus query", - ) - - parser.add_argument( - "--metrics-directory", - type=str, - default=METRICS_DIRECTORY, - help="Directory containing metrics definitions and source queries", - ) - - parser.add_argument( - "--kafka-host", - help="Name of the kafka broker", - type=str, - default="kafka", - ) - - parser.add_argument( - "--kafka-port", help="Port the connect to the broker over", type=int, default=9092 - ) - - parser.add_argument( - "--csr-host", help="Hostname of the schema registry", type=str, - default="schema-registry" - ) - parser.add_argument( - "--csr-port", help="Port that schema registry is listening on", type=int, - default=8081 - ) - - sub_parsers = parser.add_subparsers() - - # Create topics subcommand and flags - create_topic_parser = sub_parsers.add_parser("create-topics") - create_topic_parser.set_defaults(action=create_topics) - - # Scrape subcommand and flags - scrape_parser = sub_parsers.add_parser("scrape") - scrape_parser.set_defaults(action=scrape) - - scrape_parser.add_argument( - "--grafana-location", - type=str, - default='localhost:3000', - help="Default URL net location (host and port) for Grafana", - ) - - scrape_parser.add_argument( - "--mz-workers", - type=int, - default=os.environ.get('MZ_WORKERS', 0), - help="How many workers materialized is configured to use", - ) - - scrape_parser.add_argument( - "--mzbench-git-ref", - type=str, - default=os.environ.get('MZBENCH_GIT_REF', 'undefined'), - help="The materialized image tag", - ) - - scrape_parser.add_argument( - "--mzbench-id", - type=str, - default=os.environ.get('MZBENCH_ID', str(uuid.uuid4())), - help="Unique string that identifies the grouping of benchmark runs", - ) - - scrape_parser.add_argument( - "--mzbench-run-id", - type=str, - default=os.environ.get('MZBENCH_RUN_ID', str(uuid.uuid4())), - help="Unique string that identifies this particular benchmark run", - ) - - scrape_parser.add_argument( - "--prometheus-host", - type=str, - default='dashboard', - help="Hostname of the prometheus instance to query", - ) - - scrape_parser.add_argument( - "--prometheus-port", - type=int, - default=9090, - help="Port on which the prometheus instance is running", - ) - - scrape_parser.add_argument( - "-t", - "--timeout-seconds", - type=int, - default=900, - help="Length of time to wait until the metric reaches the specified value", - ) - - scrape_parser.add_argument( - "prom_query", - type=str, - help="Prometheus query to run", - ) - - scrape_parser.add_argument( - "expected_value", - type=float, - help="Expected value of the metric queried", - ) - - args = parser.parse_args() - return args - -if __name__ == '__main__': - sys.exit(run(parse_args()))